1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.owasp.dependencycheck.utils;
19
20 import java.io.BufferedOutputStream;
21 import java.io.File;
22 import java.io.FileOutputStream;
23 import java.io.IOException;
24 import java.io.InputStream;
25 import java.net.HttpURLConnection;
26 import java.net.URISyntaxException;
27 import java.net.URL;
28 import java.security.InvalidAlgorithmParameterException;
29 import java.util.logging.Level;
30 import java.util.logging.Logger;
31 import java.util.zip.GZIPInputStream;
32 import java.util.zip.InflaterInputStream;
33
34
35
36
37
38
39 public final class Downloader {
40
41
42
43
44 private static final Logger LOGGER = Logger.getLogger(Downloader.class.getName());
45
46
47
48 private static final int MAX_REDIRECT_ATTEMPTS = 5;
49
50
51
52
53 private Downloader() {
54 }
55
56
57
58
59
60
61
62
63 public static void fetchFile(URL url, File outputPath) throws DownloadFailedException {
64 fetchFile(url, outputPath, true);
65 }
66
67
68
69
70
71
72
73
74
75 public static void fetchFile(URL url, File outputPath, boolean useProxy) throws DownloadFailedException {
76 if ("file".equalsIgnoreCase(url.getProtocol())) {
77 File file;
78 try {
79 file = new File(url.toURI());
80 } catch (URISyntaxException ex) {
81 final String msg = String.format("Download failed, unable to locate '%s'", url.toString());
82 throw new DownloadFailedException(msg);
83 }
84 if (file.exists()) {
85 try {
86 org.apache.commons.io.FileUtils.copyFile(file, outputPath);
87 } catch (IOException ex) {
88 final String msg = String.format("Download failed, unable to copy '%s' to '%s'", url.toString(), outputPath.getAbsolutePath());
89 throw new DownloadFailedException(msg);
90 }
91 } else {
92 final String msg = String.format("Download failed, file ('%s') does not exist", url.toString());
93 throw new DownloadFailedException(msg);
94 }
95 } else {
96 HttpURLConnection conn = null;
97 try {
98 LOGGER.fine(String.format("Attempting download of %s", url.toString()));
99 conn = URLConnectionFactory.createHttpURLConnection(url, useProxy);
100 conn.setRequestProperty("Accept-Encoding", "gzip, deflate");
101 conn.connect();
102 int status = conn.getResponseCode();
103 int redirectCount = 0;
104 while ((status == HttpURLConnection.HTTP_MOVED_TEMP
105 || status == HttpURLConnection.HTTP_MOVED_PERM
106 || status == HttpURLConnection.HTTP_SEE_OTHER)
107 && MAX_REDIRECT_ATTEMPTS > redirectCount++) {
108 final String location = conn.getHeaderField("Location");
109 try {
110 conn.disconnect();
111 } finally {
112 conn = null;
113 }
114 LOGGER.fine(String.format("Download is being redirected from %s to %s", url.toString(), location));
115 conn = URLConnectionFactory.createHttpURLConnection(new URL(location), useProxy);
116 conn.setRequestProperty("Accept-Encoding", "gzip, deflate");
117 conn.connect();
118 status = conn.getResponseCode();
119 }
120 if (status != 200) {
121 try {
122 conn.disconnect();
123 } finally {
124 conn = null;
125 }
126 final String msg = String.format("Error downloading file %s; received response code %s.", url.toString(), status);
127 throw new DownloadFailedException(msg);
128
129 }
130 } catch (IOException ex) {
131 try {
132 if (conn != null) {
133 conn.disconnect();
134 }
135 } finally {
136 conn = null;
137 }
138 final String msg = String.format("Error downloading file %s; unable to connect.", url.toString());
139 throw new DownloadFailedException(msg, ex);
140 }
141
142 final String encoding = conn.getContentEncoding();
143 BufferedOutputStream writer = null;
144 InputStream reader = null;
145 try {
146 if (encoding != null && "gzip".equalsIgnoreCase(encoding)) {
147 reader = new GZIPInputStream(conn.getInputStream());
148 } else if (encoding != null && "deflate".equalsIgnoreCase(encoding)) {
149 reader = new InflaterInputStream(conn.getInputStream());
150 } else {
151 reader = conn.getInputStream();
152 }
153
154 writer = new BufferedOutputStream(new FileOutputStream(outputPath));
155 final byte[] buffer = new byte[4096];
156 int bytesRead;
157 while ((bytesRead = reader.read(buffer)) > 0) {
158 writer.write(buffer, 0, bytesRead);
159 }
160 LOGGER.fine(String.format("Download of %s complete", url.toString()));
161 } catch (IOException ex) {
162 analyzeException(ex);
163 final String msg = String.format("Error saving '%s' to file '%s'%nConnection Timeout: %d%nEncoding: %s%n",
164 url.toString(), outputPath.getAbsolutePath(), conn.getConnectTimeout(), encoding);
165 throw new DownloadFailedException(msg, ex);
166 } catch (Throwable ex) {
167 final String msg = String.format("Unexpected exception saving '%s' to file '%s'%nConnection Timeout: %d%nEncoding: %s%n",
168 url.toString(), outputPath.getAbsolutePath(), conn.getConnectTimeout(), encoding);
169 throw new DownloadFailedException(msg, ex);
170 } finally {
171 if (writer != null) {
172 try {
173 writer.close();
174 } catch (IOException ex) {
175 LOGGER.log(Level.FINEST, "Error closing the writer in Downloader.", ex);
176 }
177 }
178 if (reader != null) {
179 try {
180 reader.close();
181 } catch (IOException ex) {
182 LOGGER.log(Level.FINEST, "Error closing the reader in Downloader.", ex);
183 }
184 }
185 try {
186 conn.disconnect();
187 } finally {
188 conn = null;
189 }
190 }
191 }
192 }
193
194
195
196
197
198
199
200
201
202 public static long getLastModified(URL url) throws DownloadFailedException {
203 long timestamp = 0;
204
205 if ("file".equalsIgnoreCase(url.getProtocol())) {
206 File lastModifiedFile;
207 try {
208 lastModifiedFile = new File(url.toURI());
209 } catch (URISyntaxException ex) {
210 final String msg = String.format("Unable to locate '%s'", url.toString());
211 throw new DownloadFailedException(msg);
212 }
213 timestamp = lastModifiedFile.lastModified();
214 } else {
215 HttpURLConnection conn = null;
216 try {
217 conn = URLConnectionFactory.createHttpURLConnection(url);
218 conn.setRequestMethod("HEAD");
219 conn.connect();
220 final int t = conn.getResponseCode();
221 if (t >= 200 && t < 300) {
222 timestamp = conn.getLastModified();
223 } else {
224 throw new DownloadFailedException("HEAD request returned a non-200 status code");
225 }
226 } catch (URLConnectionFailureException ex) {
227 throw new DownloadFailedException("Error creating URL Connection for HTTP HEAD request.", ex);
228 } catch (IOException ex) {
229 analyzeException(ex);
230 throw new DownloadFailedException("Error making HTTP HEAD request.", ex);
231 } finally {
232 if (conn != null) {
233 try {
234 conn.disconnect();
235 } finally {
236 conn = null;
237 }
238 }
239 }
240 }
241 return timestamp;
242 }
243
244
245
246
247
248
249
250
251 protected static void analyzeException(IOException ex) throws DownloadFailedException {
252 Throwable cause = ex;
253 while (cause != null) {
254 if (cause instanceof InvalidAlgorithmParameterException) {
255 final String keystore = System.getProperty("javax.net.ssl.keyStore");
256 final String version = System.getProperty("java.version");
257 final String vendor = System.getProperty("java.vendor");
258 LOGGER.info("Error making HTTPS request - InvalidAlgorithmParameterException");
259 LOGGER.info("There appears to be an issue with the installation of Java and the cacerts."
260 + "See closed issue #177 here: https://github.com/jeremylong/DependencyCheck/issues/177");
261 LOGGER.info(String.format("Java Info:%njavax.net.ssl.keyStore='%s'%njava.version='%s'%njava.vendor='%s'",
262 keystore, version, vendor));
263 throw new DownloadFailedException("Error making HTTPS request. Please see the log for more details.");
264 }
265 cause = cause.getCause();
266 }
267 }
268 }