Coverage Report

Coverage Report - org.owasp.dependencycheck.utils.Downloader

Classes in this File

Line Coverage

Branch Coverage

Complexity

Downloader

8/141

1/46

7.857

 /*
  * This file is part of dependency-check-core.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
  *
  *     http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
  *
  * Copyright (c) 2012 Jeremy Long. All Rights Reserved.
  */
 package org.owasp.dependencycheck.utils;
 
 import java.io.BufferedOutputStream;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import java.io.File;
 import java.io.FileOutputStream;
 import java.io.IOException;
 import java.io.InputStream;
 import java.net.HttpURLConnection;
 import java.net.URISyntaxException;
 import java.net.URL;
 import java.security.InvalidAlgorithmParameterException;
 import java.util.zip.GZIPInputStream;
 import java.util.zip.InflaterInputStream;
 
 import static java.lang.String.format;
 
 /**
  * A utility to download files from the Internet.
  *
  * @author Jeremy Long
  */
 public final class Downloader {
 
     /**
      * The logger.
      */
     private static final Logger LOGGER = LoggerFactory.getLogger(Downloader.class);
     /**
      * The maximum number of redirects that will be followed when attempting to download a file.
      */
     private static final int MAX_REDIRECT_ATTEMPTS = 5;
 
     /**
      * The default HTTP request method for query timestamp
      */
     private static final String HEAD = "HEAD";
 
     /**
      * The HTTP request method which can be used by query timestamp
      */
     private static final String GET = "GET";
 
     /**
      * Private constructor for utility class.
      */
     private Downloader() {
     }
 
     /**
      * Retrieves a file from a given URL and saves it to the outputPath.
      *
      * @param url the URL of the file to download
      * @param outputPath the path to the save the file to
      * @throws DownloadFailedException is thrown if there is an error downloading the file
      */
     public static void fetchFile(URL url, File outputPath) throws DownloadFailedException {
         fetchFile(url, outputPath, true);
     }
 
     /**
      * Retrieves a file from a given URL and saves it to the outputPath.
      *
      * @param url the URL of the file to download
      * @param outputPath the path to the save the file to
      * @param useProxy whether to use the configured proxy when downloading files
      * @throws DownloadFailedException is thrown if there is an error downloading the file
      */
     public static void fetchFile(URL url, File outputPath, boolean useProxy) throws DownloadFailedException {
         if ("file".equalsIgnoreCase(url.getProtocol())) {
             File file;
             try {
                 file = new File(url.toURI());
             } catch (URISyntaxException ex) {
                 final String msg = format("Download failed, unable to locate '%s'", url.toString());
                 throw new DownloadFailedException(msg);
             }
             if (file.exists()) {
                 try {
                     org.apache.commons.io.FileUtils.copyFile(file, outputPath);
                 } catch (IOException ex) {
                     final String msg = format("Download failed, unable to copy '%s' to '%s'", url.toString(), outputPath.getAbsolutePath());
                     throw new DownloadFailedException(msg);
                 }
             } else {
                 final String msg = format("Download failed, file ('%s') does not exist", url.toString());
                 throw new DownloadFailedException(msg);
             }
         } else {
             HttpURLConnection conn = null;
             try {
                 LOGGER.debug("Attempting download of {}", url.toString());
                 conn = URLConnectionFactory.createHttpURLConnection(url, useProxy);
                 conn.setRequestProperty("Accept-Encoding", "gzip, deflate");
                 conn.connect();
                 int status = conn.getResponseCode();
                 int redirectCount = 0;
                 while ((status == HttpURLConnection.HTTP_MOVED_TEMP
                         || status == HttpURLConnection.HTTP_MOVED_PERM
                         || status == HttpURLConnection.HTTP_SEE_OTHER)
                         && MAX_REDIRECT_ATTEMPTS > redirectCount++) {
                     final String location = conn.getHeaderField("Location");
                     try {
                         conn.disconnect();
                     } finally {
                         conn = null;
                     }
                     LOGGER.debug("Download is being redirected from {} to {}", url.toString(), location);
                     conn = URLConnectionFactory.createHttpURLConnection(new URL(location), useProxy);
                     conn.setRequestProperty("Accept-Encoding", "gzip, deflate");
                     conn.connect();
                     status = conn.getResponseCode();
                 }
                 if (status != 200) {
                     try {
                         conn.disconnect();
                     } finally {
                         conn = null;
                     }
                     final String msg = format("Error downloading file %s; received response code %s.", url.toString(), status);
                     throw new DownloadFailedException(msg);
 
                 }
             } catch (IOException ex) {
                 try {
                     if (conn != null) {
                         conn.disconnect();
                     }
                 } finally {
                     conn = null;
                 }
                 final String msg = format("Error downloading file %s; unable to connect.", url.toString());
                 throw new DownloadFailedException(msg, ex);
             }
 
             final String encoding = conn.getContentEncoding();
             BufferedOutputStream writer = null;
             InputStream reader = null;
             try {
                 if (encoding != null && "gzip".equalsIgnoreCase(encoding)) {
                     reader = new GZIPInputStream(conn.getInputStream());
                 } else if (encoding != null && "deflate".equalsIgnoreCase(encoding)) {
                     reader = new InflaterInputStream(conn.getInputStream());
                 } else {
                     reader = conn.getInputStream();
                 }
 
                 writer = new BufferedOutputStream(new FileOutputStream(outputPath));
                 final byte[] buffer = new byte[4096];
                 int bytesRead;
                 while ((bytesRead = reader.read(buffer)) > 0) {
                     writer.write(buffer, 0, bytesRead);
                 }
                 LOGGER.debug("Download of {} complete", url.toString());
             } catch (IOException ex) {
                 analyzeException(ex);
                 final String msg = format("Error saving '%s' to file '%s'%nConnection Timeout: %d%nEncoding: %s%n",
                         url.toString(), outputPath.getAbsolutePath(), conn.getConnectTimeout(), encoding);
                 throw new DownloadFailedException(msg, ex);
             } catch (Throwable ex) {
                 final String msg = format("Unexpected exception saving '%s' to file '%s'%nConnection Timeout: %d%nEncoding: %s%n",
                         url.toString(), outputPath.getAbsolutePath(), conn.getConnectTimeout(), encoding);
                 throw new DownloadFailedException(msg, ex);
             } finally {
                 if (writer != null) {
                     try {
                         writer.close();
                     } catch (IOException ex) {
                         LOGGER.trace("Error closing the writer in Downloader.", ex);
                     }
                 }
                 if (reader != null) {
                     try {
                         reader.close();
                     } catch (IOException ex) {
                         LOGGER.trace("Error closing the reader in Downloader.", ex);
                     }
                 }
                 try {
                     conn.disconnect();
                 } finally {
                     conn = null;
                 }
             }
         }
     }
 
     /**
      * Makes an HTTP Head request to retrieve the last modified date of the given URL. If the file:// protocol is specified, then
      * the lastTimestamp of the file is returned.
      *
      * @param url the URL to retrieve the timestamp from
      * @return an epoch timestamp
      * @throws DownloadFailedException is thrown if an exception occurs making the HTTP request
      */
     public static long getLastModified(URL url) throws DownloadFailedException {
         long timestamp = 0;
         //TODO add the FTP protocol?
         if ("file".equalsIgnoreCase(url.getProtocol())) {
             File lastModifiedFile;
             try {
                 lastModifiedFile = new File(url.toURI());
             } catch (URISyntaxException ex) {
                 final String msg = format("Unable to locate '%s'", url.toString());
                 throw new DownloadFailedException(msg);
             }
             timestamp = lastModifiedFile.lastModified();
         } else {
             final String httpMethod = determineHttpMethod();
             HttpURLConnection conn = null;
             try {
                 conn = URLConnectionFactory.createHttpURLConnection(url);
                 conn.setRequestMethod(httpMethod);
                 conn.connect();
                 final int t = conn.getResponseCode();
                 if (t >= 200 && t < 300) {
                     timestamp = conn.getLastModified();
                 } else {
                     throw new DownloadFailedException(format("%s request returned a non-200 status code", httpMethod));
                 }
             } catch (URLConnectionFailureException ex) {
                 throw new DownloadFailedException(format("Error creating URL Connection for HTTP %s request.", httpMethod), ex);
             } catch (IOException ex) {
                 analyzeException(ex);
                 try {
                     //retry
                     if (!Settings.getBoolean(Settings.KEYS.DOWNLOADER_QUICK_QUERY_TIMESTAMP)) {
                         Settings.setBoolean(Settings.KEYS.DOWNLOADER_QUICK_QUERY_TIMESTAMP, true);
                         return getLastModified(url);
                     }
                 } catch (InvalidSettingException ex1) {
                     LOGGER.debug("invalid setting?", ex);
                 }
 
                 throw new DownloadFailedException(format("Error making HTTP %s request.", httpMethod), ex);
             } finally {
                 if (conn != null) {
                     try {
                         conn.disconnect();
                     } finally {
                         conn = null;
                     }
                 }
             }
         }
         return timestamp;
     }
 
     /**
      * Analyzes the IOException, logs the appropriate information for debugging purposes, and then throws a
      * DownloadFailedException that wraps the IO Exception.
      *
      * @param ex the original exception
      * @throws DownloadFailedException a wrapper exception that contains the original exception as the cause
      */
     protected static void analyzeException(IOException ex) throws DownloadFailedException {
         Throwable cause = ex;
         while (cause != null) {
             if (cause instanceof InvalidAlgorithmParameterException) {
                 final String keystore = System.getProperty("javax.net.ssl.keyStore");
                 final String version = System.getProperty("java.version");
                 final String vendor = System.getProperty("java.vendor");
                 LOGGER.info("Error making HTTPS request - InvalidAlgorithmParameterException");
                 LOGGER.info("There appears to be an issue with the installation of Java and the cacerts."
                         + "See closed issue #177 here: https://github.com/jeremylong/DependencyCheck/issues/177");
                 LOGGER.info("Java Info:\njavax.net.ssl.keyStore='{}'\njava.version='{}'\njava.vendor='{}'",
                         keystore, version, vendor);
                 throw new DownloadFailedException("Error making HTTPS request. Please see the log for more details.");
             }
             cause = cause.getCause();
         }
     }
 
     /**
      * Returns the HEAD or GET HTTP method. HEAD is the default.
      *
      * @return the HTTP method to use
      */
     private static String determineHttpMethod() {
         return isQuickQuery() ? HEAD : GET;
     }
 
     /**
      * Determines if the HTTP method GET or HEAD should be used to check the timestamp on external resources.
      *
      * @return true if configured to use HEAD requests
      */
     private static boolean isQuickQuery() {
         boolean quickQuery;
 
         try {
             quickQuery = Settings.getBoolean(Settings.KEYS.DOWNLOADER_QUICK_QUERY_TIMESTAMP, true);
         } catch (InvalidSettingException e) {
             quickQuery = true;
         }
         return quickQuery;
     }
 }

1		/*
2		* This file is part of dependency-check-core.
3		*
4		* Licensed under the Apache License, Version 2.0 (the "License");
5		* you may not use this file except in compliance with the License.
6		* You may obtain a copy of the License at
7		*
8		* http://www.apache.org/licenses/LICENSE-2.0
9		*
10		* Unless required by applicable law or agreed to in writing, software
11		* distributed under the License is distributed on an "AS IS" BASIS,
12		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13		* See the License for the specific language governing permissions and
14		* limitations under the License.
15		*
16		* Copyright (c) 2012 Jeremy Long. All Rights Reserved.
17		*/
18		package org.owasp.dependencycheck.utils;
19
20		import java.io.BufferedOutputStream;
21		import org.slf4j.Logger;
22		import org.slf4j.LoggerFactory;
23
24		import java.io.File;
25		import java.io.FileOutputStream;
26		import java.io.IOException;
27		import java.io.InputStream;
28		import java.net.HttpURLConnection;
29		import java.net.URISyntaxException;
30		import java.net.URL;
31		import java.security.InvalidAlgorithmParameterException;
32		import java.util.zip.GZIPInputStream;
33		import java.util.zip.InflaterInputStream;
34
35		import static java.lang.String.format;
36
37		/**
38		* A utility to download files from the Internet.
39		*
40		* @author Jeremy Long
41		*/
42		public final class Downloader {
43
44		/**
45		* The logger.
46		*/
47	1	private static final Logger LOGGER = LoggerFactory.getLogger(Downloader.class);
48		/**
49		* The maximum number of redirects that will be followed when attempting to download a file.
50		*/
51		private static final int MAX_REDIRECT_ATTEMPTS = 5;
52
53		/**
54		* The default HTTP request method for query timestamp
55		*/
56		private static final String HEAD = "HEAD";
57
58		/**
59		* The HTTP request method which can be used by query timestamp
60		*/
61		private static final String GET = "GET";
62
63		/**
64		* Private constructor for utility class.
65		*/
66	0	private Downloader() {
67	0	}
68
69		/**
70		* Retrieves a file from a given URL and saves it to the outputPath.
71		*
72		* @param url the URL of the file to download
73		* @param outputPath the path to the save the file to
74		* @throws DownloadFailedException is thrown if there is an error downloading the file
75		*/
76		public static void fetchFile(URL url, File outputPath) throws DownloadFailedException {
77	0	fetchFile(url, outputPath, true);
78	0	}
79
80		/**
81		* Retrieves a file from a given URL and saves it to the outputPath.
82		*
83		* @param url the URL of the file to download
84		* @param outputPath the path to the save the file to
85		* @param useProxy whether to use the configured proxy when downloading files
86		* @throws DownloadFailedException is thrown if there is an error downloading the file
87		*/
88		public static void fetchFile(URL url, File outputPath, boolean useProxy) throws DownloadFailedException {
89	0	if ("file".equalsIgnoreCase(url.getProtocol())) {
90		File file;
91		try {
92	0	file = new File(url.toURI());
93	0	} catch (URISyntaxException ex) {
94	0	final String msg = format("Download failed, unable to locate '%s'", url.toString());
95	0	throw new DownloadFailedException(msg);
96	0	}
97	0	if (file.exists()) {
98		try {
99	0	org.apache.commons.io.FileUtils.copyFile(file, outputPath);
100	0	} catch (IOException ex) {
101	0	final String msg = format("Download failed, unable to copy '%s' to '%s'", url.toString(), outputPath.getAbsolutePath());
102	0	throw new DownloadFailedException(msg);
103	0	}
104		} else {
105	0	final String msg = format("Download failed, file ('%s') does not exist", url.toString());
106	0	throw new DownloadFailedException(msg);
107		}
108	0	} else {
109	0	HttpURLConnection conn = null;
110		try {
111	0	LOGGER.debug("Attempting download of {}", url.toString());
112	0	conn = URLConnectionFactory.createHttpURLConnection(url, useProxy);
113	0	conn.setRequestProperty("Accept-Encoding", "gzip, deflate");
114	0	conn.connect();
115	0	int status = conn.getResponseCode();
116	0	int redirectCount = 0;
117	0	while ((status == HttpURLConnection.HTTP_MOVED_TEMP
118		\|\| status == HttpURLConnection.HTTP_MOVED_PERM
119		\|\| status == HttpURLConnection.HTTP_SEE_OTHER)
120		&& MAX_REDIRECT_ATTEMPTS > redirectCount++) {
121	0	final String location = conn.getHeaderField("Location");
122		try {
123	0	conn.disconnect();
124		} finally {
125	0	conn = null;
126	0	}
127	0	LOGGER.debug("Download is being redirected from {} to {}", url.toString(), location);
128	0	conn = URLConnectionFactory.createHttpURLConnection(new URL(location), useProxy);
129	0	conn.setRequestProperty("Accept-Encoding", "gzip, deflate");
130	0	conn.connect();
131	0	status = conn.getResponseCode();
132	0	}
133	0	if (status != 200) {
134		try {
135	0	conn.disconnect();
136		} finally {
137	0	conn = null;
138	0	}
139	0	final String msg = format("Error downloading file %s; received response code %s.", url.toString(), status);
140	0	throw new DownloadFailedException(msg);
141
142		}
143	0	} catch (IOException ex) {
144		try {
145	0	if (conn != null) {
146	0	conn.disconnect();
147		}
148		} finally {
149	0	conn = null;
150	0	}
151	0	final String msg = format("Error downloading file %s; unable to connect.", url.toString());
152	0	throw new DownloadFailedException(msg, ex);
153	0	}
154
155	0	final String encoding = conn.getContentEncoding();
156	0	BufferedOutputStream writer = null;
157	0	InputStream reader = null;
158		try {
159	0	if (encoding != null && "gzip".equalsIgnoreCase(encoding)) {
160	0	reader = new GZIPInputStream(conn.getInputStream());
161	0	} else if (encoding != null && "deflate".equalsIgnoreCase(encoding)) {
162	0	reader = new InflaterInputStream(conn.getInputStream());
163		} else {
164	0	reader = conn.getInputStream();
165		}
166
167	0	writer = new BufferedOutputStream(new FileOutputStream(outputPath));
168	0	final byte[] buffer = new byte[4096];
169		int bytesRead;
170	0	while ((bytesRead = reader.read(buffer)) > 0) {
171	0	writer.write(buffer, 0, bytesRead);
172		}
173	0	LOGGER.debug("Download of {} complete", url.toString());
174	0	} catch (IOException ex) {
175	0	analyzeException(ex);
176	0	final String msg = format("Error saving '%s' to file '%s'%nConnection Timeout: %d%nEncoding: %s%n",
177	0	url.toString(), outputPath.getAbsolutePath(), conn.getConnectTimeout(), encoding);
178	0	throw new DownloadFailedException(msg, ex);
179	0	} catch (Throwable ex) {
180	0	final String msg = format("Unexpected exception saving '%s' to file '%s'%nConnection Timeout: %d%nEncoding: %s%n",
181	0	url.toString(), outputPath.getAbsolutePath(), conn.getConnectTimeout(), encoding);
182	0	throw new DownloadFailedException(msg, ex);
183		} finally {
184	0	if (writer != null) {
185		try {
186	0	writer.close();
187	0	} catch (IOException ex) {
188	0	LOGGER.trace("Error closing the writer in Downloader.", ex);
189	0	}
190		}
191	0	if (reader != null) {
192		try {
193	0	reader.close();
194	0	} catch (IOException ex) {
195	0	LOGGER.trace("Error closing the reader in Downloader.", ex);
196	0	}
197		}
198		try {
199	0	conn.disconnect();
200		} finally {
201	0	conn = null;
202	0	}
203	0	}
204		}
205	0	}
206
207		/**
208		* Makes an HTTP Head request to retrieve the last modified date of the given URL. If the file:// protocol is specified, then
209		* the lastTimestamp of the file is returned.
210		*
211		* @param url the URL to retrieve the timestamp from
212		* @return an epoch timestamp
213		* @throws DownloadFailedException is thrown if an exception occurs making the HTTP request
214		*/
215		public static long getLastModified(URL url) throws DownloadFailedException {
216	1	long timestamp = 0;
217		//TODO add the FTP protocol?
218	1	if ("file".equalsIgnoreCase(url.getProtocol())) {
219		File lastModifiedFile;
220		try {
221	1	lastModifiedFile = new File(url.toURI());
222	0	} catch (URISyntaxException ex) {
223	0	final String msg = format("Unable to locate '%s'", url.toString());
224	0	throw new DownloadFailedException(msg);
225	1	}
226	1	timestamp = lastModifiedFile.lastModified();
227	1	} else {
228	0	final String httpMethod = determineHttpMethod();
229	0	HttpURLConnection conn = null;
230		try {
231	0	conn = URLConnectionFactory.createHttpURLConnection(url);
232	0	conn.setRequestMethod(httpMethod);
233	0	conn.connect();
234	0	final int t = conn.getResponseCode();
235	0	if (t >= 200 && t < 300) {
236	0	timestamp = conn.getLastModified();
237		} else {
238	0	throw new DownloadFailedException(format("%s request returned a non-200 status code", httpMethod));
239		}
240	0	} catch (URLConnectionFailureException ex) {
241	0	throw new DownloadFailedException(format("Error creating URL Connection for HTTP %s request.", httpMethod), ex);
242	0	} catch (IOException ex) {
243	0	analyzeException(ex);
244		try {
245		//retry
246	0	if (!Settings.getBoolean(Settings.KEYS.DOWNLOADER_QUICK_QUERY_TIMESTAMP)) {
247	0	Settings.setBoolean(Settings.KEYS.DOWNLOADER_QUICK_QUERY_TIMESTAMP, true);
248	0	return getLastModified(url);
249		}
250	0	} catch (InvalidSettingException ex1) {
251	0	LOGGER.debug("invalid setting?", ex);
252	0	}
253
254	0	throw new DownloadFailedException(format("Error making HTTP %s request.", httpMethod), ex);
255		} finally {
256	0	if (conn != null) {
257		try {
258	0	conn.disconnect();
259		} finally {
260	0	conn = null;
261	0	}
262		}
263		}
264		}
265	1	return timestamp;
266		}
267
268		/**
269		* Analyzes the IOException, logs the appropriate information for debugging purposes, and then throws a
270		* DownloadFailedException that wraps the IO Exception.
271		*
272		* @param ex the original exception
273		* @throws DownloadFailedException a wrapper exception that contains the original exception as the cause
274		*/
275		protected static void analyzeException(IOException ex) throws DownloadFailedException {
276	0	Throwable cause = ex;
277	0	while (cause != null) {
278	0	if (cause instanceof InvalidAlgorithmParameterException) {
279	0	final String keystore = System.getProperty("javax.net.ssl.keyStore");
280	0	final String version = System.getProperty("java.version");
281	0	final String vendor = System.getProperty("java.vendor");
282	0	LOGGER.info("Error making HTTPS request - InvalidAlgorithmParameterException");
283	0	LOGGER.info("There appears to be an issue with the installation of Java and the cacerts."
284		+ "See closed issue #177 here: https://github.com/jeremylong/DependencyCheck/issues/177");
285	0	LOGGER.info("Java Info:\njavax.net.ssl.keyStore='{}'\njava.version='{}'\njava.vendor='{}'",
286		keystore, version, vendor);
287	0	throw new DownloadFailedException("Error making HTTPS request. Please see the log for more details.");
288		}
289	0	cause = cause.getCause();
290		}
291	0	}
292
293		/**
294		* Returns the HEAD or GET HTTP method. HEAD is the default.
295		*
296		* @return the HTTP method to use
297		*/
298		private static String determineHttpMethod() {
299	0	return isQuickQuery() ? HEAD : GET;
300		}
301
302		/**
303		* Determines if the HTTP method GET or HEAD should be used to check the timestamp on external resources.
304		*
305		* @return true if configured to use HEAD requests
306		*/
307		private static boolean isQuickQuery() {
308		boolean quickQuery;
309
310		try {
311	0	quickQuery = Settings.getBoolean(Settings.KEYS.DOWNLOADER_QUICK_QUERY_TIMESTAMP, true);
312	0	} catch (InvalidSettingException e) {
313	0	quickQuery = true;
314	0	}
315	0	return quickQuery;
316		}
317		}