Coverage Report

Coverage Report - org.owasp.dependencycheck.data.update.nvd.DownloadTask

Classes in this File

Line Coverage

Branch Coverage

Complexity

DownloadTask

55%

60/108

35%

23/64

5.364

 /*
  * This file is part of dependency-check-core.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
  *
  *     http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
  *
  * Copyright (c) 2013 Jeremy Long. All Rights Reserved.
  */
 package org.owasp.dependencycheck.data.update.nvd;
 
 import java.io.File;
 import java.io.FileInputStream;
 import java.io.FileNotFoundException;
 import java.io.FileOutputStream;
 import java.io.IOException;
 import java.io.InputStream;
 import java.net.URL;
 import java.util.concurrent.Callable;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Future;
 import java.util.zip.GZIPInputStream;
 import org.apache.commons.io.FileUtils;
 import org.owasp.dependencycheck.data.nvdcve.CveDB;
 import org.owasp.dependencycheck.data.update.exception.UpdateException;
 import org.owasp.dependencycheck.utils.DownloadFailedException;
 import org.owasp.dependencycheck.utils.Downloader;
 import org.owasp.dependencycheck.utils.Settings;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 /**
  * A callable object to download two files.
  *
  * @author Jeremy Long
  */
 public class DownloadTask implements Callable<Future<ProcessTask>> {
 
     /**
      * The Logger.
      */
     private static final Logger LOGGER = LoggerFactory.getLogger(DownloadTask.class);
 
     /**
      * Simple constructor for the callable download task.
      *
      * @param nvdCveInfo the NVD CVE info
      * @param processor the processor service to submit the downloaded files to
      * @param cveDB the CVE DB to use to store the vulnerability data
      * @param settings a reference to the global settings object; this is
      * necessary so that when the thread is started the dependencies have a
      * correct reference to the global settings.
      * @throws UpdateException thrown if temporary files could not be created
      */
     public DownloadTask(NvdCveInfo nvdCveInfo, ExecutorService processor, CveDB cveDB, Settings settings) throws UpdateException {
         this.nvdCveInfo = nvdCveInfo;
         this.processorService = processor;
         this.cveDB = cveDB;
         this.settings = settings;
 
         final File file1;
         final File file2;
 
         try {
             file1 = File.createTempFile("cve" + nvdCveInfo.getId() + '_', ".xml", Settings.getTempDirectory());
             file2 = File.createTempFile("cve_1_2_" + nvdCveInfo.getId() + '_', ".xml", Settings.getTempDirectory());
         } catch (IOException ex) {
             throw new UpdateException("Unable to create temporary files", ex);
         }
         this.first = file1;
         this.second = file2;
 
     }
     /**
      * The CVE DB to use when processing the files.
      */
     private final CveDB cveDB;
     /**
      * The processor service to pass the results of the download to.
      */
     private final ExecutorService processorService;
     /**
      * The NVD CVE Meta Data.
      */
     private NvdCveInfo nvdCveInfo;
     /**
      * A reference to the global settings object.
      */
     private final Settings settings;
 
     /**
      * Get the value of nvdCveInfo.
      *
      * @return the value of nvdCveInfo
      */
     public NvdCveInfo getNvdCveInfo() {
         return nvdCveInfo;
     }
 
     /**
      * Set the value of nvdCveInfo.
      *
      * @param nvdCveInfo new value of nvdCveInfo
      */
     public void setNvdCveInfo(NvdCveInfo nvdCveInfo) {
         this.nvdCveInfo = nvdCveInfo;
     }
     /**
      * a file.
      */
     private File first;
 
     /**
      * Get the value of first.
      *
      * @return the value of first
      */
     public File getFirst() {
         return first;
     }
 
     /**
      * Set the value of first.
      *
      * @param first new value of first
      */
     public void setFirst(File first) {
         this.first = first;
     }
     /**
      * a file.
      */
     private File second;
 
     /**
      * Get the value of second.
      *
      * @return the value of second
      */
     public File getSecond() {
         return second;
     }
 
     /**
      * Set the value of second.
      *
      * @param second new value of second
      */
     public void setSecond(File second) {
         this.second = second;
     }
 
     @Override
     public Future<ProcessTask> call() throws Exception {
         try {
             Settings.setInstance(settings);
             final URL url1 = new URL(nvdCveInfo.getUrl());
             final URL url2 = new URL(nvdCveInfo.getOldSchemaVersionUrl());
             LOGGER.info("Download Started for NVD CVE - {}", nvdCveInfo.getId());
             final long startDownload = System.currentTimeMillis();
             try {
                 Downloader.fetchFile(url1, first);
                 Downloader.fetchFile(url2, second);
             } catch (DownloadFailedException ex) {
                 LOGGER.warn("Download Failed for NVD CVE - {}\nSome CVEs may not be reported.", nvdCveInfo.getId());
                 if (Settings.getString(Settings.KEYS.PROXY_SERVER) == null) {
                     LOGGER.info(
                             "If you are behind a proxy you may need to configure dependency-check to use the proxy.");
                 }
                 LOGGER.debug("", ex);
                 return null;
             }
             if (url1.toExternalForm().endsWith(".xml.gz") && !isXml(first)) {
                 extractGzip(first);
             }
             if (url2.toExternalForm().endsWith(".xml.gz") && !isXml(second)) {
                 extractGzip(second);
             }
 
             LOGGER.info("Download Complete for NVD CVE - {}  ({} ms)", nvdCveInfo.getId(),
                     System.currentTimeMillis() - startDownload);
             if (this.processorService == null) {
                 return null;
             }
             final ProcessTask task = new ProcessTask(cveDB, this, settings);
             return this.processorService.submit(task);
 
         } catch (Throwable ex) {
             LOGGER.warn("An exception occurred downloading NVD CVE - {}\nSome CVEs may not be reported.", nvdCveInfo.getId());
             LOGGER.debug("Download Task Failed", ex);
         } finally {
             Settings.cleanup(false);
         }
         return null;
     }
 
     /**
      * Attempts to delete the files that were downloaded.
      */
     public void cleanup() {
         if (first != null && first.exists() && first.delete()) {
             LOGGER.debug("Failed to delete first temporary file {}", second.toString());
             first.deleteOnExit();
         }
         if (second != null && second.exists() && !second.delete()) {
             LOGGER.debug("Failed to delete second temporary file {}", second.toString());
             second.deleteOnExit();
         }
     }
 
     /**
      * Checks the file header to see if it is an XML file.
      *
      * @param file the file to check
      * @return true if the file is XML
      */
     public static boolean isXml(File file) {
         if (file == null || !file.isFile()) {
             return false;
         }
         InputStream is = null;
         try {
             is = new FileInputStream(file);
 
             final byte[] buf = new byte[5];
             int read = 0;
             try {
                 read = is.read(buf);
             } catch (IOException ex) {
                 return false;
             }
             return read == 5
                     && buf[0] == '<'
                     && (buf[1] == '?')
                     && (buf[2] == 'x' || buf[2] == 'X')
                     && (buf[3] == 'm' || buf[3] == 'M')
                     && (buf[4] == 'l' || buf[4] == 'L');
         } catch (FileNotFoundException ex) {
             return false;
         } finally {
             if (is != null) {
                 try {
                     is.close();
                 } catch (IOException ex) {
                     LOGGER.debug("Error closing stream", ex);
                 }
             }
         }
     }
 
     /**
      * Extracts the file contained in a gzip archive. The extracted file is
      * placed in the exact same path as the file specified.
      *
      * @param file the archive file
      * @throws FileNotFoundException thrown if the file does not exist
      * @throws IOException thrown if there is an error extracting the file.
      */
     private void extractGzip(File file) throws FileNotFoundException, IOException {
         final String originalPath = file.getPath();
         final File gzip = new File(originalPath + ".gz");
         if (gzip.isFile() && !gzip.delete()) {
             LOGGER.debug("Failed to delete initial temporary file when extracting 'gz' {}", gzip.toString());
             gzip.deleteOnExit();
         }
         if (!file.renameTo(gzip)) {
             throw new IOException("Unable to rename '" + file.getPath() + "'");
         }
         final File newfile = new File(originalPath);
 
         final byte[] buffer = new byte[4096];
 
         GZIPInputStream cin = null;
         FileOutputStream out = null;
         try {
             cin = new GZIPInputStream(new FileInputStream(gzip));
             out = new FileOutputStream(newfile);
 
             int len;
             while ((len = cin.read(buffer)) > 0) {
                 out.write(buffer, 0, len);
             }
         } finally {
             if (cin != null) {
                 try {
                     cin.close();
                 } catch (IOException ex) {
                     LOGGER.trace("ignore", ex);
                 }
             }
             if (out != null) {
                 try {
                     out.close();
                 } catch (IOException ex) {
                     LOGGER.trace("ignore", ex);
                 }
             }
             if (gzip.isFile() && !FileUtils.deleteQuietly(gzip)) {
                 LOGGER.debug("Failed to delete temporary file when extracting 'gz' {}", gzip.toString());
                 gzip.deleteOnExit();
             }
         }
     }
 }

1		/*
2		* This file is part of dependency-check-core.
3		*
4		* Licensed under the Apache License, Version 2.0 (the "License");
5		* you may not use this file except in compliance with the License.
6		* You may obtain a copy of the License at
7		*
8		* http://www.apache.org/licenses/LICENSE-2.0
9		*
10		* Unless required by applicable law or agreed to in writing, software
11		* distributed under the License is distributed on an "AS IS" BASIS,
12		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13		* See the License for the specific language governing permissions and
14		* limitations under the License.
15		*
16		* Copyright (c) 2013 Jeremy Long. All Rights Reserved.
17		*/
18		package org.owasp.dependencycheck.data.update.nvd;
19
20		import java.io.File;
21		import java.io.FileInputStream;
22		import java.io.FileNotFoundException;
23		import java.io.FileOutputStream;
24		import java.io.IOException;
25		import java.io.InputStream;
26		import java.net.URL;
27		import java.util.concurrent.Callable;
28		import java.util.concurrent.ExecutorService;
29		import java.util.concurrent.Future;
30		import java.util.zip.GZIPInputStream;
31		import org.apache.commons.io.FileUtils;
32		import org.owasp.dependencycheck.data.nvdcve.CveDB;
33		import org.owasp.dependencycheck.data.update.exception.UpdateException;
34		import org.owasp.dependencycheck.utils.DownloadFailedException;
35		import org.owasp.dependencycheck.utils.Downloader;
36		import org.owasp.dependencycheck.utils.Settings;
37		import org.slf4j.Logger;
38		import org.slf4j.LoggerFactory;
39
40		/**
41		* A callable object to download two files.
42		*
43		* @author Jeremy Long
44		*/
45	0	public class DownloadTask implements Callable<Future<ProcessTask>> {
46
47		/**
48		* The Logger.
49		*/
50	1	private static final Logger LOGGER = LoggerFactory.getLogger(DownloadTask.class);
51
52		/**
53		* Simple constructor for the callable download task.
54		*
55		* @param nvdCveInfo the NVD CVE info
56		* @param processor the processor service to submit the downloaded files to
57		* @param cveDB the CVE DB to use to store the vulnerability data
58		* @param settings a reference to the global settings object; this is
59		* necessary so that when the thread is started the dependencies have a
60		* correct reference to the global settings.
61		* @throws UpdateException thrown if temporary files could not be created
62		*/
63	1	public DownloadTask(NvdCveInfo nvdCveInfo, ExecutorService processor, CveDB cveDB, Settings settings) throws UpdateException {
64	1	this.nvdCveInfo = nvdCveInfo;
65	1	this.processorService = processor;
66	1	this.cveDB = cveDB;
67	1	this.settings = settings;
68
69		final File file1;
70		final File file2;
71
72		try {
73	1	file1 = File.createTempFile("cve" + nvdCveInfo.getId() + '_', ".xml", Settings.getTempDirectory());
74	1	file2 = File.createTempFile("cve_1_2_" + nvdCveInfo.getId() + '_', ".xml", Settings.getTempDirectory());
75	0	} catch (IOException ex) {
76	0	throw new UpdateException("Unable to create temporary files", ex);
77	1	}
78	1	this.first = file1;
79	1	this.second = file2;
80
81	1	}
82		/**
83		* The CVE DB to use when processing the files.
84		*/
85		private final CveDB cveDB;
86		/**
87		* The processor service to pass the results of the download to.
88		*/
89		private final ExecutorService processorService;
90		/**
91		* The NVD CVE Meta Data.
92		*/
93		private NvdCveInfo nvdCveInfo;
94		/**
95		* A reference to the global settings object.
96		*/
97		private final Settings settings;
98
99		/**
100		* Get the value of nvdCveInfo.
101		*
102		* @return the value of nvdCveInfo
103		*/
104		public NvdCveInfo getNvdCveInfo() {
105	0	return nvdCveInfo;
106		}
107
108		/**
109		* Set the value of nvdCveInfo.
110		*
111		* @param nvdCveInfo new value of nvdCveInfo
112		*/
113		public void setNvdCveInfo(NvdCveInfo nvdCveInfo) {
114	0	this.nvdCveInfo = nvdCveInfo;
115	0	}
116		/**
117		* a file.
118		*/
119		private File first;
120
121		/**
122		* Get the value of first.
123		*
124		* @return the value of first
125		*/
126		public File getFirst() {
127	0	return first;
128		}
129
130		/**
131		* Set the value of first.
132		*
133		* @param first new value of first
134		*/
135		public void setFirst(File first) {
136	0	this.first = first;
137	0	}
138		/**
139		* a file.
140		*/
141		private File second;
142
143		/**
144		* Get the value of second.
145		*
146		* @return the value of second
147		*/
148		public File getSecond() {
149	0	return second;
150		}
151
152		/**
153		* Set the value of second.
154		*
155		* @param second new value of second
156		*/
157		public void setSecond(File second) {
158	0	this.second = second;
159	0	}
160
161		@Override
162		public Future<ProcessTask> call() throws Exception {
163		try {
164	1	Settings.setInstance(settings);
165	1	final URL url1 = new URL(nvdCveInfo.getUrl());
166	1	final URL url2 = new URL(nvdCveInfo.getOldSchemaVersionUrl());
167	1	LOGGER.info("Download Started for NVD CVE - {}", nvdCveInfo.getId());
168	1	final long startDownload = System.currentTimeMillis();
169		try {
170	1	Downloader.fetchFile(url1, first);
171	1	Downloader.fetchFile(url2, second);
172	0	} catch (DownloadFailedException ex) {
173	0	LOGGER.warn("Download Failed for NVD CVE - {}\nSome CVEs may not be reported.", nvdCveInfo.getId());
174	0	if (Settings.getString(Settings.KEYS.PROXY_SERVER) == null) {
175	0	LOGGER.info(
176		"If you are behind a proxy you may need to configure dependency-check to use the proxy.");
177		}
178	0	LOGGER.debug("", ex);
179	0	return null;
180	1	}
181	1	if (url1.toExternalForm().endsWith(".xml.gz") && !isXml(first)) {
182	1	extractGzip(first);
183		}
184	1	if (url2.toExternalForm().endsWith(".xml.gz") && !isXml(second)) {
185	1	extractGzip(second);
186		}
187
188	2	LOGGER.info("Download Complete for NVD CVE - {} ({} ms)", nvdCveInfo.getId(),
189	1	System.currentTimeMillis() - startDownload);
190	1	if (this.processorService == null) {
191	2	return null;
192		}
193	0	final ProcessTask task = new ProcessTask(cveDB, this, settings);
194	0	return this.processorService.submit(task);
195
196	0	} catch (Throwable ex) {
197	0	LOGGER.warn("An exception occurred downloading NVD CVE - {}\nSome CVEs may not be reported.", nvdCveInfo.getId());
198	0	LOGGER.debug("Download Task Failed", ex);
199		} finally {
200	1	Settings.cleanup(false);
201	0	}
202	0	return null;
203		}
204
205		/**
206		* Attempts to delete the files that were downloaded.
207		*/
208		public void cleanup() {
209	0	if (first != null && first.exists() && first.delete()) {
210	0	LOGGER.debug("Failed to delete first temporary file {}", second.toString());
211	0	first.deleteOnExit();
212		}
213	0	if (second != null && second.exists() && !second.delete()) {
214	0	LOGGER.debug("Failed to delete second temporary file {}", second.toString());
215	0	second.deleteOnExit();
216		}
217	0	}
218
219		/**
220		* Checks the file header to see if it is an XML file.
221		*
222		* @param file the file to check
223		* @return true if the file is XML
224		*/
225		public static boolean isXml(File file) {
226	4	if (file == null \|\| !file.isFile()) {
227	0	return false;
228		}
229	4	InputStream is = null;
230		try {
231	4	is = new FileInputStream(file);
232
233	4	final byte[] buf = new byte[5];
234	4	int read = 0;
235		try {
236	4	read = is.read(buf);
237	0	} catch (IOException ex) {
238	0	return false;
239	4	}
240	8	return read == 5
241		&& buf[0] == '<'
242		&& (buf[1] == '?')
243		&& (buf[2] == 'x' \|\| buf[2] == 'X')
244		&& (buf[3] == 'm' \|\| buf[3] == 'M')
245		&& (buf[4] == 'l' \|\| buf[4] == 'L');
246	0	} catch (FileNotFoundException ex) {
247	0	return false;
248		} finally {
249	4	if (is != null) {
250		try {
251	4	is.close();
252	0	} catch (IOException ex) {
253	0	LOGGER.debug("Error closing stream", ex);
254	4	}
255		}
256		}
257		}
258
259		/**
260		* Extracts the file contained in a gzip archive. The extracted file is
261		* placed in the exact same path as the file specified.
262		*
263		* @param file the archive file
264		* @throws FileNotFoundException thrown if the file does not exist
265		* @throws IOException thrown if there is an error extracting the file.
266		*/
267		private void extractGzip(File file) throws FileNotFoundException, IOException {
268	2	final String originalPath = file.getPath();
269	2	final File gzip = new File(originalPath + ".gz");
270	2	if (gzip.isFile() && !gzip.delete()) {
271	0	LOGGER.debug("Failed to delete initial temporary file when extracting 'gz' {}", gzip.toString());
272	0	gzip.deleteOnExit();
273		}
274	2	if (!file.renameTo(gzip)) {
275	0	throw new IOException("Unable to rename '" + file.getPath() + "'");
276		}
277	2	final File newfile = new File(originalPath);
278
279	2	final byte[] buffer = new byte[4096];
280
281	2	GZIPInputStream cin = null;
282	2	FileOutputStream out = null;
283		try {
284	2	cin = new GZIPInputStream(new FileInputStream(gzip));
285	2	out = new FileOutputStream(newfile);
286
287		int len;
288	801	while ((len = cin.read(buffer)) > 0) {
289	799	out.write(buffer, 0, len);
290		}
291		} finally {
292	2	if (cin != null) {
293		try {
294	2	cin.close();
295	0	} catch (IOException ex) {
296	0	LOGGER.trace("ignore", ex);
297	2	}
298		}
299	2	if (out != null) {
300		try {
301	2	out.close();
302	0	} catch (IOException ex) {
303	0	LOGGER.trace("ignore", ex);
304	2	}
305		}
306	2	if (gzip.isFile() && !FileUtils.deleteQuietly(gzip)) {
307	0	LOGGER.debug("Failed to delete temporary file when extracting 'gz' {}", gzip.toString());
308	0	gzip.deleteOnExit();
309		}
310		}
311	2	}
312		}