Coverage Report

Coverage Report - org.owasp.dependencycheck.data.update.StandardUpdateTask

Classes in this File

Line Coverage

Branch Coverage

Complexity

StandardUpdateTask

0/126

0/60

14.75

 /*
  * This file is part of dependency-check-core.
  *
  * Dependency-check-core is free software: you can redistribute it and/or modify it
  * under the terms of the GNU General Public License as published by the Free
  * Software Foundation, either version 3 of the License, or (at your option) any
  * later version.
  *
  * Dependency-check-core is distributed in the hope that it will be useful, but
  * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
  * details.
  *
  * You should have received a copy of the GNU General Public License along with
  * dependency-check-core. If not, see http://www.gnu.org/licenses/.
  *
  * Copyright (c) 2012 Jeremy Long. All Rights Reserved.
  */
 package org.owasp.dependencycheck.data.update;
 
 import org.owasp.dependencycheck.data.nvdcve.InvalidDataException;
 import java.io.File;
 import java.io.FileNotFoundException;
 import java.io.IOException;
 import javax.xml.parsers.ParserConfigurationException;
 import org.xml.sax.SAXException;
 import java.net.MalformedURLException;
 import java.net.URL;
 import java.sql.SQLException;
 import java.util.Calendar;
 import java.util.Date;
 import java.util.logging.Level;
 import java.util.logging.Logger;
 import org.owasp.dependencycheck.data.UpdateException;
 import org.owasp.dependencycheck.data.nvdcve.CveDB;
 import org.owasp.dependencycheck.utils.DownloadFailedException;
 import org.owasp.dependencycheck.utils.Downloader;
 import org.owasp.dependencycheck.utils.Settings;
 import org.owasp.dependencycheck.data.nvdcve.DatabaseException;
 import org.owasp.dependencycheck.utils.InvalidSettingException;
 import static org.owasp.dependencycheck.data.update.DataStoreMetaInfo.MODIFIED;
 
 /**
  * Class responsible for updating the CPE and NVDCVE data stores.
  *
  * @author Jeremy Long (jeremy.long@owasp.org)
  */
 public class StandardUpdateTask extends AbstractUpdateTask {
 
     /**
      * Constructs a new Standard Update Task.
      *
      * @param properties information about the data store
      * @throws MalformedURLException thrown if a configured URL is malformed
      * @throws DownloadFailedException thrown if a timestamp cannot be checked
      * on a configured URL
      * @throws UpdateException thrown if there is an exception generating the
      * update task
      */
     public StandardUpdateTask(DataStoreMetaInfo properties) throws MalformedURLException, DownloadFailedException, UpdateException {
         super(properties);
     }
 
     /**
      * <p>Downloads the latest NVD CVE XML file from the web and imports it into
      * the current CVE Database.</p>
      *
      * @throws UpdateException is thrown if there is an error updating the
      * database
      */
     @Override
     public void update() throws UpdateException {
         try {
             int maxUpdates = 0;
             for (NvdCveInfo cve : getUpdateable()) {
                 if (cve.getNeedsUpdate()) {
                     maxUpdates += 1;
                 }
             }
             if (maxUpdates > 3) {
                 Logger.getLogger(StandardUpdateTask.class.getName()).log(Level.INFO,
                         "NVD CVE requires several updates; this could take a couple of minutes.");
             }
             if (maxUpdates > 0) {
                 openDataStores();
             }
 
             int count = 0;
             for (NvdCveInfo cve : getUpdateable()) {
                 if (cve.getNeedsUpdate()) {
                     count += 1;
                     Logger.getLogger(StandardUpdateTask.class.getName()).log(Level.INFO,
                             "Updating NVD CVE ({0} of {1})", new Object[]{count, maxUpdates});
                     URL url = new URL(cve.getUrl());
                     File outputPath = null;
                     File outputPath12 = null;
                     try {
                         Logger.getLogger(StandardUpdateTask.class.getName()).log(Level.INFO,
                                 "Downloading {0}", cve.getUrl());
                         outputPath = File.createTempFile("cve" + cve.getId() + "_", ".xml");
                         Downloader.fetchFile(url, outputPath);
 
                         url = new URL(cve.getOldSchemaVersionUrl());
                         outputPath12 = File.createTempFile("cve_1_2_" + cve.getId() + "_", ".xml");
                         Downloader.fetchFile(url, outputPath12);
 
                         Logger.getLogger(StandardUpdateTask.class.getName()).log(Level.INFO,
                                 "Processing {0}", cve.getUrl());
 
                         importXML(outputPath, outputPath12);
 
                         getCveDB().commit();
                         getCpeIndex().commit();
                         getProperties().save(cve);
 
                         Logger.getLogger(StandardUpdateTask.class.getName()).log(Level.INFO,
                                 "Completed update {0} of {1}", new Object[]{count, maxUpdates});
                     } catch (FileNotFoundException ex) {
                         throw new UpdateException(ex);
                     } catch (ParserConfigurationException ex) {
                         throw new UpdateException(ex);
                     } catch (SAXException ex) {
                         throw new UpdateException(ex);
                     } catch (IOException ex) {
                         throw new UpdateException(ex);
                     } catch (SQLException ex) {
                         throw new UpdateException(ex);
                     } catch (DatabaseException ex) {
                         throw new UpdateException(ex);
                     } catch (ClassNotFoundException ex) {
                         throw new UpdateException(ex);
                     } finally {
                         boolean deleted = false;
                         try {
                             if (outputPath != null && outputPath.exists()) {
                                 deleted = outputPath.delete();
                             }
                         } finally {
                             if (outputPath != null && (outputPath.exists() || !deleted)) {
                                 outputPath.deleteOnExit();
                             }
                         }
                         try {
                             deleted = false;
                             if (outputPath12 != null && outputPath12.exists()) {
                                 deleted = outputPath12.delete();
                             }
                         } finally {
                             if (outputPath12 != null && (outputPath12.exists() || !deleted)) {
                                 outputPath12.deleteOnExit();
                             }
                         }
                     }
                 }
             }
             if (maxUpdates >= 1) { //ensure the modified file date gets written
                 getProperties().save(getUpdateable().get(MODIFIED));
                 getCveDB().cleanupDatabase();
             }
         } catch (MalformedURLException ex) {
             throw new UpdateException(ex);
         } finally {
             closeDataStores();
         }
     }
 
     /**
      * Determines if the index needs to be updated. This is done by fetching the
      * NVD CVE meta data and checking the last update date. If the data needs to
      * be refreshed this method will return the NvdCveUrl for the files that
      * need to be updated.
      *
      * @return the collection of files that need to be updated
      * @throws MalformedURLException is thrown if the URL for the NVD CVE Meta
      * data is incorrect
      * @throws DownloadFailedException is thrown if there is an error.
      * downloading the NVD CVE download data file
      * @throws UpdateException Is thrown if there is an issue with the last
      * updated properties file
      */
     @Override
     protected Updateable updatesNeeded() throws MalformedURLException, DownloadFailedException, UpdateException {
         Updateable updates = null;
         try {
             updates = retrieveCurrentTimestampsFromWeb();
         } catch (InvalidDataException ex) {
             final String msg = "Unable to retrieve valid timestamp from nvd cve downloads page";
             Logger.getLogger(StandardUpdateTask.class.getName()).log(Level.FINE, msg, ex);
             throw new DownloadFailedException(msg, ex);
         } catch (InvalidSettingException ex) {
             Logger.getLogger(StandardUpdateTask.class.getName()).log(Level.FINE, "Invalid setting found when retrieving timestamps", ex);
             throw new DownloadFailedException("Invalid settings", ex);
         }
 
         if (updates == null) {
             throw new DownloadFailedException("Unable to retrieve the timestamps of the currently published NVD CVE data");
         }
         final DataStoreMetaInfo properties = getProperties();
         if (!properties.isEmpty()) {
             try {
                 float version;
 
                 if (properties.getProperty("version") == null) {
                     setDeleteAndRecreate(true);
                 } else {
                     try {
                         version = Float.parseFloat(properties.getProperty("version"));
                         final float currentVersion = Float.parseFloat(CveDB.DB_SCHEMA_VERSION);
                         if (currentVersion > version) {
                             setDeleteAndRecreate(true);
                         }
                     } catch (NumberFormatException ex) {
                         setDeleteAndRecreate(true);
                     }
                 }
 
                 if (shouldDeleteAndRecreate()) {
                     return updates;
                 }
 
                 final long lastUpdated = Long.parseLong(properties.getProperty(DataStoreMetaInfo.LAST_UPDATED, "0"));
                 final Date now = new Date();
                 final int days = Settings.getInt(Settings.KEYS.CVE_MODIFIED_VALID_FOR_DAYS, 7);
                 if (lastUpdated == updates.getTimeStamp(MODIFIED)) {
                     updates.clear(); //we don't need to update anything.
                 } else if (withinRange(lastUpdated, now.getTime(), days)) {
                     for (NvdCveInfo entry : updates) {
                         if (MODIFIED.equals(entry.getId())) {
                             entry.setNeedsUpdate(true);
                         } else {
                             entry.setNeedsUpdate(false);
                         }
                     }
                 } else { //we figure out which of the several XML files need to be downloaded.
                     for (NvdCveInfo entry : updates) {
                         if (MODIFIED.equals(entry.getId())) {
                             entry.setNeedsUpdate(true);
                         } else {
                             long currentTimestamp = 0;
                             try {
                                 currentTimestamp = Long.parseLong(properties.getProperty(DataStoreMetaInfo.LAST_UPDATED_BASE + entry.getId(), "0"));
                             } catch (NumberFormatException ex) {
                                 final String msg = String.format("Error parsing '%s' '%s' from nvdcve.lastupdated",
                                         DataStoreMetaInfo.LAST_UPDATED_BASE, entry.getId());
                                 Logger.getLogger(StandardUpdateTask.class.getName()).log(Level.FINE, msg, ex);
                             }
                             if (currentTimestamp == entry.getTimestamp()) {
                                 entry.setNeedsUpdate(false);
                             }
                         }
                     }
                 }
             } catch (NumberFormatException ex) {
                 final String msg = "An invalid schema version or timestamp exists in the data.properties file.";
                 Logger.getLogger(StandardUpdateTask.class.getName()).log(Level.WARNING, msg);
                 Logger.getLogger(StandardUpdateTask.class.getName()).log(Level.FINE, null, ex);
             }
         }
         return updates;
     }
 
     /**
      * Retrieves the timestamps from the NVD CVE meta data file.
      *
      * @return the timestamp from the currently published nvdcve downloads page
      * @throws MalformedURLException thrown if the URL for the NVD CCE Meta data
      * is incorrect.
      * @throws DownloadFailedException thrown if there is an error downloading
      * the nvd cve meta data file
      * @throws InvalidDataException thrown if there is an exception parsing the
      * timestamps
      * @throws InvalidSettingException thrown if the settings are invalid
      */
     private Updateable retrieveCurrentTimestampsFromWeb()
             throws MalformedURLException, DownloadFailedException, InvalidDataException, InvalidSettingException {
 
         final Updateable updates = new Updateable();
         updates.add(MODIFIED, Settings.getString(Settings.KEYS.CVE_MODIFIED_20_URL),
                 Settings.getString(Settings.KEYS.CVE_MODIFIED_12_URL),
                 false);
 
         final int start = Settings.getInt(Settings.KEYS.CVE_START_YEAR);
         final int end = Calendar.getInstance().get(Calendar.YEAR);
         final String baseUrl20 = Settings.getString(Settings.KEYS.CVE_SCHEMA_2_0);
         final String baseUrl12 = Settings.getString(Settings.KEYS.CVE_SCHEMA_1_2);
         for (int i = start; i <= end; i++) {
             updates.add(Integer.toString(i), String.format(baseUrl20, i),
                     String.format(baseUrl12, i),
                     true);
         }
 
         return updates;
     }
 }

1		/*
2		* This file is part of dependency-check-core.
3		*
4		* Dependency-check-core is free software: you can redistribute it and/or modify it
5		* under the terms of the GNU General Public License as published by the Free
6		* Software Foundation, either version 3 of the License, or (at your option) any
7		* later version.
8		*
9		* Dependency-check-core is distributed in the hope that it will be useful, but
10		* WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11		* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
12		* details.
13		*
14		* You should have received a copy of the GNU General Public License along with
15		* dependency-check-core. If not, see http://www.gnu.org/licenses/.
16		*
17		* Copyright (c) 2012 Jeremy Long. All Rights Reserved.
18		*/
19		package org.owasp.dependencycheck.data.update;
20
21		import org.owasp.dependencycheck.data.nvdcve.InvalidDataException;
22		import java.io.File;
23		import java.io.FileNotFoundException;
24		import java.io.IOException;
25		import javax.xml.parsers.ParserConfigurationException;
26		import org.xml.sax.SAXException;
27		import java.net.MalformedURLException;
28		import java.net.URL;
29		import java.sql.SQLException;
30		import java.util.Calendar;
31		import java.util.Date;
32		import java.util.logging.Level;
33		import java.util.logging.Logger;
34		import org.owasp.dependencycheck.data.UpdateException;
35		import org.owasp.dependencycheck.data.nvdcve.CveDB;
36		import org.owasp.dependencycheck.utils.DownloadFailedException;
37		import org.owasp.dependencycheck.utils.Downloader;
38		import org.owasp.dependencycheck.utils.Settings;
39		import org.owasp.dependencycheck.data.nvdcve.DatabaseException;
40		import org.owasp.dependencycheck.utils.InvalidSettingException;
41		import static org.owasp.dependencycheck.data.update.DataStoreMetaInfo.MODIFIED;
42
43		/**
44		* Class responsible for updating the CPE and NVDCVE data stores.
45		*
46		* @author Jeremy Long (jeremy.long@owasp.org)
47		*/
48		public class StandardUpdateTask extends AbstractUpdateTask {
49
50		/**
51		* Constructs a new Standard Update Task.
52		*
53		* @param properties information about the data store
54		* @throws MalformedURLException thrown if a configured URL is malformed
55		* @throws DownloadFailedException thrown if a timestamp cannot be checked
56		* on a configured URL
57		* @throws UpdateException thrown if there is an exception generating the
58		* update task
59		*/
60		public StandardUpdateTask(DataStoreMetaInfo properties) throws MalformedURLException, DownloadFailedException, UpdateException {
61	0	super(properties);
62	0	}
63
64		/**
65		* <p>Downloads the latest NVD CVE XML file from the web and imports it into
66		* the current CVE Database.</p>
67		*
68		* @throws UpdateException is thrown if there is an error updating the
69		* database
70		*/
71		@Override
72		public void update() throws UpdateException {
73		try {
74	0	int maxUpdates = 0;
75	0	for (NvdCveInfo cve : getUpdateable()) {
76	0	if (cve.getNeedsUpdate()) {
77	0	maxUpdates += 1;
78		}
79		}
80	0	if (maxUpdates > 3) {
81	0	Logger.getLogger(StandardUpdateTask.class.getName()).log(Level.INFO,
82		"NVD CVE requires several updates; this could take a couple of minutes.");
83		}
84	0	if (maxUpdates > 0) {
85	0	openDataStores();
86		}
87
88	0	int count = 0;
89	0	for (NvdCveInfo cve : getUpdateable()) {
90	0	if (cve.getNeedsUpdate()) {
91	0	count += 1;
92	0	Logger.getLogger(StandardUpdateTask.class.getName()).log(Level.INFO,
93		"Updating NVD CVE ({0} of {1})", new Object[]{count, maxUpdates});
94	0	URL url = new URL(cve.getUrl());
95	0	File outputPath = null;
96	0	File outputPath12 = null;
97		try {
98	0	Logger.getLogger(StandardUpdateTask.class.getName()).log(Level.INFO,
99		"Downloading {0}", cve.getUrl());
100	0	outputPath = File.createTempFile("cve" + cve.getId() + "_", ".xml");
101	0	Downloader.fetchFile(url, outputPath);
102
103	0	url = new URL(cve.getOldSchemaVersionUrl());
104	0	outputPath12 = File.createTempFile("cve_1_2_" + cve.getId() + "_", ".xml");
105	0	Downloader.fetchFile(url, outputPath12);
106
107	0	Logger.getLogger(StandardUpdateTask.class.getName()).log(Level.INFO,
108		"Processing {0}", cve.getUrl());
109
110	0	importXML(outputPath, outputPath12);
111
112	0	getCveDB().commit();
113	0	getCpeIndex().commit();
114	0	getProperties().save(cve);
115
116	0	Logger.getLogger(StandardUpdateTask.class.getName()).log(Level.INFO,
117		"Completed update {0} of {1}", new Object[]{count, maxUpdates});
118	0	} catch (FileNotFoundException ex) {
119	0	throw new UpdateException(ex);
120	0	} catch (ParserConfigurationException ex) {
121	0	throw new UpdateException(ex);
122	0	} catch (SAXException ex) {
123	0	throw new UpdateException(ex);
124	0	} catch (IOException ex) {
125	0	throw new UpdateException(ex);
126	0	} catch (SQLException ex) {
127	0	throw new UpdateException(ex);
128	0	} catch (DatabaseException ex) {
129	0	throw new UpdateException(ex);
130	0	} catch (ClassNotFoundException ex) {
131	0	throw new UpdateException(ex);
132		} finally {
133	0	boolean deleted = false;
134		try {
135	0	if (outputPath != null && outputPath.exists()) {
136	0	deleted = outputPath.delete();
137		}
138		} finally {
139	0	if (outputPath != null && (outputPath.exists() \|\| !deleted)) {
140	0	outputPath.deleteOnExit();
141		}
142		}
143		try {
144	0	deleted = false;
145	0	if (outputPath12 != null && outputPath12.exists()) {
146	0	deleted = outputPath12.delete();
147		}
148		} finally {
149	0	if (outputPath12 != null && (outputPath12.exists() \|\| !deleted)) {
150	0	outputPath12.deleteOnExit();
151		}
152		}
153	0	}
154	0	}
155		}
156	0	if (maxUpdates >= 1) { //ensure the modified file date gets written
157	0	getProperties().save(getUpdateable().get(MODIFIED));
158	0	getCveDB().cleanupDatabase();
159		}
160	0	} catch (MalformedURLException ex) {
161	0	throw new UpdateException(ex);
162		} finally {
163	0	closeDataStores();
164	0	}
165	0	}
166
167		/**
168		* Determines if the index needs to be updated. This is done by fetching the
169		* NVD CVE meta data and checking the last update date. If the data needs to
170		* be refreshed this method will return the NvdCveUrl for the files that
171		* need to be updated.
172		*
173		* @return the collection of files that need to be updated
174		* @throws MalformedURLException is thrown if the URL for the NVD CVE Meta
175		* data is incorrect
176		* @throws DownloadFailedException is thrown if there is an error.
177		* downloading the NVD CVE download data file
178		* @throws UpdateException Is thrown if there is an issue with the last
179		* updated properties file
180		*/
181		@Override
182		protected Updateable updatesNeeded() throws MalformedURLException, DownloadFailedException, UpdateException {
183	0	Updateable updates = null;
184		try {
185	0	updates = retrieveCurrentTimestampsFromWeb();
186	0	} catch (InvalidDataException ex) {
187	0	final String msg = "Unable to retrieve valid timestamp from nvd cve downloads page";
188	0	Logger.getLogger(StandardUpdateTask.class.getName()).log(Level.FINE, msg, ex);
189	0	throw new DownloadFailedException(msg, ex);
190	0	} catch (InvalidSettingException ex) {
191	0	Logger.getLogger(StandardUpdateTask.class.getName()).log(Level.FINE, "Invalid setting found when retrieving timestamps", ex);
192	0	throw new DownloadFailedException("Invalid settings", ex);
193	0	}
194
195	0	if (updates == null) {
196	0	throw new DownloadFailedException("Unable to retrieve the timestamps of the currently published NVD CVE data");
197		}
198	0	final DataStoreMetaInfo properties = getProperties();
199	0	if (!properties.isEmpty()) {
200		try {
201		float version;
202
203	0	if (properties.getProperty("version") == null) {
204	0	setDeleteAndRecreate(true);
205		} else {
206		try {
207	0	version = Float.parseFloat(properties.getProperty("version"));
208	0	final float currentVersion = Float.parseFloat(CveDB.DB_SCHEMA_VERSION);
209	0	if (currentVersion > version) {
210	0	setDeleteAndRecreate(true);
211		}
212	0	} catch (NumberFormatException ex) {
213	0	setDeleteAndRecreate(true);
214	0	}
215		}
216
217	0	if (shouldDeleteAndRecreate()) {
218	0	return updates;
219		}
220
221	0	final long lastUpdated = Long.parseLong(properties.getProperty(DataStoreMetaInfo.LAST_UPDATED, "0"));
222	0	final Date now = new Date();
223	0	final int days = Settings.getInt(Settings.KEYS.CVE_MODIFIED_VALID_FOR_DAYS, 7);
224	0	if (lastUpdated == updates.getTimeStamp(MODIFIED)) {
225	0	updates.clear(); //we don't need to update anything.
226	0	} else if (withinRange(lastUpdated, now.getTime(), days)) {
227	0	for (NvdCveInfo entry : updates) {
228	0	if (MODIFIED.equals(entry.getId())) {
229	0	entry.setNeedsUpdate(true);
230		} else {
231	0	entry.setNeedsUpdate(false);
232		}
233		}
234		} else { //we figure out which of the several XML files need to be downloaded.
235	0	for (NvdCveInfo entry : updates) {
236	0	if (MODIFIED.equals(entry.getId())) {
237	0	entry.setNeedsUpdate(true);
238		} else {
239	0	long currentTimestamp = 0;
240		try {
241	0	currentTimestamp = Long.parseLong(properties.getProperty(DataStoreMetaInfo.LAST_UPDATED_BASE + entry.getId(), "0"));
242	0	} catch (NumberFormatException ex) {
243	0	final String msg = String.format("Error parsing '%s' '%s' from nvdcve.lastupdated",
244		DataStoreMetaInfo.LAST_UPDATED_BASE, entry.getId());
245	0	Logger.getLogger(StandardUpdateTask.class.getName()).log(Level.FINE, msg, ex);
246	0	}
247	0	if (currentTimestamp == entry.getTimestamp()) {
248	0	entry.setNeedsUpdate(false);
249		}
250	0	}
251		}
252		}
253	0	} catch (NumberFormatException ex) {
254	0	final String msg = "An invalid schema version or timestamp exists in the data.properties file.";
255	0	Logger.getLogger(StandardUpdateTask.class.getName()).log(Level.WARNING, msg);
256	0	Logger.getLogger(StandardUpdateTask.class.getName()).log(Level.FINE, null, ex);
257	0	}
258		}
259	0	return updates;
260		}
261
262		/**
263		* Retrieves the timestamps from the NVD CVE meta data file.
264		*
265		* @return the timestamp from the currently published nvdcve downloads page
266		* @throws MalformedURLException thrown if the URL for the NVD CCE Meta data
267		* is incorrect.
268		* @throws DownloadFailedException thrown if there is an error downloading
269		* the nvd cve meta data file
270		* @throws InvalidDataException thrown if there is an exception parsing the
271		* timestamps
272		* @throws InvalidSettingException thrown if the settings are invalid
273		*/
274		private Updateable retrieveCurrentTimestampsFromWeb()
275		throws MalformedURLException, DownloadFailedException, InvalidDataException, InvalidSettingException {
276
277	0	final Updateable updates = new Updateable();
278	0	updates.add(MODIFIED, Settings.getString(Settings.KEYS.CVE_MODIFIED_20_URL),
279		Settings.getString(Settings.KEYS.CVE_MODIFIED_12_URL),
280		false);
281
282	0	final int start = Settings.getInt(Settings.KEYS.CVE_START_YEAR);
283	0	final int end = Calendar.getInstance().get(Calendar.YEAR);
284	0	final String baseUrl20 = Settings.getString(Settings.KEYS.CVE_SCHEMA_2_0);
285	0	final String baseUrl12 = Settings.getString(Settings.KEYS.CVE_SCHEMA_1_2);
286	0	for (int i = start; i <= end; i++) {
287	0	updates.add(Integer.toString(i), String.format(baseUrl20, i),
288		String.format(baseUrl12, i),
289		true);
290		}
291
292	0	return updates;
293		}
294		}