Coverage Report

Coverage Report - org.owasp.dependencycheck.data.update.StandardUpdate

Classes in this File

Line Coverage

Branch Coverage

Complexity

StandardUpdate

0/197

0/68

6.583

 /*
  * This file is part of dependency-check-core.
  *
  * Dependency-check-core is free software: you can redistribute it and/or modify it
  * under the terms of the GNU General Public License as published by the Free
  * Software Foundation, either version 3 of the License, or (at your option) any
  * later version.
  *
  * Dependency-check-core is distributed in the hope that it will be useful, but
  * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
  * details.
  *
  * You should have received a copy of the GNU General Public License along with
  * dependency-check-core. If not, see http://www.gnu.org/licenses/.
  *
  * Copyright (c) 2012 Jeremy Long. All Rights Reserved.
  */
 package org.owasp.dependencycheck.data.update;
 
 import org.owasp.dependencycheck.data.nvdcve.InvalidDataException;
 import java.io.File;
 import java.io.IOException;
 import java.net.MalformedURLException;
 import java.sql.SQLException;
 import java.util.Calendar;
 import java.util.Date;
 import java.util.HashSet;
 import java.util.Iterator;
 import java.util.Set;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
 import java.util.concurrent.Future;
 import java.util.logging.Level;
 import java.util.logging.Logger;
 import org.owasp.dependencycheck.data.UpdateException;
 import org.owasp.dependencycheck.data.nvdcve.CveDB;
 import org.owasp.dependencycheck.utils.DownloadFailedException;
 import org.owasp.dependencycheck.utils.Settings;
 import org.owasp.dependencycheck.data.nvdcve.DatabaseException;
 import org.owasp.dependencycheck.utils.InvalidSettingException;
 import static org.owasp.dependencycheck.data.update.DataStoreMetaInfo.MODIFIED;
 import org.owasp.dependencycheck.utils.FileUtils;
 
 /**
  * Class responsible for updating the NVDCVE data store.
  *
  * @author Jeremy Long (jeremy.long@owasp.org)
  */
 public class StandardUpdate {
 
     /**
      * The max thread pool size to use when downloading files.
      */
     public static final int MAX_THREAD_POOL_SIZE = Settings.getInt(Settings.KEYS.MAX_DOWNLOAD_THREAD_POOL_SIZE, 3);
     /**
      * Information about the timestamps and URLs for data that needs to be
      * updated.
      */
     private DataStoreMetaInfo properties;
     /**
      * A collection of updateable NVD CVE items.
      */
     private Updateable updateable;
     /**
      * A flag indicating whether or not the current data store should be
      * deleted.
      */
     private boolean deleteAndRecreate = false;
     /**
      * Reference to the Cve Database.
      */
     private CveDB cveDB = null;
 
     /**
      * Gets whether or not an update is needed.
      *
      * @return true or false depending on whether an update is needed
      */
     public boolean isUpdateNeeded() {
         return updateable.isUpdateNeeded();
     }
 
     /**
      * Set the value of deleteAndRecreate.
      *
      * @param deleteAndRecreate new value of deleteAndRecreate
      */
     protected void setDeleteAndRecreate(boolean deleteAndRecreate) {
         this.deleteAndRecreate = deleteAndRecreate;
     }
 
     /**
      * Get the value of deleteAndRecreate.
      *
      * @return the value of deleteAndRecreate
      */
     public boolean shouldDeleteAndRecreate() {
         return deleteAndRecreate;
     }
 
     /**
      * Constructs a new Standard Update Task.
      *
      * @throws MalformedURLException thrown if a configured URL is malformed
      * @throws DownloadFailedException thrown if a timestamp cannot be checked
      * on a configured URL
      * @throws UpdateException thrown if there is an exception generating the
      * update task
      */
     public StandardUpdate() throws MalformedURLException, DownloadFailedException, UpdateException {
         properties = new DataStoreMetaInfo();
         updateable = updatesNeeded();
     }
 
     /**
      * <p>Downloads the latest NVD CVE XML file from the web and imports it into
      * the current CVE Database.</p>
      *
      * @throws UpdateException is thrown if there is an error updating the
      * database
      */
     public void update() throws UpdateException {
         int maxUpdates = 0;
         try {
             for (NvdCveInfo cve : updateable) {
                 if (cve.getNeedsUpdate()) {
                     maxUpdates += 1;
                 }
             }
             if (maxUpdates <= 0) {
                 return;
             }
             if (maxUpdates > 3) {
                 Logger.getLogger(StandardUpdate.class.getName()).log(Level.INFO,
                         "NVD CVE requires several updates; this could take a couple of minutes.");
             }
             if (maxUpdates > 0) {
                 openDataStores();
             }
 
             final int poolSize = (MAX_THREAD_POOL_SIZE > maxUpdates) ? MAX_THREAD_POOL_SIZE : maxUpdates;
             final ExecutorService downloadExecutor = Executors.newFixedThreadPool(poolSize);
             final ExecutorService processExecutor = Executors.newSingleThreadExecutor();
             final Set<Future<CallableDownloadTask>> downloadFutures = new HashSet<Future<CallableDownloadTask>>(maxUpdates);
             final Set<Future<ProcessTask>> processFutures = new HashSet<Future<ProcessTask>>(maxUpdates);
             int ctr = 0;
             for (NvdCveInfo cve : updateable) {
                 if (cve.getNeedsUpdate()) {
                     ctr += 1;
                     final File file1;
                     final File file2;
                     try {
                         file1 = File.createTempFile("cve" + cve.getId() + "_", ".xml");
                         file2 = File.createTempFile("cve_1_2_" + cve.getId() + "_", ".xml");
                     } catch (IOException ex) {
                         throw new UpdateException(ex);
                     }
                     final CallableDownloadTask call = new CallableDownloadTask(cve, file1, file2);
                     downloadFutures.add(downloadExecutor.submit(call));
 
                     boolean waitForFuture = ctr % 2 == 0;
 
                     final Iterator<Future<CallableDownloadTask>> itr = downloadFutures.iterator();
                     while (itr.hasNext()) {
                         final Future<CallableDownloadTask> future = itr.next();
                         if (waitForFuture) { //only allow two NVD/CVE files to be downloaded at a time
                             spinWaitForFuture(future);
                         }
                         if (future.isDone()) { //if we find something complete, add it to the process queue
                             try {
                                 final CallableDownloadTask filePair = future.get();
                                 itr.remove();
                                 final ProcessTask task = new ProcessTask(cveDB, properties, filePair);
                                 processFutures.add(processExecutor.submit(task));
                             } catch (InterruptedException ex) {
                                 downloadExecutor.shutdownNow();
                                 Logger.getLogger(StandardUpdate.class.getName()).log(Level.FINE, "Thread was interupted", ex);
                                 throw new UpdateException(ex);
                             } catch (ExecutionException ex) {
                                 downloadExecutor.shutdownNow();
                                 Logger.getLogger(StandardUpdate.class.getName()).log(Level.SEVERE, null, ex);
                                 throw new UpdateException(ex);
                             }
                         }
                     }
 
                 }
             }
 
             try {
                 final Iterator<Future<CallableDownloadTask>> itr = downloadFutures.iterator();
                 while (itr.hasNext()) {
                     final Future<CallableDownloadTask> future = itr.next();
                     final CallableDownloadTask filePair = future.get();
                     final ProcessTask task = new ProcessTask(cveDB, properties, filePair);
                     processFutures.add(processExecutor.submit(task));
                 }
             } catch (InterruptedException ex) {
                 downloadExecutor.shutdownNow();
                 Logger.getLogger(StandardUpdate.class.getName()).log(Level.FINE, "Thread was interupted during download", ex);
                 throw new UpdateException(ex);
             } catch (ExecutionException ex) {
                 downloadExecutor.shutdownNow();
                 Logger.getLogger(StandardUpdate.class.getName()).log(Level.FINE, "Execution Exception during download", ex);
                 throw new UpdateException(ex);
             } finally {
                 downloadExecutor.shutdown();
             }
 
             for (Future<ProcessTask> future : processFutures) {
                 try {
                     final ProcessTask task = future.get();
                     if (task.getException() != null) {
                         throw task.getException();
                     }
                 } catch (InterruptedException ex) {
                     processExecutor.shutdownNow();
                     Logger.getLogger(StandardUpdate.class.getName()).log(Level.FINE, "Thread was interupted during processing", ex);
                     throw new UpdateException(ex);
                 } catch (ExecutionException ex) {
                     processExecutor.shutdownNow();
                     Logger.getLogger(StandardUpdate.class.getName()).log(Level.FINE, "Execution Exception during process", ex);
                     throw new UpdateException(ex);
                 } finally {
                     processExecutor.shutdown();
                 }
             }
 
             if (maxUpdates >= 1) { //ensure the modified file date gets written
                 properties.save(updateable.get(MODIFIED));
                 cveDB.cleanupDatabase();
             }
         } finally {
             closeDataStores();
         }
     }
 
     //<editor-fold defaultstate="collapsed" desc="OLD version of update() - not multithreaded">
     /*
      * TODO - remove this
      public void update() throws UpdateException {
      try {
      int maxUpdates = 0;
      for (NvdCveInfo cve : getUpdateable()) {
      if (cve.getNeedsUpdate()) {
      maxUpdates += 1;
      }
      }
      if (maxUpdates > 3) {
      Logger.getLogger(StandardUpdate.class.getName()).log(Level.INFO,
      "NVD CVE requires several updates; this could take a couple of minutes.");
      }
      if (maxUpdates > 0) {
      openDataStores();
      }
 
      int count = 0;
      for (NvdCveInfo cve : getUpdateable()) {
      if (cve.getNeedsUpdate()) {
      count += 1;
      Logger.getLogger(StandardUpdate.class.getName()).log(Level.INFO,
      "Updating NVD CVE ({0} of {1})", new Object[]{count, maxUpdates});
      URL url = new URL(cve.getUrl());
      File outputPath = null;
      File outputPath12 = null;
      try {
      Logger.getLogger(StandardUpdate.class.getName()).log(Level.INFO,
      "Downloading {0}", cve.getUrl());
      outputPath = File.createTempFile("cve" + cve.getId() + "_", ".xml");
      Downloader.fetchFile(url, outputPath);
 
      url = new URL(cve.getOldSchemaVersionUrl());
      outputPath12 = File.createTempFile("cve_1_2_" + cve.getId() + "_", ".xml");
      Downloader.fetchFile(url, outputPath12);
 
      Logger.getLogger(StandardUpdate.class.getName()).log(Level.INFO,
      "Processing {0}", cve.getUrl());
 
      importXML(outputPath, outputPath12);
 
      getCveDB().commit();
      getProperties().save(cve);
 
      Logger.getLogger(StandardUpdate.class.getName()).log(Level.INFO,
      "Completed update {0} of {1}", new Object[]{count, maxUpdates});
      } catch (FileNotFoundException ex) {
      throw new UpdateException(ex);
      } catch (ParserConfigurationException ex) {
      throw new UpdateException(ex);
      } catch (SAXException ex) {
      throw new UpdateException(ex);
      } catch (IOException ex) {
      throw new UpdateException(ex);
      } catch (SQLException ex) {
      throw new UpdateException(ex);
      } catch (DatabaseException ex) {
      throw new UpdateException(ex);
      } catch (ClassNotFoundException ex) {
      throw new UpdateException(ex);
      } finally {
      boolean deleted = false;
      try {
      if (outputPath != null && outputPath.exists()) {
      deleted = outputPath.delete();
      }
      } finally {
      if (outputPath != null && (outputPath.exists() || !deleted)) {
      outputPath.deleteOnExit();
      }
      }
      try {
      deleted = false;
      if (outputPath12 != null && outputPath12.exists()) {
      deleted = outputPath12.delete();
      }
      } finally {
      if (outputPath12 != null && (outputPath12.exists() || !deleted)) {
      outputPath12.deleteOnExit();
      }
      }
      }
      }
      }
      if (maxUpdates >= 1) { //ensure the modified file date gets written
      getProperties().save(getUpdateable().get(MODIFIED));
      getCveDB().cleanupDatabase();
      }
      } catch (MalformedURLException ex) {
      throw new UpdateException(ex);
      } finally {
      closeDataStores();
      }
      }
      */
     //</editor-fold>
     /**
      * Determines if the index needs to be updated. This is done by fetching the
      * NVD CVE meta data and checking the last update date. If the data needs to
      * be refreshed this method will return the NvdCveUrl for the files that
      * need to be updated.
      *
      * @return the collection of files that need to be updated
      * @throws MalformedURLException is thrown if the URL for the NVD CVE Meta
      * data is incorrect
      * @throws DownloadFailedException is thrown if there is an error.
      * downloading the NVD CVE download data file
      * @throws UpdateException Is thrown if there is an issue with the last
      * updated properties file
      */
     protected final Updateable updatesNeeded() throws MalformedURLException, DownloadFailedException, UpdateException {
         Updateable updates = null;
         try {
             updates = retrieveCurrentTimestampsFromWeb();
         } catch (InvalidDataException ex) {
             final String msg = "Unable to retrieve valid timestamp from nvd cve downloads page";
             Logger
                     .getLogger(StandardUpdate.class
                     .getName()).log(Level.FINE, msg, ex);
             throw new DownloadFailedException(msg, ex);
         } catch (InvalidSettingException ex) {
             Logger.getLogger(StandardUpdate.class
                     .getName()).log(Level.FINE, "Invalid setting found when retrieving timestamps", ex);
             throw new DownloadFailedException(
                     "Invalid settings", ex);
         }
 
         if (updates == null) {
             throw new DownloadFailedException("Unable to retrieve the timestamps of the currently published NVD CVE data");
         }
         if (!properties.isEmpty()) {
             try {
                 float version;
 
                 if (properties.getProperty("version") == null) {
                     deleteAndRecreate = true;
                 } else {
                     try {
                         version = Float.parseFloat(properties.getProperty("version"));
                         final float currentVersion = Float.parseFloat(CveDB.DB_SCHEMA_VERSION);
                         if (currentVersion > version) {
                             deleteAndRecreate = true;
                         }
                     } catch (NumberFormatException ex) {
                         deleteAndRecreate = true;
                     }
                 }
 
                 if (deleteAndRecreate) {
                     return updates;
                 }
 
                 final long lastUpdated = Long.parseLong(properties.getProperty(DataStoreMetaInfo.LAST_UPDATED, "0"));
                 final Date now = new Date();
                 final int days = Settings.getInt(Settings.KEYS.CVE_MODIFIED_VALID_FOR_DAYS, 7);
                 if (lastUpdated == updates.getTimeStamp(MODIFIED)) {
                     updates.clear(); //we don't need to update anything.
                 } else if (withinRange(lastUpdated, now.getTime(), days)) {
                     for (NvdCveInfo entry : updates) {
                         if (MODIFIED.equals(entry.getId())) {
                             entry.setNeedsUpdate(true);
                         } else {
                             entry.setNeedsUpdate(false);
                         }
                     }
                 } else { //we figure out which of the several XML files need to be downloaded.
                     for (NvdCveInfo entry : updates) {
                         if (MODIFIED.equals(entry.getId())) {
                             entry.setNeedsUpdate(true);
                         } else {
                             long currentTimestamp = 0;
                             try {
                                 currentTimestamp = Long.parseLong(properties.getProperty(DataStoreMetaInfo.LAST_UPDATED_BASE + entry.getId(), "0"));
                             } catch (NumberFormatException ex) {
                                 final String msg = String.format("Error parsing '%s' '%s' from nvdcve.lastupdated",
                                         DataStoreMetaInfo.LAST_UPDATED_BASE, entry.getId());
                                 Logger
                                         .getLogger(StandardUpdate.class
                                         .getName()).log(Level.FINE, msg, ex);
                             }
                             if (currentTimestamp == entry.getTimestamp()) {
                                 entry.setNeedsUpdate(false);
                             }
                         }
                     }
                 }
             } catch (NumberFormatException ex) {
                 final String msg = "An invalid schema version or timestamp exists in the data.properties file.";
                 Logger
                         .getLogger(StandardUpdate.class
                         .getName()).log(Level.WARNING, msg);
                 Logger.getLogger(StandardUpdate.class
                         .getName()).log(Level.FINE, null, ex);
             }
         }
         return updates;
     }
 
     /**
      * Retrieves the timestamps from the NVD CVE meta data file.
      *
      * @return the timestamp from the currently published nvdcve downloads page
      * @throws MalformedURLException thrown if the URL for the NVD CCE Meta data
      * is incorrect.
      * @throws DownloadFailedException thrown if there is an error downloading
      * the nvd cve meta data file
      * @throws InvalidDataException thrown if there is an exception parsing the
      * timestamps
      * @throws InvalidSettingException thrown if the settings are invalid
      */
     private Updateable retrieveCurrentTimestampsFromWeb()
             throws MalformedURLException, DownloadFailedException, InvalidDataException, InvalidSettingException {
 
         final Updateable updates = new Updateable();
         updates.add(MODIFIED, Settings.getString(Settings.KEYS.CVE_MODIFIED_20_URL),
                 Settings.getString(Settings.KEYS.CVE_MODIFIED_12_URL),
                 false);
 
         final int start = Settings.getInt(Settings.KEYS.CVE_START_YEAR);
         final int end = Calendar.getInstance().get(Calendar.YEAR);
         final String baseUrl20 = Settings.getString(Settings.KEYS.CVE_SCHEMA_2_0);
         final String baseUrl12 = Settings.getString(Settings.KEYS.CVE_SCHEMA_1_2);
         for (int i = start; i <= end; i++) {
             updates.add(Integer.toString(i), String.format(baseUrl20, i),
                     String.format(baseUrl12, i),
                     true);
         }
 
         return updates;
     }
 
     /**
      * Deletes the existing data directories.
      *
      * @throws IOException thrown if the directory cannot be deleted
      */
     protected void deleteExistingData() throws IOException {
         File data = Settings.getDataFile(Settings.KEYS.CVE_DATA_DIRECTORY);
         if (data.exists()) {
             FileUtils.delete(data);
         }
         data = DataStoreMetaInfo.getPropertiesFile();
         if (data.exists()) {
             FileUtils.delete(data);
         }
     }
 
     /**
      * Closes the CVE and CPE data stores.
      */
     protected void closeDataStores() {
         if (cveDB != null) {
             try {
                 cveDB.close();
             } catch (Exception ignore) {
                 Logger.getLogger(StandardUpdate.class.getName()).log(Level.FINEST, "Error closing the cveDB", ignore);
             }
         }
     }
 
     /**
      * Opens the CVE and CPE data stores.
      *
      * @throws UpdateException thrown if a data store cannot be opened
      */
     protected void openDataStores() throws UpdateException {
         //open the cve and cpe data stores
         try {
             cveDB = new CveDB();
             cveDB.open();
         } catch (IOException ex) {
             closeDataStores();
             Logger.getLogger(StandardUpdate.class.getName()).log(Level.FINE, "IO Error opening databases", ex);
             throw new UpdateException("Error updating the CPE/CVE data, please see the log file for more details.");
         } catch (SQLException ex) {
             closeDataStores();
             Logger.getLogger(StandardUpdate.class.getName()).log(Level.FINE, "SQL Exception opening databases", ex);
             throw new UpdateException("Error updating the CPE/CVE data, please see the log file for more details.");
         } catch (DatabaseException ex) {
             closeDataStores();
             Logger.getLogger(StandardUpdate.class.getName()).log(Level.FINE, "Database Exception opening databases", ex);
             throw new UpdateException("Error updating the CPE/CVE data, please see the log file for more details.");
         } catch (ClassNotFoundException ex) {
             closeDataStores();
             Logger.getLogger(StandardUpdate.class.getName()).log(Level.FINE, "Class not found exception opening databases", ex);
             throw new UpdateException("Error updating the CPE/CVE data, please see the log file for more details.");
         }
     }
 
     /**
      * Determines if the epoch date is within the range specified of the
      * compareTo epoch time. This takes the (compareTo-date)/1000/60/60/24 to
      * get the number of days. If the calculated days is less then the range the
      * date is considered valid.
      *
      * @param date the date to be checked.
      * @param compareTo the date to compare to.
      * @param range the range in days to be considered valid.
      * @return whether or not the date is within the range.
      */
     protected boolean withinRange(long date, long compareTo, int range) {
         final double differenceInDays = (compareTo - date) / 1000.0 / 60.0 / 60.0 / 24.0;
         return differenceInDays < range;
     }
 
     private void spinWaitForFuture(final Future<CallableDownloadTask> future) {
         //then wait for downloads to finish
         while (!future.isDone()) {
             try {
                 Thread.sleep(1000);
             } catch (InterruptedException ex) {
                 Logger.getLogger(StandardUpdate.class.getName()).log(Level.FINE, null, ex);
             }
         }
     }
 }

1		/*
2		* This file is part of dependency-check-core.
3		*
4		* Dependency-check-core is free software: you can redistribute it and/or modify it
5		* under the terms of the GNU General Public License as published by the Free
6		* Software Foundation, either version 3 of the License, or (at your option) any
7		* later version.
8		*
9		* Dependency-check-core is distributed in the hope that it will be useful, but
10		* WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11		* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
12		* details.
13		*
14		* You should have received a copy of the GNU General Public License along with
15		* dependency-check-core. If not, see http://www.gnu.org/licenses/.
16		*
17		* Copyright (c) 2012 Jeremy Long. All Rights Reserved.
18		*/
19		package org.owasp.dependencycheck.data.update;
20
21		import org.owasp.dependencycheck.data.nvdcve.InvalidDataException;
22		import java.io.File;
23		import java.io.IOException;
24		import java.net.MalformedURLException;
25		import java.sql.SQLException;
26		import java.util.Calendar;
27		import java.util.Date;
28		import java.util.HashSet;
29		import java.util.Iterator;
30		import java.util.Set;
31		import java.util.concurrent.ExecutionException;
32		import java.util.concurrent.ExecutorService;
33		import java.util.concurrent.Executors;
34		import java.util.concurrent.Future;
35		import java.util.logging.Level;
36		import java.util.logging.Logger;
37		import org.owasp.dependencycheck.data.UpdateException;
38		import org.owasp.dependencycheck.data.nvdcve.CveDB;
39		import org.owasp.dependencycheck.utils.DownloadFailedException;
40		import org.owasp.dependencycheck.utils.Settings;
41		import org.owasp.dependencycheck.data.nvdcve.DatabaseException;
42		import org.owasp.dependencycheck.utils.InvalidSettingException;
43		import static org.owasp.dependencycheck.data.update.DataStoreMetaInfo.MODIFIED;
44		import org.owasp.dependencycheck.utils.FileUtils;
45
46		/**
47		* Class responsible for updating the NVDCVE data store.
48		*
49		* @author Jeremy Long (jeremy.long@owasp.org)
50		*/
51		public class StandardUpdate {
52
53		/**
54		* The max thread pool size to use when downloading files.
55		*/
56	0	public static final int MAX_THREAD_POOL_SIZE = Settings.getInt(Settings.KEYS.MAX_DOWNLOAD_THREAD_POOL_SIZE, 3);
57		/**
58		* Information about the timestamps and URLs for data that needs to be
59		* updated.
60		*/
61		private DataStoreMetaInfo properties;
62		/**
63		* A collection of updateable NVD CVE items.
64		*/
65		private Updateable updateable;
66		/**
67		* A flag indicating whether or not the current data store should be
68		* deleted.
69		*/
70	0	private boolean deleteAndRecreate = false;
71		/**
72		* Reference to the Cve Database.
73		*/
74	0	private CveDB cveDB = null;
75
76		/**
77		* Gets whether or not an update is needed.
78		*
79		* @return true or false depending on whether an update is needed
80		*/
81		public boolean isUpdateNeeded() {
82	0	return updateable.isUpdateNeeded();
83		}
84
85		/**
86		* Set the value of deleteAndRecreate.
87		*
88		* @param deleteAndRecreate new value of deleteAndRecreate
89		*/
90		protected void setDeleteAndRecreate(boolean deleteAndRecreate) {
91	0	this.deleteAndRecreate = deleteAndRecreate;
92	0	}
93
94		/**
95		* Get the value of deleteAndRecreate.
96		*
97		* @return the value of deleteAndRecreate
98		*/
99		public boolean shouldDeleteAndRecreate() {
100	0	return deleteAndRecreate;
101		}
102
103		/**
104		* Constructs a new Standard Update Task.
105		*
106		* @throws MalformedURLException thrown if a configured URL is malformed
107		* @throws DownloadFailedException thrown if a timestamp cannot be checked
108		* on a configured URL
109		* @throws UpdateException thrown if there is an exception generating the
110		* update task
111		*/
112	0	public StandardUpdate() throws MalformedURLException, DownloadFailedException, UpdateException {
113	0	properties = new DataStoreMetaInfo();
114	0	updateable = updatesNeeded();
115	0	}
116
117		/**
118		* <p>Downloads the latest NVD CVE XML file from the web and imports it into
119		* the current CVE Database.</p>
120		*
121		* @throws UpdateException is thrown if there is an error updating the
122		* database
123		*/
124		public void update() throws UpdateException {
125	0	int maxUpdates = 0;
126		try {
127	0	for (NvdCveInfo cve : updateable) {
128	0	if (cve.getNeedsUpdate()) {
129	0	maxUpdates += 1;
130		}
131		}
132	0	if (maxUpdates <= 0) {
133		return;
134		}
135	0	if (maxUpdates > 3) {
136	0	Logger.getLogger(StandardUpdate.class.getName()).log(Level.INFO,
137		"NVD CVE requires several updates; this could take a couple of minutes.");
138		}
139	0	if (maxUpdates > 0) {
140	0	openDataStores();
141		}
142
143	0	final int poolSize = (MAX_THREAD_POOL_SIZE > maxUpdates) ? MAX_THREAD_POOL_SIZE : maxUpdates;
144	0	final ExecutorService downloadExecutor = Executors.newFixedThreadPool(poolSize);
145	0	final ExecutorService processExecutor = Executors.newSingleThreadExecutor();
146	0	final Set<Future<CallableDownloadTask>> downloadFutures = new HashSet<Future<CallableDownloadTask>>(maxUpdates);
147	0	final Set<Future<ProcessTask>> processFutures = new HashSet<Future<ProcessTask>>(maxUpdates);
148	0	int ctr = 0;
149	0	for (NvdCveInfo cve : updateable) {
150	0	if (cve.getNeedsUpdate()) {
151	0	ctr += 1;
152		final File file1;
153		final File file2;
154		try {
155	0	file1 = File.createTempFile("cve" + cve.getId() + "_", ".xml");
156	0	file2 = File.createTempFile("cve_1_2_" + cve.getId() + "_", ".xml");
157	0	} catch (IOException ex) {
158	0	throw new UpdateException(ex);
159	0	}
160	0	final CallableDownloadTask call = new CallableDownloadTask(cve, file1, file2);
161	0	downloadFutures.add(downloadExecutor.submit(call));
162
163	0	boolean waitForFuture = ctr % 2 == 0;
164
165	0	final Iterator<Future<CallableDownloadTask>> itr = downloadFutures.iterator();
166	0	while (itr.hasNext()) {
167	0	final Future<CallableDownloadTask> future = itr.next();
168	0	if (waitForFuture) { //only allow two NVD/CVE files to be downloaded at a time
169	0	spinWaitForFuture(future);
170		}
171	0	if (future.isDone()) { //if we find something complete, add it to the process queue
172		try {
173	0	final CallableDownloadTask filePair = future.get();
174	0	itr.remove();
175	0	final ProcessTask task = new ProcessTask(cveDB, properties, filePair);
176	0	processFutures.add(processExecutor.submit(task));
177	0	} catch (InterruptedException ex) {
178	0	downloadExecutor.shutdownNow();
179	0	Logger.getLogger(StandardUpdate.class.getName()).log(Level.FINE, "Thread was interupted", ex);
180	0	throw new UpdateException(ex);
181	0	} catch (ExecutionException ex) {
182	0	downloadExecutor.shutdownNow();
183	0	Logger.getLogger(StandardUpdate.class.getName()).log(Level.SEVERE, null, ex);
184	0	throw new UpdateException(ex);
185	0	}
186		}
187	0	}
188
189	0	}
190		}
191
192		try {
193	0	final Iterator<Future<CallableDownloadTask>> itr = downloadFutures.iterator();
194	0	while (itr.hasNext()) {
195	0	final Future<CallableDownloadTask> future = itr.next();
196	0	final CallableDownloadTask filePair = future.get();
197	0	final ProcessTask task = new ProcessTask(cveDB, properties, filePair);
198	0	processFutures.add(processExecutor.submit(task));
199	0	}
200	0	} catch (InterruptedException ex) {
201	0	downloadExecutor.shutdownNow();
202	0	Logger.getLogger(StandardUpdate.class.getName()).log(Level.FINE, "Thread was interupted during download", ex);
203	0	throw new UpdateException(ex);
204	0	} catch (ExecutionException ex) {
205	0	downloadExecutor.shutdownNow();
206	0	Logger.getLogger(StandardUpdate.class.getName()).log(Level.FINE, "Execution Exception during download", ex);
207	0	throw new UpdateException(ex);
208		} finally {
209	0	downloadExecutor.shutdown();
210	0	}
211
212	0	for (Future<ProcessTask> future : processFutures) {
213		try {
214	0	final ProcessTask task = future.get();
215	0	if (task.getException() != null) {
216	0	throw task.getException();
217		}
218	0	} catch (InterruptedException ex) {
219	0	processExecutor.shutdownNow();
220	0	Logger.getLogger(StandardUpdate.class.getName()).log(Level.FINE, "Thread was interupted during processing", ex);
221	0	throw new UpdateException(ex);
222	0	} catch (ExecutionException ex) {
223	0	processExecutor.shutdownNow();
224	0	Logger.getLogger(StandardUpdate.class.getName()).log(Level.FINE, "Execution Exception during process", ex);
225	0	throw new UpdateException(ex);
226		} finally {
227	0	processExecutor.shutdown();
228	0	}
229		}
230
231	0	if (maxUpdates >= 1) { //ensure the modified file date gets written
232	0	properties.save(updateable.get(MODIFIED));
233	0	cveDB.cleanupDatabase();
234		}
235		} finally {
236	0	closeDataStores();
237	0	}
238	0	}
239
240		//<editor-fold defaultstate="collapsed" desc="OLD version of update() - not multithreaded">
241		/*
242		* TODO - remove this
243		public void update() throws UpdateException {
244		try {
245		int maxUpdates = 0;
246		for (NvdCveInfo cve : getUpdateable()) {
247		if (cve.getNeedsUpdate()) {
248		maxUpdates += 1;
249		}
250		}
251		if (maxUpdates > 3) {
252		Logger.getLogger(StandardUpdate.class.getName()).log(Level.INFO,
253		"NVD CVE requires several updates; this could take a couple of minutes.");
254		}
255		if (maxUpdates > 0) {
256		openDataStores();
257		}
258
259		int count = 0;
260		for (NvdCveInfo cve : getUpdateable()) {
261		if (cve.getNeedsUpdate()) {
262		count += 1;
263		Logger.getLogger(StandardUpdate.class.getName()).log(Level.INFO,
264		"Updating NVD CVE ({0} of {1})", new Object[]{count, maxUpdates});
265		URL url = new URL(cve.getUrl());
266		File outputPath = null;
267		File outputPath12 = null;
268		try {
269		Logger.getLogger(StandardUpdate.class.getName()).log(Level.INFO,
270		"Downloading {0}", cve.getUrl());
271		outputPath = File.createTempFile("cve" + cve.getId() + "_", ".xml");
272		Downloader.fetchFile(url, outputPath);
273
274		url = new URL(cve.getOldSchemaVersionUrl());
275		outputPath12 = File.createTempFile("cve_1_2_" + cve.getId() + "_", ".xml");
276		Downloader.fetchFile(url, outputPath12);
277
278		Logger.getLogger(StandardUpdate.class.getName()).log(Level.INFO,
279		"Processing {0}", cve.getUrl());
280
281		importXML(outputPath, outputPath12);
282
283		getCveDB().commit();
284		getProperties().save(cve);
285
286		Logger.getLogger(StandardUpdate.class.getName()).log(Level.INFO,
287		"Completed update {0} of {1}", new Object[]{count, maxUpdates});
288		} catch (FileNotFoundException ex) {
289		throw new UpdateException(ex);
290		} catch (ParserConfigurationException ex) {
291		throw new UpdateException(ex);
292		} catch (SAXException ex) {
293		throw new UpdateException(ex);
294		} catch (IOException ex) {
295		throw new UpdateException(ex);
296		} catch (SQLException ex) {
297		throw new UpdateException(ex);
298		} catch (DatabaseException ex) {
299		throw new UpdateException(ex);
300		} catch (ClassNotFoundException ex) {
301		throw new UpdateException(ex);
302		} finally {
303		boolean deleted = false;
304		try {
305		if (outputPath != null && outputPath.exists()) {
306		deleted = outputPath.delete();
307		}
308		} finally {
309		if (outputPath != null && (outputPath.exists() \|\| !deleted)) {
310		outputPath.deleteOnExit();
311		}
312		}
313		try {
314		deleted = false;
315		if (outputPath12 != null && outputPath12.exists()) {
316		deleted = outputPath12.delete();
317		}
318		} finally {
319		if (outputPath12 != null && (outputPath12.exists() \|\| !deleted)) {
320		outputPath12.deleteOnExit();
321		}
322		}
323		}
324		}
325		}
326		if (maxUpdates >= 1) { //ensure the modified file date gets written
327		getProperties().save(getUpdateable().get(MODIFIED));
328		getCveDB().cleanupDatabase();
329		}
330		} catch (MalformedURLException ex) {
331		throw new UpdateException(ex);
332		} finally {
333		closeDataStores();
334		}
335		}
336		*/
337		//</editor-fold>
338		/**
339		* Determines if the index needs to be updated. This is done by fetching the
340		* NVD CVE meta data and checking the last update date. If the data needs to
341		* be refreshed this method will return the NvdCveUrl for the files that
342		* need to be updated.
343		*
344		* @return the collection of files that need to be updated
345		* @throws MalformedURLException is thrown if the URL for the NVD CVE Meta
346		* data is incorrect
347		* @throws DownloadFailedException is thrown if there is an error.
348		* downloading the NVD CVE download data file
349		* @throws UpdateException Is thrown if there is an issue with the last
350		* updated properties file
351		*/
352		protected final Updateable updatesNeeded() throws MalformedURLException, DownloadFailedException, UpdateException {
353	0	Updateable updates = null;
354		try {
355	0	updates = retrieveCurrentTimestampsFromWeb();
356	0	} catch (InvalidDataException ex) {
357	0	final String msg = "Unable to retrieve valid timestamp from nvd cve downloads page";
358	0	Logger
359		.getLogger(StandardUpdate.class
360		.getName()).log(Level.FINE, msg, ex);
361	0	throw new DownloadFailedException(msg, ex);
362	0	} catch (InvalidSettingException ex) {
363	0	Logger.getLogger(StandardUpdate.class
364		.getName()).log(Level.FINE, "Invalid setting found when retrieving timestamps", ex);
365	0	throw new DownloadFailedException(
366		"Invalid settings", ex);
367	0	}
368
369	0	if (updates == null) {
370	0	throw new DownloadFailedException("Unable to retrieve the timestamps of the currently published NVD CVE data");
371		}
372	0	if (!properties.isEmpty()) {
373		try {
374		float version;
375
376	0	if (properties.getProperty("version") == null) {
377	0	deleteAndRecreate = true;
378		} else {
379		try {
380	0	version = Float.parseFloat(properties.getProperty("version"));
381	0	final float currentVersion = Float.parseFloat(CveDB.DB_SCHEMA_VERSION);
382	0	if (currentVersion > version) {
383	0	deleteAndRecreate = true;
384		}
385	0	} catch (NumberFormatException ex) {
386	0	deleteAndRecreate = true;
387	0	}
388		}
389
390	0	if (deleteAndRecreate) {
391	0	return updates;
392		}
393
394	0	final long lastUpdated = Long.parseLong(properties.getProperty(DataStoreMetaInfo.LAST_UPDATED, "0"));
395	0	final Date now = new Date();
396	0	final int days = Settings.getInt(Settings.KEYS.CVE_MODIFIED_VALID_FOR_DAYS, 7);
397	0	if (lastUpdated == updates.getTimeStamp(MODIFIED)) {
398	0	updates.clear(); //we don't need to update anything.
399	0	} else if (withinRange(lastUpdated, now.getTime(), days)) {
400	0	for (NvdCveInfo entry : updates) {
401	0	if (MODIFIED.equals(entry.getId())) {
402	0	entry.setNeedsUpdate(true);
403		} else {
404	0	entry.setNeedsUpdate(false);
405		}
406		}
407		} else { //we figure out which of the several XML files need to be downloaded.
408	0	for (NvdCveInfo entry : updates) {
409	0	if (MODIFIED.equals(entry.getId())) {
410	0	entry.setNeedsUpdate(true);
411		} else {
412	0	long currentTimestamp = 0;
413		try {
414	0	currentTimestamp = Long.parseLong(properties.getProperty(DataStoreMetaInfo.LAST_UPDATED_BASE + entry.getId(), "0"));
415	0	} catch (NumberFormatException ex) {
416	0	final String msg = String.format("Error parsing '%s' '%s' from nvdcve.lastupdated",
417		DataStoreMetaInfo.LAST_UPDATED_BASE, entry.getId());
418	0	Logger
419		.getLogger(StandardUpdate.class
420		.getName()).log(Level.FINE, msg, ex);
421	0	}
422	0	if (currentTimestamp == entry.getTimestamp()) {
423	0	entry.setNeedsUpdate(false);
424		}
425	0	}
426		}
427		}
428	0	} catch (NumberFormatException ex) {
429	0	final String msg = "An invalid schema version or timestamp exists in the data.properties file.";
430	0	Logger
431		.getLogger(StandardUpdate.class
432		.getName()).log(Level.WARNING, msg);
433	0	Logger.getLogger(StandardUpdate.class
434		.getName()).log(Level.FINE, null, ex);
435	0	}
436		}
437	0	return updates;
438		}
439
440		/**
441		* Retrieves the timestamps from the NVD CVE meta data file.
442		*
443		* @return the timestamp from the currently published nvdcve downloads page
444		* @throws MalformedURLException thrown if the URL for the NVD CCE Meta data
445		* is incorrect.
446		* @throws DownloadFailedException thrown if there is an error downloading
447		* the nvd cve meta data file
448		* @throws InvalidDataException thrown if there is an exception parsing the
449		* timestamps
450		* @throws InvalidSettingException thrown if the settings are invalid
451		*/
452		private Updateable retrieveCurrentTimestampsFromWeb()
453		throws MalformedURLException, DownloadFailedException, InvalidDataException, InvalidSettingException {
454
455	0	final Updateable updates = new Updateable();
456	0	updates.add(MODIFIED, Settings.getString(Settings.KEYS.CVE_MODIFIED_20_URL),
457		Settings.getString(Settings.KEYS.CVE_MODIFIED_12_URL),
458		false);
459
460	0	final int start = Settings.getInt(Settings.KEYS.CVE_START_YEAR);
461	0	final int end = Calendar.getInstance().get(Calendar.YEAR);
462	0	final String baseUrl20 = Settings.getString(Settings.KEYS.CVE_SCHEMA_2_0);
463	0	final String baseUrl12 = Settings.getString(Settings.KEYS.CVE_SCHEMA_1_2);
464	0	for (int i = start; i <= end; i++) {
465	0	updates.add(Integer.toString(i), String.format(baseUrl20, i),
466		String.format(baseUrl12, i),
467		true);
468		}
469
470	0	return updates;
471		}
472
473		/**
474		* Deletes the existing data directories.
475		*
476		* @throws IOException thrown if the directory cannot be deleted
477		*/
478		protected void deleteExistingData() throws IOException {
479	0	File data = Settings.getDataFile(Settings.KEYS.CVE_DATA_DIRECTORY);
480	0	if (data.exists()) {
481	0	FileUtils.delete(data);
482		}
483	0	data = DataStoreMetaInfo.getPropertiesFile();
484	0	if (data.exists()) {
485	0	FileUtils.delete(data);
486		}
487	0	}
488
489		/**
490		* Closes the CVE and CPE data stores.
491		*/
492		protected void closeDataStores() {
493	0	if (cveDB != null) {
494		try {
495	0	cveDB.close();
496	0	} catch (Exception ignore) {
497	0	Logger.getLogger(StandardUpdate.class.getName()).log(Level.FINEST, "Error closing the cveDB", ignore);
498	0	}
499		}
500	0	}
501
502		/**
503		* Opens the CVE and CPE data stores.
504		*
505		* @throws UpdateException thrown if a data store cannot be opened
506		*/
507		protected void openDataStores() throws UpdateException {
508		//open the cve and cpe data stores
509		try {
510	0	cveDB = new CveDB();
511	0	cveDB.open();
512	0	} catch (IOException ex) {
513	0	closeDataStores();
514	0	Logger.getLogger(StandardUpdate.class.getName()).log(Level.FINE, "IO Error opening databases", ex);
515	0	throw new UpdateException("Error updating the CPE/CVE data, please see the log file for more details.");
516	0	} catch (SQLException ex) {
517	0	closeDataStores();
518	0	Logger.getLogger(StandardUpdate.class.getName()).log(Level.FINE, "SQL Exception opening databases", ex);
519	0	throw new UpdateException("Error updating the CPE/CVE data, please see the log file for more details.");
520	0	} catch (DatabaseException ex) {
521	0	closeDataStores();
522	0	Logger.getLogger(StandardUpdate.class.getName()).log(Level.FINE, "Database Exception opening databases", ex);
523	0	throw new UpdateException("Error updating the CPE/CVE data, please see the log file for more details.");
524	0	} catch (ClassNotFoundException ex) {
525	0	closeDataStores();
526	0	Logger.getLogger(StandardUpdate.class.getName()).log(Level.FINE, "Class not found exception opening databases", ex);
527	0	throw new UpdateException("Error updating the CPE/CVE data, please see the log file for more details.");
528	0	}
529	0	}
530
531		/**
532		* Determines if the epoch date is within the range specified of the
533		* compareTo epoch time. This takes the (compareTo-date)/1000/60/60/24 to
534		* get the number of days. If the calculated days is less then the range the
535		* date is considered valid.
536		*
537		* @param date the date to be checked.
538		* @param compareTo the date to compare to.
539		* @param range the range in days to be considered valid.
540		* @return whether or not the date is within the range.
541		*/
542		protected boolean withinRange(long date, long compareTo, int range) {
543	0	final double differenceInDays = (compareTo - date) / 1000.0 / 60.0 / 60.0 / 24.0;
544	0	return differenceInDays < range;
545		}
546
547		private void spinWaitForFuture(final Future<CallableDownloadTask> future) {
548		//then wait for downloads to finish
549	0	while (!future.isDone()) {
550		try {
551	0	Thread.sleep(1000);
552	0	} catch (InterruptedException ex) {
553	0	Logger.getLogger(StandardUpdate.class.getName()).log(Level.FINE, null, ex);
554	0	}
555		}
556	0	}
557		}