Updates the data store to the latest version.
+ *
+ * @throws UpdateException is thrown if there is an error updating the
+ * database
+ */
+ public abstract void update() throws UpdateException;
+ /**
+ * A flag indicating whether or not the current data store should be
+ * deleted.
+ */
+ private boolean deleteAndRecreate = false;
+ protected Updateable updatesNeeded = null;
+
+ /**
+ * Get the value of deleteAndRecreate
+ *
+ * @return the value of deleteAndRecreate
+ */
+ public boolean shouldDeleteAndRecreate() {
+ return deleteAndRecreate;
+ }
+
+ /**
+ * Set the value of deleteAndRecreate
+ *
+ * @param deleteAndRecreate new value of deleteAndRecreate
+ */
+ public void setDeleteAndRecreate(boolean deleteAndRecreate) {
+ this.deleteAndRecreate = deleteAndRecreate;
+ }
+
+ /**
+ * Deletes the existing data directories.
+ *
+ * @throws IOException thrown if the directory cannot be deleted
+ */
+ protected void deleteExistingData() throws IOException {
+ File data = Settings.getFile(Settings.KEYS.CVE_DATA_DIRECTORY);
+ if (data.exists()) {
+ FileUtils.delete(data);
+ }
+ data = Settings.getFile(Settings.KEYS.CPE_DATA_DIRECTORY);
+ if (data.exists()) {
+ FileUtils.delete(data);
+ }
+ data = DataStoreMetaInfo.getPropertiesFile();
+ if (data.exists()) {
+ FileUtils.delete(data);
+ }
+ }
+
+ /**
+ * Closes the CVE and CPE data stores.
+ */
+ protected void closeDataStores() {
+ if (cveDB != null) {
+ try {
+ cveDB.close();
+ } catch (Exception ignore) {
+ Logger.getLogger(AbstractUpdate.class.getName()).log(Level.FINEST, "Error closing the cveDB", ignore);
+ }
+ }
+ if (cpeIndex != null) {
+ try {
+ cpeIndex.close();
+ } catch (Exception ignore) {
+ Logger.getLogger(AbstractUpdate.class.getName()).log(Level.FINEST, "Error closing the cpeIndex", ignore);
+ }
+ }
+ }
+
+ /**
+ * Opens the CVE and CPE data stores.
+ *
+ * @throws UpdateException thrown if a data store cannot be opened
+ */
+ protected void openDataStores() throws UpdateException {
+ //open the cve and cpe data stores
+ try {
+ cveDB = new CveDB();
+ cveDB.open();
+ cpeIndex = new CpeIndexWriter();
+ cpeIndex.open();
+ } catch (IOException ex) {
+ closeDataStores();
+ Logger.getLogger(AbstractUpdate.class.getName()).log(Level.FINE, "IO Error opening databases", ex);
+ throw new UpdateException("Error updating the CPE/CVE data, please see the log file for more details.");
+ } catch (SQLException ex) {
+ closeDataStores();
+ Logger.getLogger(AbstractUpdate.class.getName()).log(Level.FINE, "SQL Exception opening databases", ex);
+ throw new UpdateException("Error updating the CPE/CVE data, please see the log file for more details.");
+ } catch (DatabaseException ex) {
+ closeDataStores();
+ Logger.getLogger(AbstractUpdate.class.getName()).log(Level.FINE, "Database Exception opening databases", ex);
+ throw new UpdateException("Error updating the CPE/CVE data, please see the log file for more details.");
+ } catch (ClassNotFoundException ex) {
+ closeDataStores();
+ Logger.getLogger(AbstractUpdate.class.getName()).log(Level.FINE, "Class not found exception opening databases", ex);
+ throw new UpdateException("Error updating the CPE/CVE data, please see the log file for more details.");
+ }
+ }
+
+ /**
+ * Determines if the epoch date is within the range specified of the
+ * compareTo epoch time. This takes the (compareTo-date)/1000/60/60/24 to
+ * get the number of days. If the calculated days is less then the range the
+ * date is considered valid.
+ *
+ * @param date the date to be checked.
+ * @param compareTo the date to compare to.
+ * @param range the range in days to be considered valid.
+ * @return whether or not the date is within the range.
+ */
+ protected boolean withinRange(long date, long compareTo, int range) {
+ final double differenceInDays = (compareTo - date) / 1000.0 / 60.0 / 60.0 / 24.0;
+ return differenceInDays < range;
+ }
+
+ /**
+ * Imports the NVD CVE XML File into the Lucene Index.
+ *
+ * @param file the file containing the NVD CVE XML
+ * @param oldVersion contains the file containing the NVD CVE XML 1.2
+ * @throws ParserConfigurationException is thrown if there is a parser
+ * configuration exception
+ * @throws SAXException is thrown if there is a SAXException
+ * @throws IOException is thrown if there is a IO Exception
+ * @throws SQLException is thrown if there is a SQL exception
+ * @throws DatabaseException is thrown if there is a database exception
+ * @throws ClassNotFoundException thrown if the h2 database driver cannot be
+ * loaded
+ */
+ protected void importXML(File file, File oldVersion)
+ throws ParserConfigurationException, SAXException, IOException, SQLException, DatabaseException, ClassNotFoundException {
+
+ final SAXParserFactory factory = SAXParserFactory.newInstance();
+ final SAXParser saxParser = factory.newSAXParser();
+
+ final NvdCve12Handler cve12Handler = new NvdCve12Handler();
+ saxParser.parse(oldVersion, cve12Handler);
+ final Map> prevVersionVulnMap = cve12Handler.getVulnerabilities();
+
+ final NvdCve20Handler cve20Handler = new NvdCve20Handler();
+ cve20Handler.setCveDB(cveDB);
+ cve20Handler.setPrevVersionVulnMap(prevVersionVulnMap);
+ cve20Handler.setCpeIndex(cpeIndex);
+ saxParser.parse(file, cve20Handler);
+ }
+}
diff --git a/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/BatchUpdate.java b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/BatchUpdate.java
new file mode 100644
index 000000000..4095e072c
--- /dev/null
+++ b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/BatchUpdate.java
@@ -0,0 +1,266 @@
+/*
+ * This file is part of dependency-check-core.
+ *
+ * Dependency-check-core is free software: you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation, either version 3 of the License, or (at your option) any
+ * later version.
+ *
+ * Dependency-check-core is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+ * details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * dependency-check-core. If not, see http://www.gnu.org/licenses/.
+ *
+ * Copyright (c) 2012 Jeremy Long. All Rights Reserved.
+ */
+package org.owasp.dependencycheck.data.update;
+
+import org.owasp.dependencycheck.data.nvdcve.InvalidDataException;
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import javax.xml.parsers.ParserConfigurationException;
+import org.xml.sax.SAXException;
+import java.net.MalformedURLException;
+import java.net.URISyntaxException;
+import java.net.URL;
+import java.sql.SQLException;
+import java.util.Calendar;
+import java.util.Date;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+import org.owasp.dependencycheck.data.UpdateException;
+import org.owasp.dependencycheck.data.nvdcve.CveDB;
+import org.owasp.dependencycheck.utils.DownloadFailedException;
+import org.owasp.dependencycheck.utils.Downloader;
+import org.owasp.dependencycheck.utils.FileUtils;
+import org.owasp.dependencycheck.utils.Settings;
+import org.owasp.dependencycheck.data.nvdcve.DatabaseException;
+import org.owasp.dependencycheck.utils.InvalidSettingException;
+import static org.owasp.dependencycheck.data.update.DataStoreMetaInfo.BATCH;
+import static org.owasp.dependencycheck.data.update.DataStoreMetaInfo.MODIFIED;
+
+/**
+ * Class responsible for updating the CPE and NVDCVE data stores.
+ *
+ * @author Jeremy Long (jeremy.long@owasp.org)
+ */
+public class BatchUpdate extends AbstractUpdate {
+
+ public BatchUpdate() throws MalformedURLException, DownloadFailedException, UpdateException {
+ super();
+ }
+ /**
+ * A flag indicating whether or not the batch update should be performed.
+ */
+ private boolean doBatchUpdate;
+
+ /**
+ * Get the value of doBatchUpdate
+ *
+ * @return the value of doBatchUpdate
+ */
+ protected boolean isDoBatchUpdate() {
+ return doBatchUpdate;
+ }
+
+ /**
+ * Set the value of doBatchUpdate
+ *
+ * @param doBatchUpdate new value of doBatchUpdate
+ */
+ protected void setDoBatchUpdate(boolean doBatchUpdate) {
+ this.doBatchUpdate = doBatchUpdate;
+ }
+
+ /**
+ * Downloads the latest NVD CVE XML file from the web and imports it into
+ * the current CVE Database.
+ *
+ * @throws UpdateException is thrown if there is an error updating the
+ * database
+ */
+ @Override
+ public void update() throws UpdateException {
+ if (properties.isBatchUpdateMode() && doBatchUpdate) {
+ final String batchSrc = Settings.getString(Settings.KEYS.BATCH_UPDATE_URL);
+ File tmp = null;
+ try {
+ deleteExistingData();
+ final File dataDirectory = CveDB.getDataDirectory().getParentFile();
+ final URL batchUrl = new URL(batchSrc);
+ if ("file".equals(batchUrl.getProtocol())) {
+ try {
+ tmp = new File(batchUrl.toURI());
+ } catch (URISyntaxException ex) {
+ final String msg = String.format("Invalid batch update URI: %s", batchSrc);
+ throw new UpdateException(msg, ex);
+ }
+ } else if ("http".equals(batchUrl.getProtocol())
+ || "https".equals(batchUrl.getProtocol())) {
+ tmp = File.createTempFile("batch_", ".zip");
+ Downloader.fetchFile(batchUrl, tmp);
+ }
+ //TODO add FTP?
+ FileUtils.extractFiles(tmp, dataDirectory);
+
+ } catch (IOException ex) {
+ final String msg = String.format("IO Exception Occured performing batch update using: %s", batchSrc);
+ throw new UpdateException(msg, ex);
+ } finally {
+ if (tmp != null && !tmp.delete()) {
+ tmp.deleteOnExit();
+ }
+ }
+ }
+ }
+
+ /**
+ * Determines if the index needs to be updated. This is done by fetching the
+ * NVD CVE meta data and checking the last update date. If the data needs to
+ * be refreshed this method will return the NvdCveUrl for the files that
+ * need to be updated.
+ *
+ * @return the collection of files that need to be updated
+ * @throws MalformedURLException is thrown if the URL for the NVD CVE Meta
+ * data is incorrect
+ * @throws DownloadFailedException is thrown if there is an error.
+ * downloading the NVD CVE download data file
+ * @throws UpdateException Is thrown if there is an issue with the last
+ * updated properties file
+ */
+ @Override
+ public Updateable updatesNeeded() throws MalformedURLException, DownloadFailedException, UpdateException {
+ Updateable updates = null;
+ try {
+ updates = retrieveCurrentTimestampsFromWeb();
+ } catch (InvalidDataException ex) {
+ final String msg = "Unable to retrieve valid timestamp from nvd cve downloads page";
+ Logger.getLogger(BatchUpdate.class.getName()).log(Level.FINE, msg, ex);
+ throw new DownloadFailedException(msg, ex);
+ } catch (InvalidSettingException ex) {
+ Logger.getLogger(BatchUpdate.class.getName()).log(Level.FINE, "Invalid setting found when retrieving timestamps", ex);
+ throw new DownloadFailedException("Invalid settings", ex);
+ }
+
+ if (updates == null) {
+ throw new DownloadFailedException("Unable to retrieve the timestamps of the currently published NVD CVE data");
+ }
+
+ if (!properties.isEmpty()) {
+ try {
+ boolean deleteAndRecreate = false;
+ float version;
+
+ if (properties.getProperty("version") == null) {
+ deleteAndRecreate = true;
+ } else {
+ try {
+ version = Float.parseFloat(properties.getProperty("version"));
+ final float currentVersion = Float.parseFloat(CveDB.DB_SCHEMA_VERSION);
+ if (currentVersion > version) {
+ deleteAndRecreate = true;
+ }
+ } catch (NumberFormatException ex) {
+ deleteAndRecreate = true;
+ }
+ }
+
+ final NvdCveInfo batchInfo = updates.get(BATCH);
+ if (properties.isBatchUpdateMode() && batchInfo != null) {
+ final long lastUpdated = Long.parseLong(properties.getProperty(DataStoreMetaInfo.BATCH, "0"));
+ if (lastUpdated != batchInfo.getTimestamp()) {
+ deleteAndRecreate = true;
+ }
+ }
+
+ if (deleteAndRecreate) {
+ setDoBatchUpdate(properties.isBatchUpdateMode());
+ try {
+ deleteExistingData();
+ } catch (IOException ex) {
+ final String msg = "Unable to delete existing data";
+ Logger.getLogger(BatchUpdate.class.getName()).log(Level.WARNING, msg);
+ Logger.getLogger(BatchUpdate.class.getName()).log(Level.FINE, null, ex);
+ }
+ return updates;
+ }
+
+ final long lastUpdated = Long.parseLong(properties.getProperty(DataStoreMetaInfo.LAST_UPDATED, "0"));
+ final Date now = new Date();
+ final int days = Settings.getInt(Settings.KEYS.CVE_MODIFIED_VALID_FOR_DAYS, 7);
+ final int start = Settings.getInt(Settings.KEYS.CVE_START_YEAR, 2002);
+ final int end = Calendar.getInstance().get(Calendar.YEAR);
+ if (lastUpdated == updates.get(MODIFIED).getTimestamp()) {
+ updates.clear(); //we don't need to update anything.
+ setDoBatchUpdate(properties.isBatchUpdateMode());
+ } else if (withinRange(lastUpdated, now.getTime(), days)) {
+ updates.get(MODIFIED).setNeedsUpdate(true);
+ if (properties.isBatchUpdateMode()) {
+ setDoBatchUpdate(false);
+ } else {
+ for (int i = start; i <= end; i++) {
+ updates.get(String.valueOf(i)).setNeedsUpdate(false);
+ }
+ }
+ } else if (properties.isBatchUpdateMode()) {
+ updates.get(MODIFIED).setNeedsUpdate(true);
+ setDoBatchUpdate(true);
+ } else { //we figure out which of the several XML files need to be downloaded.
+ updates.get(MODIFIED).setNeedsUpdate(false);
+ for (int i = start; i <= end; i++) {
+ final NvdCveInfo cve = updates.get(String.valueOf(i));
+ long currentTimestamp = 0;
+ try {
+ currentTimestamp = Long.parseLong(properties.getProperty(DataStoreMetaInfo.LAST_UPDATED_BASE + String.valueOf(i), "0"));
+ } catch (NumberFormatException ex) {
+ final String msg = String.format("Error parsing '%s' '%s' from nvdcve.lastupdated",
+ DataStoreMetaInfo.LAST_UPDATED_BASE, String.valueOf(i));
+ Logger.getLogger(BatchUpdate.class.getName()).log(Level.FINE, msg, ex);
+ }
+ if (currentTimestamp == cve.getTimestamp()) {
+ cve.setNeedsUpdate(false); //they default to true.
+ }
+ }
+ }
+ } catch (NumberFormatException ex) {
+ final String msg = "An invalid schema version or timestamp exists in the data.properties file.";
+ Logger.getLogger(BatchUpdate.class.getName()).log(Level.WARNING, msg);
+ Logger.getLogger(BatchUpdate.class.getName()).log(Level.FINE, null, ex);
+ setDoBatchUpdate(properties.isBatchUpdateMode());
+ }
+ }
+ return updates;
+ }
+
+ /**
+ * Retrieves the timestamps from the NVD CVE meta data file.
+ *
+ * @return the timestamp from the currently published nvdcve downloads page
+ * @throws MalformedURLException thrown if the URL for the NVD CCE Meta data
+ * is incorrect.
+ * @throws DownloadFailedException thrown if there is an error downloading
+ * the nvd cve meta data file
+ * @throws InvalidDataException thrown if there is an exception parsing the
+ * timestamps
+ * @throws InvalidSettingException thrown if the settings are invalid
+ */
+ private Updateable retrieveCurrentTimestampsFromWeb()
+ throws MalformedURLException, DownloadFailedException, InvalidDataException, InvalidSettingException {
+ Updateable updates = new Updateable();
+ updates.add(BATCH, Settings.getString(Settings.KEYS.BATCH_UPDATE_URL),
+ null, false);
+
+ String url = Settings.getString(Settings.KEYS.CVE_MODIFIED_20_URL, "");
+ if (!url.isEmpty()) {
+ final NvdCveInfo item = new NvdCveInfo();
+ updates.add(MODIFIED, url,
+ Settings.getString(Settings.KEYS.CVE_MODIFIED_12_URL),
+ false);
+ }
+ return updates;
+ }
+}
diff --git a/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/DatabaseUpdater.java b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/DatabaseUpdater.java
index f69ae3c63..5227d51c0 100644
--- a/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/DatabaseUpdater.java
+++ b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/DatabaseUpdater.java
@@ -18,40 +18,19 @@
*/
package org.owasp.dependencycheck.data.update;
-import org.owasp.dependencycheck.data.nvdcve.NvdCve12Handler;
-import org.owasp.dependencycheck.data.nvdcve.NvdCve20Handler;
-import org.owasp.dependencycheck.data.nvdcve.InvalidDataException;
import java.io.File;
-import java.io.FileNotFoundException;
import java.io.IOException;
-import javax.xml.parsers.ParserConfigurationException;
-import org.xml.sax.SAXException;
import org.owasp.dependencycheck.data.CachedWebDataSource;
import java.net.MalformedURLException;
-import java.net.URISyntaxException;
-import java.net.URL;
-import java.sql.SQLException;
-import java.util.Calendar;
-import java.util.Date;
-import java.util.List;
-import java.util.Map;
-import java.util.TreeMap;
import java.util.logging.Level;
import java.util.logging.Logger;
-import javax.xml.parsers.SAXParser;
-import javax.xml.parsers.SAXParserFactory;
+import org.owasp.dependencycheck.concurrency.DirectoryLockException;
+import org.owasp.dependencycheck.concurrency.DirectorySpinLock;
+import org.owasp.dependencycheck.concurrency.InvalidDirectoryException;
import org.owasp.dependencycheck.data.UpdateException;
-import org.owasp.dependencycheck.data.cpe.CpeIndexWriter;
-import org.owasp.dependencycheck.data.nvdcve.CveDB;
-import org.owasp.dependencycheck.dependency.VulnerableSoftware;
import org.owasp.dependencycheck.utils.DownloadFailedException;
-import org.owasp.dependencycheck.utils.Downloader;
import org.owasp.dependencycheck.utils.FileUtils;
import org.owasp.dependencycheck.utils.Settings;
-import org.owasp.dependencycheck.data.nvdcve.DatabaseException;
-import org.owasp.dependencycheck.utils.InvalidSettingException;
-import static org.owasp.dependencycheck.data.update.DataStoreMetaInfo.BATCH;
-import static org.owasp.dependencycheck.data.update.DataStoreMetaInfo.MODIFIED;
/**
* Class responsible for updating the CPE and NVDCVE data stores.
@@ -63,37 +42,7 @@ public class DatabaseUpdater implements CachedWebDataSource {
/**
* Utility to read and write meta-data about the data.
*/
- private DataStoreMetaInfo properties = null;
- /**
- * Reference to the Cve Database.
- */
- private CveDB cveDB = null;
- /**
- * Reference to the Cpe Index.
- */
- private CpeIndexWriter cpeIndex = null;
- /**
- * A flag indicating whether or not the batch update should be performed.
- */
- private boolean doBatchUpdate;
-
- /**
- * Get the value of doBatchUpdate
- *
- * @return the value of doBatchUpdate
- */
- protected boolean isDoBatchUpdate() {
- return doBatchUpdate;
- }
-
- /**
- * Set the value of doBatchUpdate
- *
- * @param doBatchUpdate new value of doBatchUpdate
- */
- protected void setDoBatchUpdate(boolean doBatchUpdate) {
- this.doBatchUpdate = doBatchUpdate;
- }
+ protected DataStoreMetaInfo properties = null;
/**
* Downloads the latest NVD CVE XML file from the web and imports it into
@@ -104,149 +53,57 @@ public class DatabaseUpdater implements CachedWebDataSource {
*/
@Override
public void update() throws UpdateException {
- doBatchUpdate = false;
properties = new DataStoreMetaInfo();
+ AbstractUpdate store = null;
+ File dataDir = Settings.getFile(Settings.KEYS.DATA_DIRECTORY);
+ DirectorySpinLock lock = null;
try {
- final Map update = updateNeeded();
- int maxUpdates = 0;
- for (NvdCveInfo cve : update.values()) {
- if (cve.getNeedsUpdate()) {
- maxUpdates += 1;
- }
- }
- if (maxUpdates > 3 && !properties.isBatchUpdateMode()) {
- Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.INFO,
- "NVD CVE requires several updates; this could take a couple of minutes.");
- }
- if (maxUpdates > 0 && !isDoBatchUpdate()) {
- openDataStores();
- }
+ lock = new DirectorySpinLock(dataDir);
+ } catch (InvalidDirectoryException ex) {
+ throw new UpdateException("Unable to obtain lock on the data directory", ex);
+ } catch (DirectoryLockException ex) {
+ throw new UpdateException("Unable to obtain exclusive lock on the data directory", ex);
+ }
- if (properties.isBatchUpdateMode() && isDoBatchUpdate()) {
- try {
- performBatchUpdate();
- openDataStores();
- } catch (IOException ex) {
- throw new UpdateException("Unable to perform batch update", ex);
- }
+ try {
+ lock.obtainSharedLock();
+ if (properties.isBatchUpdateMode()) {
+ store = new BatchUpdate();
+ } else {
+ store = new StandardUpdate();
}
-
- int count = 0;
- for (NvdCveInfo cve : update.values()) {
- if (cve.getNeedsUpdate()) {
- count += 1;
- Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.INFO,
- "Updating NVD CVE ({0} of {1})", new Object[]{count, maxUpdates});
- URL url = new URL(cve.getUrl());
- File outputPath = null;
- File outputPath12 = null;
+ if (store.isUpdateNeeded()) {
+ lock.release();
+ lock.obtainExclusiveLock();
+ if (store.shouldDeleteAndRecreate()) {
try {
- Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.INFO,
- "Downloading {0}", cve.getUrl());
- outputPath = File.createTempFile("cve" + cve.getId() + "_", ".xml");
- Downloader.fetchFile(url, outputPath);
-
- url = new URL(cve.getOldSchemaVersionUrl());
- outputPath12 = File.createTempFile("cve_1_2_" + cve.getId() + "_", ".xml");
- Downloader.fetchFile(url, outputPath12);
-
- Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.INFO,
- "Processing {0}", cve.getUrl());
-
- importXML(outputPath, outputPath12);
-
- cveDB.commit();
- cpeIndex.commit();
-
- properties.save(cve);
-
- Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.INFO,
- "Completed update {0} of {1}", new Object[]{count, maxUpdates});
- } catch (FileNotFoundException ex) {
- throw new UpdateException(ex);
- } catch (ParserConfigurationException ex) {
- throw new UpdateException(ex);
- } catch (SAXException ex) {
- throw new UpdateException(ex);
+ deleteExistingData();
} catch (IOException ex) {
- throw new UpdateException(ex);
- } catch (SQLException ex) {
- throw new UpdateException(ex);
- } catch (DatabaseException ex) {
- throw new UpdateException(ex);
- } catch (ClassNotFoundException ex) {
- throw new UpdateException(ex);
- } finally {
- boolean deleted = false;
- try {
- if (outputPath != null && outputPath.exists()) {
- deleted = outputPath.delete();
- }
- } finally {
- if (outputPath != null && (outputPath.exists() || !deleted)) {
- outputPath.deleteOnExit();
- }
- }
- try {
- deleted = false;
- if (outputPath12 != null && outputPath12.exists()) {
- deleted = outputPath12.delete();
- }
- } finally {
- if (outputPath12 != null && (outputPath12.exists() || !deleted)) {
- outputPath12.deleteOnExit();
- }
- }
+ Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.WARNING, "Unable to delete the existing data directory");
+ Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.FINE, null, ex);
}
}
+ store.update();
}
- if (maxUpdates >= 1) { //ensure the modified file date gets written
- properties.save(update.get(MODIFIED));
- cveDB.cleanupDatabase();
- }
- if (update.get(BATCH) != null) {
- properties.save(update.get(BATCH));
- }
+ } catch (DirectoryLockException ex) {
+ Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.WARNING,
+ "Unable to obtain lock on data directory, unable to update the data to use the most current data.");
+ Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.FINE, null, ex);
} catch (MalformedURLException ex) {
- throw new UpdateException(ex);
+ Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.WARNING,
+ "NVD CVE properties files contain an invalid URL, unable to update the data to use the most current data.");
+ Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.FINE, null, ex);
} catch (DownloadFailedException ex) {
- throw new UpdateException(ex);
+ Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.WARNING,
+ "Unable to download the NVD CVE data, unable to update the data to use the most current data.");
+ Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.FINE, null, ex);
} finally {
- closeDataStores();
+ if (lock != null) {
+ lock.release();
+ }
}
}
- /**
- * Imports the NVD CVE XML File into the Lucene Index.
- *
- * @param file the file containing the NVD CVE XML
- * @param oldVersion contains the file containing the NVD CVE XML 1.2
- * @throws ParserConfigurationException is thrown if there is a parser
- * configuration exception
- * @throws SAXException is thrown if there is a SAXException
- * @throws IOException is thrown if there is a IO Exception
- * @throws SQLException is thrown if there is a SQL exception
- * @throws DatabaseException is thrown if there is a database exception
- * @throws ClassNotFoundException thrown if the h2 database driver cannot be
- * loaded
- */
- private void importXML(File file, File oldVersion)
- throws ParserConfigurationException, SAXException, IOException, SQLException, DatabaseException, ClassNotFoundException {
-
- final SAXParserFactory factory = SAXParserFactory.newInstance();
- final SAXParser saxParser = factory.newSAXParser();
-
- final NvdCve12Handler cve12Handler = new NvdCve12Handler();
- saxParser.parse(oldVersion, cve12Handler);
- final Map> prevVersionVulnMap = cve12Handler.getVulnerabilities();
-
- final NvdCve20Handler cve20Handler = new NvdCve20Handler();
- cve20Handler.setCveDB(cveDB);
- cve20Handler.setPrevVersionVulnMap(prevVersionVulnMap);
- cve20Handler.setCpeIndex(cpeIndex);
- saxParser.parse(file, cve20Handler);
- }
-
/**
* Deletes the existing data directories.
*
@@ -266,307 +123,4 @@ public class DatabaseUpdater implements CachedWebDataSource {
FileUtils.delete(data);
}
}
-
- /**
- * Performs the batch update based on the configured batch update URL.
- *
- * @throws UpdateException thrown if there is an exception during the update
- * process
- */
- private void performBatchUpdate() throws UpdateException {
- if (properties.isBatchUpdateMode() && doBatchUpdate) {
- final String batchSrc = Settings.getString(Settings.KEYS.BATCH_UPDATE_URL);
- File tmp = null;
- try {
- deleteExistingData();
- final File dataDirectory = CveDB.getDataDirectory().getParentFile();
- final URL batchUrl = new URL(batchSrc);
- if ("file".equals(batchUrl.getProtocol())) {
- try {
- tmp = new File(batchUrl.toURI());
- } catch (URISyntaxException ex) {
- final String msg = String.format("Invalid batch update URI: %s", batchSrc);
- throw new UpdateException(msg, ex);
- }
- } else if ("http".equals(batchUrl.getProtocol())
- || "https".equals(batchUrl.getProtocol())) {
- tmp = File.createTempFile("batch_", ".zip");
- Downloader.fetchFile(batchUrl, tmp);
- }
- //TODO add FTP?
- FileUtils.extractFiles(tmp, dataDirectory);
-
- } catch (IOException ex) {
- final String msg = String.format("IO Exception Occured performing batch update using: %s", batchSrc);
- throw new UpdateException(msg, ex);
- } finally {
- if (tmp != null && !tmp.delete()) {
- tmp.deleteOnExit();
- }
- }
- }
- }
-
- /**
- * Closes the CVE and CPE data stores.
- */
- private void closeDataStores() {
- if (cveDB != null) {
- try {
- cveDB.close();
- } catch (Exception ignore) {
- Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.FINEST, "Error closing the cveDB", ignore);
- }
- }
- if (cpeIndex != null) {
- try {
- cpeIndex.close();
- } catch (Exception ignore) {
- Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.FINEST, "Error closing the cpeIndex", ignore);
- }
- }
- }
-
- /**
- * Opens the CVE and CPE data stores.
- *
- * @throws UpdateException thrown if a data store cannot be opened
- */
- private void openDataStores() throws UpdateException {
- //open the cve and cpe data stores
- try {
- cveDB = new CveDB();
- cveDB.open();
- cpeIndex = new CpeIndexWriter();
- cpeIndex.open();
- } catch (IOException ex) {
- closeDataStores();
- Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.FINE, "IO Error opening databases", ex);
- throw new UpdateException("Error updating the CPE/CVE data, please see the log file for more details.");
- } catch (SQLException ex) {
- closeDataStores();
- Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.FINE, "SQL Exception opening databases", ex);
- throw new UpdateException("Error updating the CPE/CVE data, please see the log file for more details.");
- } catch (DatabaseException ex) {
- closeDataStores();
- Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.FINE, "Database Exception opening databases", ex);
- throw new UpdateException("Error updating the CPE/CVE data, please see the log file for more details.");
- } catch (ClassNotFoundException ex) {
- closeDataStores();
- Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.FINE, "Class not found exception opening databases", ex);
- throw new UpdateException("Error updating the CPE/CVE data, please see the log file for more details.");
- }
- }
-
- /**
- * Determines if the index needs to be updated. This is done by fetching the
- * NVD CVE meta data and checking the last update date. If the data needs to
- * be refreshed this method will return the NvdCveUrl for the files that
- * need to be updated.
- *
- * @return the NvdCveUrl of the files that need to be updated.
- * @throws MalformedURLException is thrown if the URL for the NVD CVE Meta
- * data is incorrect.
- * @throws DownloadFailedException is thrown if there is an error.
- * downloading the NVD CVE download data file.
- * @throws UpdateException Is thrown if there is an issue with the last
- * updated properties file.
- */
- private Map updateNeeded() throws MalformedURLException, DownloadFailedException, UpdateException {
-
- Map currentlyPublished;
- try {
- currentlyPublished = retrieveCurrentTimestampsFromWeb();
- } catch (InvalidDataException ex) {
- final String msg = "Unable to retrieve valid timestamp from nvd cve downloads page";
- Logger.getLogger(DataStoreMetaInfo.class.getName()).log(Level.FINE, msg, ex);
- throw new DownloadFailedException(msg, ex);
- } catch (InvalidSettingException ex) {
- Logger.getLogger(DataStoreMetaInfo.class.getName()).log(Level.FINE, "Invalid setting found when retrieving timestamps", ex);
- throw new DownloadFailedException("Invalid settings", ex);
- }
-
- if (currentlyPublished == null) {
- throw new DownloadFailedException("Unable to retrieve the timestamps of the currently published NVD CVE data");
- }
-
-// final File cpeDataDirectory;
-// try {
-// cpeDataDirectory = CveDB.getDataDirectory();
-// } catch (IOException ex) {
-// String msg;
-// try {
-// msg = String.format("Unable to create the CVE Data Directory '%s'",
-// Settings.getFile(Settings.KEYS.CVE_DATA_DIRECTORY).getCanonicalPath());
-// } catch (IOException ex1) {
-// msg = String.format("Unable to create the CVE Data Directory, this is likely a configuration issue: '%s%s%s'",
-// Settings.getString(Settings.KEYS.DATA_DIRECTORY, ""),
-// File.separator,
-// Settings.getString(Settings.KEYS.CVE_DATA_DIRECTORY, ""));
-// }
-// throw new UpdateException(msg, ex);
-// }
-
- if (!properties.isEmpty()) {
- try {
- boolean deleteAndRecreate = false;
- float version;
-
- if (properties.getProperty("version") == null) {
- deleteAndRecreate = true;
- } else {
- try {
- version = Float.parseFloat(properties.getProperty("version"));
- final float currentVersion = Float.parseFloat(CveDB.DB_SCHEMA_VERSION);
- if (currentVersion > version) {
- deleteAndRecreate = true;
- }
- } catch (NumberFormatException ex) {
- deleteAndRecreate = true;
- }
- }
-
- final NvdCveInfo batchInfo = currentlyPublished.get(BATCH);
- if (properties.isBatchUpdateMode() && batchInfo != null) {
- final long lastUpdated = Long.parseLong(properties.getProperty(DataStoreMetaInfo.BATCH, "0"));
- if (lastUpdated != batchInfo.getTimestamp()) {
- deleteAndRecreate = true;
- }
- }
-
- if (deleteAndRecreate) {
- setDoBatchUpdate(properties.isBatchUpdateMode());
- try {
- deleteExistingData();
- } catch (IOException ex) {
- final String msg = "Unable to delete existing data";
- Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.WARNING, msg);
- Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.FINE, null, ex);
- }
- return currentlyPublished;
- }
-
- final long lastUpdated = Long.parseLong(properties.getProperty(DataStoreMetaInfo.LAST_UPDATED, "0"));
- final Date now = new Date();
- final int days = Settings.getInt(Settings.KEYS.CVE_MODIFIED_VALID_FOR_DAYS, 7);
- final int start = Settings.getInt(Settings.KEYS.CVE_START_YEAR, 2002);
- final int end = Calendar.getInstance().get(Calendar.YEAR);
- if (lastUpdated == currentlyPublished.get(MODIFIED).getTimestamp()) {
- currentlyPublished.clear(); //we don't need to update anything.
- setDoBatchUpdate(properties.isBatchUpdateMode());
- } else if (withinRange(lastUpdated, now.getTime(), days)) {
- currentlyPublished.get(MODIFIED).setNeedsUpdate(true);
- if (properties.isBatchUpdateMode()) {
- setDoBatchUpdate(false);
- } else {
- for (int i = start; i <= end; i++) {
- currentlyPublished.get(String.valueOf(i)).setNeedsUpdate(false);
- }
- }
- } else if (properties.isBatchUpdateMode()) {
- currentlyPublished.get(MODIFIED).setNeedsUpdate(true);
- setDoBatchUpdate(true);
- } else { //we figure out which of the several XML files need to be downloaded.
- currentlyPublished.get(MODIFIED).setNeedsUpdate(false);
- for (int i = start; i <= end; i++) {
- final NvdCveInfo cve = currentlyPublished.get(String.valueOf(i));
- long currentTimestamp = 0;
- try {
- currentTimestamp = Long.parseLong(properties.getProperty(DataStoreMetaInfo.LAST_UPDATED_BASE + String.valueOf(i), "0"));
- } catch (NumberFormatException ex) {
- final String msg = String.format("Error parsing '%s' '%s' from nvdcve.lastupdated",
- DataStoreMetaInfo.LAST_UPDATED_BASE, String.valueOf(i));
- Logger.getLogger(DataStoreMetaInfo.class.getName()).log(Level.FINE, msg, ex);
- }
- if (currentTimestamp == cve.getTimestamp()) {
- cve.setNeedsUpdate(false); //they default to true.
- }
- }
- }
- } catch (NumberFormatException ex) {
- final String msg = "An invalid schema version or timestamp exists in the data.properties file.";
- Logger.getLogger(DataStoreMetaInfo.class.getName()).log(Level.WARNING, msg);
- Logger.getLogger(DataStoreMetaInfo.class.getName()).log(Level.FINE, null, ex);
- setDoBatchUpdate(properties.isBatchUpdateMode());
- }
- } else {
- setDoBatchUpdate(properties.isBatchUpdateMode());
- }
- return currentlyPublished;
- }
-
- /**
- * Determines if the epoch date is within the range specified of the
- * compareTo epoch time. This takes the (compareTo-date)/1000/60/60/24 to
- * get the number of days. If the calculated days is less then the range the
- * date is considered valid.
- *
- * @param date the date to be checked.
- * @param compareTo the date to compare to.
- * @param range the range in days to be considered valid.
- * @return whether or not the date is within the range.
- */
- private boolean withinRange(long date, long compareTo, int range) {
- final double differenceInDays = (compareTo - date) / 1000.0 / 60.0 / 60.0 / 24.0;
- return differenceInDays < range;
- }
-
- /**
- * Retrieves the timestamps from the NVD CVE meta data file.
- *
- * @return the timestamp from the currently published nvdcve downloads page
- * @throws MalformedURLException thrown if the URL for the NVD CCE Meta data
- * is incorrect.
- * @throws DownloadFailedException thrown if there is an error downloading
- * the nvd cve meta data file
- * @throws InvalidDataException thrown if there is an exception parsing the
- * timestamps
- * @throws InvalidSettingException thrown if the settings are invalid
- */
- private Map retrieveCurrentTimestampsFromWeb()
- throws MalformedURLException, DownloadFailedException, InvalidDataException, InvalidSettingException {
-
- final Map map = new TreeMap();
- String retrieveUrl = Settings.getString(Settings.KEYS.CVE_MODIFIED_20_URL);
- if (retrieveUrl == null && properties.isBatchUpdateMode()) {
- final NvdCveInfo item = new NvdCveInfo();
- retrieveUrl = Settings.getString(Settings.KEYS.BATCH_UPDATE_URL);
- if (retrieveUrl == null) {
- final String msg = "Invalid configuration - neither the modified or batch update URLs are specified in the configuration.";
- Logger.getLogger(DataStoreMetaInfo.class.getName()).log(Level.SEVERE, msg);
- throw new InvalidSettingException(msg);
- }
- item.setTimestamp(Downloader.getLastModified(new URL(retrieveUrl)));
- item.setId(BATCH);
- item.setNeedsUpdate(false);
- map.put(BATCH, item);
- } else {
- NvdCveInfo item = new NvdCveInfo();
- item.setNeedsUpdate(false); //the others default to true, to make life easier later this should default to false.
- item.setId(MODIFIED);
- item.setUrl(retrieveUrl);
- item.setOldSchemaVersionUrl(Settings.getString(Settings.KEYS.CVE_MODIFIED_12_URL));
-
- item.setTimestamp(Downloader.getLastModified(new URL(retrieveUrl)));
- map.put(MODIFIED, item);
-
- //only add these urls if we are not in batch mode
- if (!properties.isBatchUpdateMode()) {
- final int start = Settings.getInt(Settings.KEYS.CVE_START_YEAR);
- final int end = Calendar.getInstance().get(Calendar.YEAR);
- final String baseUrl20 = Settings.getString(Settings.KEYS.CVE_SCHEMA_2_0);
- final String baseUrl12 = Settings.getString(Settings.KEYS.CVE_SCHEMA_1_2);
- for (int i = start; i <= end; i++) {
- retrieveUrl = String.format(baseUrl20, i);
- item = new NvdCveInfo();
- item.setId(Integer.toString(i));
- item.setUrl(retrieveUrl);
- item.setOldSchemaVersionUrl(String.format(baseUrl12, i));
- item.setTimestamp(Downloader.getLastModified(new URL(retrieveUrl)));
- map.put(item.getId(), item);
- }
- }
- }
- return map;
- }
}
diff --git a/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/StandardUpdate.java b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/StandardUpdate.java
new file mode 100644
index 000000000..6cefcaa0f
--- /dev/null
+++ b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/StandardUpdate.java
@@ -0,0 +1,292 @@
+/*
+ * This file is part of dependency-check-core.
+ *
+ * Dependency-check-core is free software: you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation, either version 3 of the License, or (at your option) any
+ * later version.
+ *
+ * Dependency-check-core is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+ * details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * dependency-check-core. If not, see http://www.gnu.org/licenses/.
+ *
+ * Copyright (c) 2012 Jeremy Long. All Rights Reserved.
+ */
+package org.owasp.dependencycheck.data.update;
+
+import org.owasp.dependencycheck.data.nvdcve.InvalidDataException;
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import javax.xml.parsers.ParserConfigurationException;
+import org.xml.sax.SAXException;
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.sql.SQLException;
+import java.util.Calendar;
+import java.util.Date;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+import org.owasp.dependencycheck.data.UpdateException;
+import org.owasp.dependencycheck.data.nvdcve.CveDB;
+import org.owasp.dependencycheck.utils.DownloadFailedException;
+import org.owasp.dependencycheck.utils.Downloader;
+import org.owasp.dependencycheck.utils.Settings;
+import org.owasp.dependencycheck.data.nvdcve.DatabaseException;
+import org.owasp.dependencycheck.utils.InvalidSettingException;
+import static org.owasp.dependencycheck.data.update.DataStoreMetaInfo.MODIFIED;
+
+/**
+ * Class responsible for updating the CPE and NVDCVE data stores.
+ *
+ * @author Jeremy Long (jeremy.long@owasp.org)
+ */
+public class StandardUpdate extends AbstractUpdate {
+
+ public StandardUpdate() throws MalformedURLException, DownloadFailedException, UpdateException {
+ super();
+ }
+
+ /**
+ * Downloads the latest NVD CVE XML file from the web and imports it into
+ * the current CVE Database.
+ *
+ * @param updatesNeeded a collection of NvdCveInfo containing information
+ * about needed updates.
+ * @throws UpdateException is thrown if there is an error updating the
+ * database
+ */
+ @Override
+ public void update() throws UpdateException {
+ try {
+ properties = new DataStoreMetaInfo();
+ int maxUpdates = 0;
+ for (NvdCveInfo cve : updatesNeeded) {
+ if (cve.getNeedsUpdate()) {
+ maxUpdates += 1;
+ }
+ }
+ if (maxUpdates > 3) {
+ Logger.getLogger(StandardUpdate.class.getName()).log(Level.INFO,
+ "NVD CVE requires several updates; this could take a couple of minutes.");
+ }
+ if (maxUpdates > 0) {
+ openDataStores();
+ }
+
+ int count = 0;
+ for (NvdCveInfo cve : updatesNeeded) {
+ if (cve.getNeedsUpdate()) {
+ count += 1;
+ Logger.getLogger(StandardUpdate.class.getName()).log(Level.INFO,
+ "Updating NVD CVE ({0} of {1})", new Object[]{count, maxUpdates});
+ URL url = new URL(cve.getUrl());
+ File outputPath = null;
+ File outputPath12 = null;
+ try {
+ Logger.getLogger(StandardUpdate.class.getName()).log(Level.INFO,
+ "Downloading {0}", cve.getUrl());
+ outputPath = File.createTempFile("cve" + cve.getId() + "_", ".xml");
+ Downloader.fetchFile(url, outputPath);
+
+ url = new URL(cve.getOldSchemaVersionUrl());
+ outputPath12 = File.createTempFile("cve_1_2_" + cve.getId() + "_", ".xml");
+ Downloader.fetchFile(url, outputPath12);
+
+ Logger.getLogger(StandardUpdate.class.getName()).log(Level.INFO,
+ "Processing {0}", cve.getUrl());
+
+ importXML(outputPath, outputPath12);
+
+ cveDB.commit();
+ cpeIndex.commit();
+
+ properties.save(cve);
+
+ Logger.getLogger(StandardUpdate.class.getName()).log(Level.INFO,
+ "Completed update {0} of {1}", new Object[]{count, maxUpdates});
+ } catch (FileNotFoundException ex) {
+ throw new UpdateException(ex);
+ } catch (ParserConfigurationException ex) {
+ throw new UpdateException(ex);
+ } catch (SAXException ex) {
+ throw new UpdateException(ex);
+ } catch (IOException ex) {
+ throw new UpdateException(ex);
+ } catch (SQLException ex) {
+ throw new UpdateException(ex);
+ } catch (DatabaseException ex) {
+ throw new UpdateException(ex);
+ } catch (ClassNotFoundException ex) {
+ throw new UpdateException(ex);
+ } finally {
+ boolean deleted = false;
+ try {
+ if (outputPath != null && outputPath.exists()) {
+ deleted = outputPath.delete();
+ }
+ } finally {
+ if (outputPath != null && (outputPath.exists() || !deleted)) {
+ outputPath.deleteOnExit();
+ }
+ }
+ try {
+ deleted = false;
+ if (outputPath12 != null && outputPath12.exists()) {
+ deleted = outputPath12.delete();
+ }
+ } finally {
+ if (outputPath12 != null && (outputPath12.exists() || !deleted)) {
+ outputPath12.deleteOnExit();
+ }
+ }
+ }
+ }
+ }
+ if (maxUpdates >= 1) { //ensure the modified file date gets written
+ properties.save(updatesNeeded.get(MODIFIED));
+ cveDB.cleanupDatabase();
+ }
+ } catch (MalformedURLException ex) {
+ throw new UpdateException(ex);
+ } finally {
+ closeDataStores();
+ }
+ }
+
+ /**
+ * Determines if the index needs to be updated. This is done by fetching the
+ * NVD CVE meta data and checking the last update date. If the data needs to
+ * be refreshed this method will return the NvdCveUrl for the files that
+ * need to be updated.
+ *
+ * @return the collection of files that need to be updated
+ * @throws MalformedURLException is thrown if the URL for the NVD CVE Meta
+ * data is incorrect
+ * @throws DownloadFailedException is thrown if there is an error.
+ * downloading the NVD CVE download data file
+ * @throws UpdateException Is thrown if there is an issue with the last
+ * updated properties file
+ */
+ @Override
+ protected Updateable updatesNeeded() throws MalformedURLException, DownloadFailedException, UpdateException {
+ Updateable updates = null;
+ try {
+ updates = retrieveCurrentTimestampsFromWeb();
+ } catch (InvalidDataException ex) {
+ final String msg = "Unable to retrieve valid timestamp from nvd cve downloads page";
+ Logger.getLogger(StandardUpdate.class.getName()).log(Level.FINE, msg, ex);
+ throw new DownloadFailedException(msg, ex);
+ } catch (InvalidSettingException ex) {
+ Logger.getLogger(StandardUpdate.class.getName()).log(Level.FINE, "Invalid setting found when retrieving timestamps", ex);
+ throw new DownloadFailedException("Invalid settings", ex);
+ }
+
+ if (updates == null) {
+ throw new DownloadFailedException("Unable to retrieve the timestamps of the currently published NVD CVE data");
+ }
+
+ if (!properties.isEmpty()) {
+ try {
+ float version;
+
+ if (properties.getProperty("version") == null) {
+ setDeleteAndRecreate(true);
+ } else {
+ try {
+ version = Float.parseFloat(properties.getProperty("version"));
+ final float currentVersion = Float.parseFloat(CveDB.DB_SCHEMA_VERSION);
+ if (currentVersion > version) {
+ setDeleteAndRecreate(true);
+ }
+ } catch (NumberFormatException ex) {
+ setDeleteAndRecreate(true);
+ }
+ }
+
+ if (shouldDeleteAndRecreate()) {
+ return updates;
+ }
+
+ final long lastUpdated = Long.parseLong(properties.getProperty(DataStoreMetaInfo.LAST_UPDATED, "0"));
+ final Date now = new Date();
+ final int days = Settings.getInt(Settings.KEYS.CVE_MODIFIED_VALID_FOR_DAYS, 7);
+ final int start = Settings.getInt(Settings.KEYS.CVE_START_YEAR, 2002);
+ final int end = Calendar.getInstance().get(Calendar.YEAR);
+ if (lastUpdated == updates.getTimeStamp(MODIFIED)) {
+ updates.clear(); //we don't need to update anything.
+ } else if (withinRange(lastUpdated, now.getTime(), days)) {
+ for (NvdCveInfo entry : updates) {
+ if (MODIFIED.equals(entry.getId())) {
+ entry.setNeedsUpdate(true);
+ } else {
+ entry.setNeedsUpdate(false);
+ }
+ }
+ } else { //we figure out which of the several XML files need to be downloaded.
+ for (NvdCveInfo entry : updates) {
+ if (MODIFIED.equals(entry.getId())) {
+ entry.setNeedsUpdate(true);
+ } else {
+ long currentTimestamp = 0;
+ try {
+ currentTimestamp = Long.parseLong(properties.getProperty(DataStoreMetaInfo.LAST_UPDATED_BASE + entry.getId(), "0"));
+ } catch (NumberFormatException ex) {
+ final String msg = String.format("Error parsing '%s' '%s' from nvdcve.lastupdated",
+ DataStoreMetaInfo.LAST_UPDATED_BASE, entry.getId());
+ Logger.getLogger(StandardUpdate.class.getName()).log(Level.FINE, msg, ex);
+ }
+ if (currentTimestamp == entry.getTimestamp()) {
+ entry.setNeedsUpdate(false);
+ }
+ }
+ }
+ }
+ } catch (NumberFormatException ex) {
+ final String msg = "An invalid schema version or timestamp exists in the data.properties file.";
+ Logger.getLogger(StandardUpdate.class.getName()).log(Level.WARNING, msg);
+ Logger.getLogger(StandardUpdate.class.getName()).log(Level.FINE, null, ex);
+ }
+ }
+ return updates;
+ }
+
+ /**
+ * Retrieves the timestamps from the NVD CVE meta data file.
+ *
+ * @return the timestamp from the currently published nvdcve downloads page
+ * @throws MalformedURLException thrown if the URL for the NVD CCE Meta data
+ * is incorrect.
+ * @throws DownloadFailedException thrown if there is an error downloading
+ * the nvd cve meta data file
+ * @throws InvalidDataException thrown if there is an exception parsing the
+ * timestamps
+ * @throws InvalidSettingException thrown if the settings are invalid
+ */
+ private Updateable retrieveCurrentTimestampsFromWeb()
+ throws MalformedURLException, DownloadFailedException, InvalidDataException, InvalidSettingException {
+
+ Updateable updates = new Updateable();
+ updates.add(MODIFIED, Settings.getString(Settings.KEYS.CVE_MODIFIED_20_URL),
+ Settings.getString(Settings.KEYS.CVE_MODIFIED_12_URL),
+ false);
+
+ //only add these urls if we are not in batch mode
+ if (!properties.isBatchUpdateMode()) {
+ final int start = Settings.getInt(Settings.KEYS.CVE_START_YEAR);
+ final int end = Calendar.getInstance().get(Calendar.YEAR);
+ final String baseUrl20 = Settings.getString(Settings.KEYS.CVE_SCHEMA_2_0);
+ final String baseUrl12 = Settings.getString(Settings.KEYS.CVE_SCHEMA_1_2);
+ for (int i = start; i <= end; i++) {
+ updates.add(Integer.toString(i), String.format(baseUrl20, i),
+ String.format(baseUrl12, i),
+ true);
+ }
+ }
+ return updates;
+ }
+}
diff --git a/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/Updateable.java b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/Updateable.java
new file mode 100644
index 000000000..02bb14c74
--- /dev/null
+++ b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/Updateable.java
@@ -0,0 +1,152 @@
+/*
+ * To change this template, choose Tools | Templates
+ * and open the template in the editor.
+ */
+package org.owasp.dependencycheck.data.update;
+
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.TreeMap;
+import org.owasp.dependencycheck.utils.DownloadFailedException;
+import org.owasp.dependencycheck.utils.Downloader;
+
+/**
+ * Contains a collection of updateable NvdCveInfo objects. This is used to
+ * determine which files need to be downloaded and processed.
+ *
+ * @author Jeremy Long (jeremy.long@owasp.org)
+ */
+public class Updateable implements java.lang.Iterable, Iterator {
+
+ /**
+ * A collection of sources of data.
+ */
+ Map collection = new TreeMap();
+
+ /**
+ * Gets whether or not an update is needed.
+ *
+ * @return true or false depending on whether an update is needed
+ */
+ public boolean isUpdateNeeded() {
+ for (NvdCveInfo item : this) {
+ if (item.getNeedsUpdate()) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ /**
+ * Adds a new entry of updateable information to the contained collection.
+ *
+ * @param id the key for the item to be added
+ * @param url the URL to download the item
+ * @param oldUrl the URL for the old version of the item (the NVD CVE old
+ * schema still contains useful data we need).
+ * @throws MalformedURLException thrown if the URL provided is invalid
+ * @throws DownloadFailedException thrown if the download fails.
+ */
+ public void add(String id, String url, String oldUrl) throws MalformedURLException, DownloadFailedException {
+ add(id, url, oldUrl, false);
+ }
+
+ /**
+ * Adds a new entry of updateable information to the contained collection.
+ *
+ * @param id the key for the item to be added
+ * @param url the URL to download the item
+ * @param oldUrl the URL for the old version of the item (the NVD CVE old
+ * schema still contains useful data we need).
+ * @param needsUpdate whether or not the data needs to be updated
+ * @throws MalformedURLException thrown if the URL provided is invalid
+ * @throws DownloadFailedException thrown if the download fails.
+ */
+ public void add(String id, String url, String oldUrl, boolean needsUpdate) throws MalformedURLException, DownloadFailedException {
+ NvdCveInfo item = new NvdCveInfo();
+ item.setNeedsUpdate(needsUpdate); //the others default to true, to make life easier later this should default to false.
+ item.setId(id);
+ item.setUrl(url);
+ item.setOldSchemaVersionUrl(oldUrl);
+ item.setTimestamp(Downloader.getLastModified(new URL(url)));
+ collection.put(id, item);
+ }
+
+ /**
+ * Clears the contained collection of NvdCveInfo entries.
+ */
+ public void clear() {
+ collection.clear();
+ }
+
+ /**
+ * Returns the timestamp for the given entry.
+ *
+ * @param key the key to lookup in the collection of NvdCveInfo items
+ * @return the timestamp for the given entry
+ */
+ public long getTimeStamp(String key) {
+ return collection.get(key).getTimestamp();
+ }
+ /**
+ * An internal iterator used to implement iterable.
+ */
+ Iterator> iterableContent = null;
+
+ /**
+ * Returns an iterator for the NvdCveInfo contained.
+ * This method is not thread safe.
+ *
+ * @return an NvdCveInfo Iterator
+ */
+ @Override
+ public Iterator iterator() {
+ iterableContent = collection.entrySet().iterator();
+ return this;
+ }
+
+ /**
+ * Returns whether or not there is another item in the collection.
+ * This method is not thread safe.
+ *
+ * @return true or false depending on whether or not another item exists in
+ * the collection
+ */
+ @Override
+ public boolean hasNext() {
+ return iterableContent.hasNext();
+ }
+
+ /**
+ * Returns the next item in the collection.
+ * This method is not thread safe.
+ *
+ * @return the next NvdCveInfo item in the collection
+ */
+ @Override
+ public NvdCveInfo next() {
+ return iterableContent.next().getValue();
+ }
+
+ /**
+ * Removes the current NvdCveInfo object from the collection.
+ * This method is not thread safe.
+ */
+ @Override
+ public void remove() {
+ iterableContent.remove();
+ }
+
+ /**
+ * Returns the specified item from the collection.
+ *
+ * @param key the key to lookup the return value
+ * @return the NvdCveInfo object stored using the specified key
+ */
+ NvdCveInfo get(String key) {
+ return collection.get(key);
+ }
+}