From 085cffa4cfcf6ce889cbab1ddfd59719e03d3bdf Mon Sep 17 00:00:00 2001 From: Jeremy Long Date: Sun, 22 Sep 2013 21:52:31 -0400 Subject: [PATCH] seperated functionality to make the update procedure easier to understand Former-commit-id: 15e86b665c007af38bf58b47097f94f7ec82bb5a --- .../data/update/AbstractUpdate.java | 241 ++++++++ .../data/update/BatchUpdate.java | 266 +++++++++ .../data/update/DatabaseUpdater.java | 526 ++---------------- .../data/update/StandardUpdate.java | 292 ++++++++++ .../data/update/Updateable.java | 152 +++++ 5 files changed, 991 insertions(+), 486 deletions(-) create mode 100644 dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/AbstractUpdate.java create mode 100644 dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/BatchUpdate.java create mode 100644 dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/StandardUpdate.java create mode 100644 dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/Updateable.java diff --git a/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/AbstractUpdate.java b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/AbstractUpdate.java new file mode 100644 index 000000000..fb0e487df --- /dev/null +++ b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/AbstractUpdate.java @@ -0,0 +1,241 @@ +/* + * This file is part of dependency-check-core. + * + * Dependency-check-core is free software: you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation, either version 3 of the License, or (at your option) any + * later version. + * + * Dependency-check-core is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more + * details. + * + * You should have received a copy of the GNU General Public License along with + * dependency-check-core. If not, see http://www.gnu.org/licenses/. + * + * Copyright (c) 2012 Jeremy Long. All Rights Reserved. + */ +package org.owasp.dependencycheck.data.update; + +import java.io.File; +import java.io.IOException; +import java.net.MalformedURLException; +import org.owasp.dependencycheck.data.CachedWebDataSource; +import java.sql.SQLException; +import java.util.List; +import java.util.Map; +import java.util.logging.Level; +import java.util.logging.Logger; +import javax.xml.parsers.ParserConfigurationException; +import javax.xml.parsers.SAXParser; +import javax.xml.parsers.SAXParserFactory; +import org.owasp.dependencycheck.data.UpdateException; +import org.owasp.dependencycheck.data.cpe.CpeIndexWriter; +import org.owasp.dependencycheck.data.nvdcve.CveDB; +import org.owasp.dependencycheck.utils.FileUtils; +import org.owasp.dependencycheck.utils.Settings; +import org.owasp.dependencycheck.data.nvdcve.DatabaseException; +import org.owasp.dependencycheck.data.nvdcve.NvdCve12Handler; +import org.owasp.dependencycheck.data.nvdcve.NvdCve20Handler; +import org.owasp.dependencycheck.dependency.VulnerableSoftware; +import org.owasp.dependencycheck.utils.DownloadFailedException; +import org.xml.sax.SAXException; + +/** + * Class responsible for updating the CPE and NVDCVE data stores. + * + * @author Jeremy Long (jeremy.long@owasp.org) + */ +public abstract class AbstractUpdate { + + private Updateable updateable; + /** + * Utility to read and write meta-data about the data. + */ + protected DataStoreMetaInfo properties = null; + /** + * Reference to the Cve Database. + */ + protected CveDB cveDB = null; + /** + * Reference to the Cpe Index. + */ + protected CpeIndexWriter cpeIndex = null; + + public AbstractUpdate() throws MalformedURLException, DownloadFailedException, UpdateException { + this.updateable = updatesNeeded(); + } + + /** + * Gets whether or not an update is needed. + * + * @return true or false depending on whether an update is needed + */ + public boolean isUpdateNeeded() { + return updateable.isUpdateNeeded(); + } + + /** + * Determines if the index needs to be updated. + * + * @return a collection of updateable resources. + * @throws MalformedURLException is thrown if the URL for the NVD CVE Meta + * data is incorrect. + * @throws DownloadFailedException is thrown if there is an error. + * downloading the NVD CVE download data file. + * @throws UpdateException Is thrown if there is an issue with the last + * updated properties file. + */ + protected abstract Updateable updatesNeeded() throws MalformedURLException, DownloadFailedException, UpdateException; + + /** + *

Updates the data store to the latest version.

+ * + * @throws UpdateException is thrown if there is an error updating the + * database + */ + public abstract void update() throws UpdateException; + /** + * A flag indicating whether or not the current data store should be + * deleted. + */ + private boolean deleteAndRecreate = false; + protected Updateable updatesNeeded = null; + + /** + * Get the value of deleteAndRecreate + * + * @return the value of deleteAndRecreate + */ + public boolean shouldDeleteAndRecreate() { + return deleteAndRecreate; + } + + /** + * Set the value of deleteAndRecreate + * + * @param deleteAndRecreate new value of deleteAndRecreate + */ + public void setDeleteAndRecreate(boolean deleteAndRecreate) { + this.deleteAndRecreate = deleteAndRecreate; + } + + /** + * Deletes the existing data directories. + * + * @throws IOException thrown if the directory cannot be deleted + */ + protected void deleteExistingData() throws IOException { + File data = Settings.getFile(Settings.KEYS.CVE_DATA_DIRECTORY); + if (data.exists()) { + FileUtils.delete(data); + } + data = Settings.getFile(Settings.KEYS.CPE_DATA_DIRECTORY); + if (data.exists()) { + FileUtils.delete(data); + } + data = DataStoreMetaInfo.getPropertiesFile(); + if (data.exists()) { + FileUtils.delete(data); + } + } + + /** + * Closes the CVE and CPE data stores. + */ + protected void closeDataStores() { + if (cveDB != null) { + try { + cveDB.close(); + } catch (Exception ignore) { + Logger.getLogger(AbstractUpdate.class.getName()).log(Level.FINEST, "Error closing the cveDB", ignore); + } + } + if (cpeIndex != null) { + try { + cpeIndex.close(); + } catch (Exception ignore) { + Logger.getLogger(AbstractUpdate.class.getName()).log(Level.FINEST, "Error closing the cpeIndex", ignore); + } + } + } + + /** + * Opens the CVE and CPE data stores. + * + * @throws UpdateException thrown if a data store cannot be opened + */ + protected void openDataStores() throws UpdateException { + //open the cve and cpe data stores + try { + cveDB = new CveDB(); + cveDB.open(); + cpeIndex = new CpeIndexWriter(); + cpeIndex.open(); + } catch (IOException ex) { + closeDataStores(); + Logger.getLogger(AbstractUpdate.class.getName()).log(Level.FINE, "IO Error opening databases", ex); + throw new UpdateException("Error updating the CPE/CVE data, please see the log file for more details."); + } catch (SQLException ex) { + closeDataStores(); + Logger.getLogger(AbstractUpdate.class.getName()).log(Level.FINE, "SQL Exception opening databases", ex); + throw new UpdateException("Error updating the CPE/CVE data, please see the log file for more details."); + } catch (DatabaseException ex) { + closeDataStores(); + Logger.getLogger(AbstractUpdate.class.getName()).log(Level.FINE, "Database Exception opening databases", ex); + throw new UpdateException("Error updating the CPE/CVE data, please see the log file for more details."); + } catch (ClassNotFoundException ex) { + closeDataStores(); + Logger.getLogger(AbstractUpdate.class.getName()).log(Level.FINE, "Class not found exception opening databases", ex); + throw new UpdateException("Error updating the CPE/CVE data, please see the log file for more details."); + } + } + + /** + * Determines if the epoch date is within the range specified of the + * compareTo epoch time. This takes the (compareTo-date)/1000/60/60/24 to + * get the number of days. If the calculated days is less then the range the + * date is considered valid. + * + * @param date the date to be checked. + * @param compareTo the date to compare to. + * @param range the range in days to be considered valid. + * @return whether or not the date is within the range. + */ + protected boolean withinRange(long date, long compareTo, int range) { + final double differenceInDays = (compareTo - date) / 1000.0 / 60.0 / 60.0 / 24.0; + return differenceInDays < range; + } + + /** + * Imports the NVD CVE XML File into the Lucene Index. + * + * @param file the file containing the NVD CVE XML + * @param oldVersion contains the file containing the NVD CVE XML 1.2 + * @throws ParserConfigurationException is thrown if there is a parser + * configuration exception + * @throws SAXException is thrown if there is a SAXException + * @throws IOException is thrown if there is a IO Exception + * @throws SQLException is thrown if there is a SQL exception + * @throws DatabaseException is thrown if there is a database exception + * @throws ClassNotFoundException thrown if the h2 database driver cannot be + * loaded + */ + protected void importXML(File file, File oldVersion) + throws ParserConfigurationException, SAXException, IOException, SQLException, DatabaseException, ClassNotFoundException { + + final SAXParserFactory factory = SAXParserFactory.newInstance(); + final SAXParser saxParser = factory.newSAXParser(); + + final NvdCve12Handler cve12Handler = new NvdCve12Handler(); + saxParser.parse(oldVersion, cve12Handler); + final Map> prevVersionVulnMap = cve12Handler.getVulnerabilities(); + + final NvdCve20Handler cve20Handler = new NvdCve20Handler(); + cve20Handler.setCveDB(cveDB); + cve20Handler.setPrevVersionVulnMap(prevVersionVulnMap); + cve20Handler.setCpeIndex(cpeIndex); + saxParser.parse(file, cve20Handler); + } +} diff --git a/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/BatchUpdate.java b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/BatchUpdate.java new file mode 100644 index 000000000..4095e072c --- /dev/null +++ b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/BatchUpdate.java @@ -0,0 +1,266 @@ +/* + * This file is part of dependency-check-core. + * + * Dependency-check-core is free software: you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation, either version 3 of the License, or (at your option) any + * later version. + * + * Dependency-check-core is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more + * details. + * + * You should have received a copy of the GNU General Public License along with + * dependency-check-core. If not, see http://www.gnu.org/licenses/. + * + * Copyright (c) 2012 Jeremy Long. All Rights Reserved. + */ +package org.owasp.dependencycheck.data.update; + +import org.owasp.dependencycheck.data.nvdcve.InvalidDataException; +import java.io.File; +import java.io.FileNotFoundException; +import java.io.IOException; +import javax.xml.parsers.ParserConfigurationException; +import org.xml.sax.SAXException; +import java.net.MalformedURLException; +import java.net.URISyntaxException; +import java.net.URL; +import java.sql.SQLException; +import java.util.Calendar; +import java.util.Date; +import java.util.logging.Level; +import java.util.logging.Logger; +import org.owasp.dependencycheck.data.UpdateException; +import org.owasp.dependencycheck.data.nvdcve.CveDB; +import org.owasp.dependencycheck.utils.DownloadFailedException; +import org.owasp.dependencycheck.utils.Downloader; +import org.owasp.dependencycheck.utils.FileUtils; +import org.owasp.dependencycheck.utils.Settings; +import org.owasp.dependencycheck.data.nvdcve.DatabaseException; +import org.owasp.dependencycheck.utils.InvalidSettingException; +import static org.owasp.dependencycheck.data.update.DataStoreMetaInfo.BATCH; +import static org.owasp.dependencycheck.data.update.DataStoreMetaInfo.MODIFIED; + +/** + * Class responsible for updating the CPE and NVDCVE data stores. + * + * @author Jeremy Long (jeremy.long@owasp.org) + */ +public class BatchUpdate extends AbstractUpdate { + + public BatchUpdate() throws MalformedURLException, DownloadFailedException, UpdateException { + super(); + } + /** + * A flag indicating whether or not the batch update should be performed. + */ + private boolean doBatchUpdate; + + /** + * Get the value of doBatchUpdate + * + * @return the value of doBatchUpdate + */ + protected boolean isDoBatchUpdate() { + return doBatchUpdate; + } + + /** + * Set the value of doBatchUpdate + * + * @param doBatchUpdate new value of doBatchUpdate + */ + protected void setDoBatchUpdate(boolean doBatchUpdate) { + this.doBatchUpdate = doBatchUpdate; + } + + /** + *

Downloads the latest NVD CVE XML file from the web and imports it into + * the current CVE Database.

+ * + * @throws UpdateException is thrown if there is an error updating the + * database + */ + @Override + public void update() throws UpdateException { + if (properties.isBatchUpdateMode() && doBatchUpdate) { + final String batchSrc = Settings.getString(Settings.KEYS.BATCH_UPDATE_URL); + File tmp = null; + try { + deleteExistingData(); + final File dataDirectory = CveDB.getDataDirectory().getParentFile(); + final URL batchUrl = new URL(batchSrc); + if ("file".equals(batchUrl.getProtocol())) { + try { + tmp = new File(batchUrl.toURI()); + } catch (URISyntaxException ex) { + final String msg = String.format("Invalid batch update URI: %s", batchSrc); + throw new UpdateException(msg, ex); + } + } else if ("http".equals(batchUrl.getProtocol()) + || "https".equals(batchUrl.getProtocol())) { + tmp = File.createTempFile("batch_", ".zip"); + Downloader.fetchFile(batchUrl, tmp); + } + //TODO add FTP? + FileUtils.extractFiles(tmp, dataDirectory); + + } catch (IOException ex) { + final String msg = String.format("IO Exception Occured performing batch update using: %s", batchSrc); + throw new UpdateException(msg, ex); + } finally { + if (tmp != null && !tmp.delete()) { + tmp.deleteOnExit(); + } + } + } + } + + /** + * Determines if the index needs to be updated. This is done by fetching the + * NVD CVE meta data and checking the last update date. If the data needs to + * be refreshed this method will return the NvdCveUrl for the files that + * need to be updated. + * + * @return the collection of files that need to be updated + * @throws MalformedURLException is thrown if the URL for the NVD CVE Meta + * data is incorrect + * @throws DownloadFailedException is thrown if there is an error. + * downloading the NVD CVE download data file + * @throws UpdateException Is thrown if there is an issue with the last + * updated properties file + */ + @Override + public Updateable updatesNeeded() throws MalformedURLException, DownloadFailedException, UpdateException { + Updateable updates = null; + try { + updates = retrieveCurrentTimestampsFromWeb(); + } catch (InvalidDataException ex) { + final String msg = "Unable to retrieve valid timestamp from nvd cve downloads page"; + Logger.getLogger(BatchUpdate.class.getName()).log(Level.FINE, msg, ex); + throw new DownloadFailedException(msg, ex); + } catch (InvalidSettingException ex) { + Logger.getLogger(BatchUpdate.class.getName()).log(Level.FINE, "Invalid setting found when retrieving timestamps", ex); + throw new DownloadFailedException("Invalid settings", ex); + } + + if (updates == null) { + throw new DownloadFailedException("Unable to retrieve the timestamps of the currently published NVD CVE data"); + } + + if (!properties.isEmpty()) { + try { + boolean deleteAndRecreate = false; + float version; + + if (properties.getProperty("version") == null) { + deleteAndRecreate = true; + } else { + try { + version = Float.parseFloat(properties.getProperty("version")); + final float currentVersion = Float.parseFloat(CveDB.DB_SCHEMA_VERSION); + if (currentVersion > version) { + deleteAndRecreate = true; + } + } catch (NumberFormatException ex) { + deleteAndRecreate = true; + } + } + + final NvdCveInfo batchInfo = updates.get(BATCH); + if (properties.isBatchUpdateMode() && batchInfo != null) { + final long lastUpdated = Long.parseLong(properties.getProperty(DataStoreMetaInfo.BATCH, "0")); + if (lastUpdated != batchInfo.getTimestamp()) { + deleteAndRecreate = true; + } + } + + if (deleteAndRecreate) { + setDoBatchUpdate(properties.isBatchUpdateMode()); + try { + deleteExistingData(); + } catch (IOException ex) { + final String msg = "Unable to delete existing data"; + Logger.getLogger(BatchUpdate.class.getName()).log(Level.WARNING, msg); + Logger.getLogger(BatchUpdate.class.getName()).log(Level.FINE, null, ex); + } + return updates; + } + + final long lastUpdated = Long.parseLong(properties.getProperty(DataStoreMetaInfo.LAST_UPDATED, "0")); + final Date now = new Date(); + final int days = Settings.getInt(Settings.KEYS.CVE_MODIFIED_VALID_FOR_DAYS, 7); + final int start = Settings.getInt(Settings.KEYS.CVE_START_YEAR, 2002); + final int end = Calendar.getInstance().get(Calendar.YEAR); + if (lastUpdated == updates.get(MODIFIED).getTimestamp()) { + updates.clear(); //we don't need to update anything. + setDoBatchUpdate(properties.isBatchUpdateMode()); + } else if (withinRange(lastUpdated, now.getTime(), days)) { + updates.get(MODIFIED).setNeedsUpdate(true); + if (properties.isBatchUpdateMode()) { + setDoBatchUpdate(false); + } else { + for (int i = start; i <= end; i++) { + updates.get(String.valueOf(i)).setNeedsUpdate(false); + } + } + } else if (properties.isBatchUpdateMode()) { + updates.get(MODIFIED).setNeedsUpdate(true); + setDoBatchUpdate(true); + } else { //we figure out which of the several XML files need to be downloaded. + updates.get(MODIFIED).setNeedsUpdate(false); + for (int i = start; i <= end; i++) { + final NvdCveInfo cve = updates.get(String.valueOf(i)); + long currentTimestamp = 0; + try { + currentTimestamp = Long.parseLong(properties.getProperty(DataStoreMetaInfo.LAST_UPDATED_BASE + String.valueOf(i), "0")); + } catch (NumberFormatException ex) { + final String msg = String.format("Error parsing '%s' '%s' from nvdcve.lastupdated", + DataStoreMetaInfo.LAST_UPDATED_BASE, String.valueOf(i)); + Logger.getLogger(BatchUpdate.class.getName()).log(Level.FINE, msg, ex); + } + if (currentTimestamp == cve.getTimestamp()) { + cve.setNeedsUpdate(false); //they default to true. + } + } + } + } catch (NumberFormatException ex) { + final String msg = "An invalid schema version or timestamp exists in the data.properties file."; + Logger.getLogger(BatchUpdate.class.getName()).log(Level.WARNING, msg); + Logger.getLogger(BatchUpdate.class.getName()).log(Level.FINE, null, ex); + setDoBatchUpdate(properties.isBatchUpdateMode()); + } + } + return updates; + } + + /** + * Retrieves the timestamps from the NVD CVE meta data file. + * + * @return the timestamp from the currently published nvdcve downloads page + * @throws MalformedURLException thrown if the URL for the NVD CCE Meta data + * is incorrect. + * @throws DownloadFailedException thrown if there is an error downloading + * the nvd cve meta data file + * @throws InvalidDataException thrown if there is an exception parsing the + * timestamps + * @throws InvalidSettingException thrown if the settings are invalid + */ + private Updateable retrieveCurrentTimestampsFromWeb() + throws MalformedURLException, DownloadFailedException, InvalidDataException, InvalidSettingException { + Updateable updates = new Updateable(); + updates.add(BATCH, Settings.getString(Settings.KEYS.BATCH_UPDATE_URL), + null, false); + + String url = Settings.getString(Settings.KEYS.CVE_MODIFIED_20_URL, ""); + if (!url.isEmpty()) { + final NvdCveInfo item = new NvdCveInfo(); + updates.add(MODIFIED, url, + Settings.getString(Settings.KEYS.CVE_MODIFIED_12_URL), + false); + } + return updates; + } +} diff --git a/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/DatabaseUpdater.java b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/DatabaseUpdater.java index f69ae3c63..5227d51c0 100644 --- a/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/DatabaseUpdater.java +++ b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/DatabaseUpdater.java @@ -18,40 +18,19 @@ */ package org.owasp.dependencycheck.data.update; -import org.owasp.dependencycheck.data.nvdcve.NvdCve12Handler; -import org.owasp.dependencycheck.data.nvdcve.NvdCve20Handler; -import org.owasp.dependencycheck.data.nvdcve.InvalidDataException; import java.io.File; -import java.io.FileNotFoundException; import java.io.IOException; -import javax.xml.parsers.ParserConfigurationException; -import org.xml.sax.SAXException; import org.owasp.dependencycheck.data.CachedWebDataSource; import java.net.MalformedURLException; -import java.net.URISyntaxException; -import java.net.URL; -import java.sql.SQLException; -import java.util.Calendar; -import java.util.Date; -import java.util.List; -import java.util.Map; -import java.util.TreeMap; import java.util.logging.Level; import java.util.logging.Logger; -import javax.xml.parsers.SAXParser; -import javax.xml.parsers.SAXParserFactory; +import org.owasp.dependencycheck.concurrency.DirectoryLockException; +import org.owasp.dependencycheck.concurrency.DirectorySpinLock; +import org.owasp.dependencycheck.concurrency.InvalidDirectoryException; import org.owasp.dependencycheck.data.UpdateException; -import org.owasp.dependencycheck.data.cpe.CpeIndexWriter; -import org.owasp.dependencycheck.data.nvdcve.CveDB; -import org.owasp.dependencycheck.dependency.VulnerableSoftware; import org.owasp.dependencycheck.utils.DownloadFailedException; -import org.owasp.dependencycheck.utils.Downloader; import org.owasp.dependencycheck.utils.FileUtils; import org.owasp.dependencycheck.utils.Settings; -import org.owasp.dependencycheck.data.nvdcve.DatabaseException; -import org.owasp.dependencycheck.utils.InvalidSettingException; -import static org.owasp.dependencycheck.data.update.DataStoreMetaInfo.BATCH; -import static org.owasp.dependencycheck.data.update.DataStoreMetaInfo.MODIFIED; /** * Class responsible for updating the CPE and NVDCVE data stores. @@ -63,37 +42,7 @@ public class DatabaseUpdater implements CachedWebDataSource { /** * Utility to read and write meta-data about the data. */ - private DataStoreMetaInfo properties = null; - /** - * Reference to the Cve Database. - */ - private CveDB cveDB = null; - /** - * Reference to the Cpe Index. - */ - private CpeIndexWriter cpeIndex = null; - /** - * A flag indicating whether or not the batch update should be performed. - */ - private boolean doBatchUpdate; - - /** - * Get the value of doBatchUpdate - * - * @return the value of doBatchUpdate - */ - protected boolean isDoBatchUpdate() { - return doBatchUpdate; - } - - /** - * Set the value of doBatchUpdate - * - * @param doBatchUpdate new value of doBatchUpdate - */ - protected void setDoBatchUpdate(boolean doBatchUpdate) { - this.doBatchUpdate = doBatchUpdate; - } + protected DataStoreMetaInfo properties = null; /** *

Downloads the latest NVD CVE XML file from the web and imports it into @@ -104,149 +53,57 @@ public class DatabaseUpdater implements CachedWebDataSource { */ @Override public void update() throws UpdateException { - doBatchUpdate = false; properties = new DataStoreMetaInfo(); + AbstractUpdate store = null; + File dataDir = Settings.getFile(Settings.KEYS.DATA_DIRECTORY); + DirectorySpinLock lock = null; try { - final Map update = updateNeeded(); - int maxUpdates = 0; - for (NvdCveInfo cve : update.values()) { - if (cve.getNeedsUpdate()) { - maxUpdates += 1; - } - } - if (maxUpdates > 3 && !properties.isBatchUpdateMode()) { - Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.INFO, - "NVD CVE requires several updates; this could take a couple of minutes."); - } - if (maxUpdates > 0 && !isDoBatchUpdate()) { - openDataStores(); - } + lock = new DirectorySpinLock(dataDir); + } catch (InvalidDirectoryException ex) { + throw new UpdateException("Unable to obtain lock on the data directory", ex); + } catch (DirectoryLockException ex) { + throw new UpdateException("Unable to obtain exclusive lock on the data directory", ex); + } - if (properties.isBatchUpdateMode() && isDoBatchUpdate()) { - try { - performBatchUpdate(); - openDataStores(); - } catch (IOException ex) { - throw new UpdateException("Unable to perform batch update", ex); - } + try { + lock.obtainSharedLock(); + if (properties.isBatchUpdateMode()) { + store = new BatchUpdate(); + } else { + store = new StandardUpdate(); } - - int count = 0; - for (NvdCveInfo cve : update.values()) { - if (cve.getNeedsUpdate()) { - count += 1; - Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.INFO, - "Updating NVD CVE ({0} of {1})", new Object[]{count, maxUpdates}); - URL url = new URL(cve.getUrl()); - File outputPath = null; - File outputPath12 = null; + if (store.isUpdateNeeded()) { + lock.release(); + lock.obtainExclusiveLock(); + if (store.shouldDeleteAndRecreate()) { try { - Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.INFO, - "Downloading {0}", cve.getUrl()); - outputPath = File.createTempFile("cve" + cve.getId() + "_", ".xml"); - Downloader.fetchFile(url, outputPath); - - url = new URL(cve.getOldSchemaVersionUrl()); - outputPath12 = File.createTempFile("cve_1_2_" + cve.getId() + "_", ".xml"); - Downloader.fetchFile(url, outputPath12); - - Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.INFO, - "Processing {0}", cve.getUrl()); - - importXML(outputPath, outputPath12); - - cveDB.commit(); - cpeIndex.commit(); - - properties.save(cve); - - Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.INFO, - "Completed update {0} of {1}", new Object[]{count, maxUpdates}); - } catch (FileNotFoundException ex) { - throw new UpdateException(ex); - } catch (ParserConfigurationException ex) { - throw new UpdateException(ex); - } catch (SAXException ex) { - throw new UpdateException(ex); + deleteExistingData(); } catch (IOException ex) { - throw new UpdateException(ex); - } catch (SQLException ex) { - throw new UpdateException(ex); - } catch (DatabaseException ex) { - throw new UpdateException(ex); - } catch (ClassNotFoundException ex) { - throw new UpdateException(ex); - } finally { - boolean deleted = false; - try { - if (outputPath != null && outputPath.exists()) { - deleted = outputPath.delete(); - } - } finally { - if (outputPath != null && (outputPath.exists() || !deleted)) { - outputPath.deleteOnExit(); - } - } - try { - deleted = false; - if (outputPath12 != null && outputPath12.exists()) { - deleted = outputPath12.delete(); - } - } finally { - if (outputPath12 != null && (outputPath12.exists() || !deleted)) { - outputPath12.deleteOnExit(); - } - } + Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.WARNING, "Unable to delete the existing data directory"); + Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.FINE, null, ex); } } + store.update(); } - if (maxUpdates >= 1) { //ensure the modified file date gets written - properties.save(update.get(MODIFIED)); - cveDB.cleanupDatabase(); - } - if (update.get(BATCH) != null) { - properties.save(update.get(BATCH)); - } + } catch (DirectoryLockException ex) { + Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.WARNING, + "Unable to obtain lock on data directory, unable to update the data to use the most current data."); + Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.FINE, null, ex); } catch (MalformedURLException ex) { - throw new UpdateException(ex); + Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.WARNING, + "NVD CVE properties files contain an invalid URL, unable to update the data to use the most current data."); + Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.FINE, null, ex); } catch (DownloadFailedException ex) { - throw new UpdateException(ex); + Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.WARNING, + "Unable to download the NVD CVE data, unable to update the data to use the most current data."); + Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.FINE, null, ex); } finally { - closeDataStores(); + if (lock != null) { + lock.release(); + } } } - /** - * Imports the NVD CVE XML File into the Lucene Index. - * - * @param file the file containing the NVD CVE XML - * @param oldVersion contains the file containing the NVD CVE XML 1.2 - * @throws ParserConfigurationException is thrown if there is a parser - * configuration exception - * @throws SAXException is thrown if there is a SAXException - * @throws IOException is thrown if there is a IO Exception - * @throws SQLException is thrown if there is a SQL exception - * @throws DatabaseException is thrown if there is a database exception - * @throws ClassNotFoundException thrown if the h2 database driver cannot be - * loaded - */ - private void importXML(File file, File oldVersion) - throws ParserConfigurationException, SAXException, IOException, SQLException, DatabaseException, ClassNotFoundException { - - final SAXParserFactory factory = SAXParserFactory.newInstance(); - final SAXParser saxParser = factory.newSAXParser(); - - final NvdCve12Handler cve12Handler = new NvdCve12Handler(); - saxParser.parse(oldVersion, cve12Handler); - final Map> prevVersionVulnMap = cve12Handler.getVulnerabilities(); - - final NvdCve20Handler cve20Handler = new NvdCve20Handler(); - cve20Handler.setCveDB(cveDB); - cve20Handler.setPrevVersionVulnMap(prevVersionVulnMap); - cve20Handler.setCpeIndex(cpeIndex); - saxParser.parse(file, cve20Handler); - } - /** * Deletes the existing data directories. * @@ -266,307 +123,4 @@ public class DatabaseUpdater implements CachedWebDataSource { FileUtils.delete(data); } } - - /** - * Performs the batch update based on the configured batch update URL. - * - * @throws UpdateException thrown if there is an exception during the update - * process - */ - private void performBatchUpdate() throws UpdateException { - if (properties.isBatchUpdateMode() && doBatchUpdate) { - final String batchSrc = Settings.getString(Settings.KEYS.BATCH_UPDATE_URL); - File tmp = null; - try { - deleteExistingData(); - final File dataDirectory = CveDB.getDataDirectory().getParentFile(); - final URL batchUrl = new URL(batchSrc); - if ("file".equals(batchUrl.getProtocol())) { - try { - tmp = new File(batchUrl.toURI()); - } catch (URISyntaxException ex) { - final String msg = String.format("Invalid batch update URI: %s", batchSrc); - throw new UpdateException(msg, ex); - } - } else if ("http".equals(batchUrl.getProtocol()) - || "https".equals(batchUrl.getProtocol())) { - tmp = File.createTempFile("batch_", ".zip"); - Downloader.fetchFile(batchUrl, tmp); - } - //TODO add FTP? - FileUtils.extractFiles(tmp, dataDirectory); - - } catch (IOException ex) { - final String msg = String.format("IO Exception Occured performing batch update using: %s", batchSrc); - throw new UpdateException(msg, ex); - } finally { - if (tmp != null && !tmp.delete()) { - tmp.deleteOnExit(); - } - } - } - } - - /** - * Closes the CVE and CPE data stores. - */ - private void closeDataStores() { - if (cveDB != null) { - try { - cveDB.close(); - } catch (Exception ignore) { - Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.FINEST, "Error closing the cveDB", ignore); - } - } - if (cpeIndex != null) { - try { - cpeIndex.close(); - } catch (Exception ignore) { - Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.FINEST, "Error closing the cpeIndex", ignore); - } - } - } - - /** - * Opens the CVE and CPE data stores. - * - * @throws UpdateException thrown if a data store cannot be opened - */ - private void openDataStores() throws UpdateException { - //open the cve and cpe data stores - try { - cveDB = new CveDB(); - cveDB.open(); - cpeIndex = new CpeIndexWriter(); - cpeIndex.open(); - } catch (IOException ex) { - closeDataStores(); - Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.FINE, "IO Error opening databases", ex); - throw new UpdateException("Error updating the CPE/CVE data, please see the log file for more details."); - } catch (SQLException ex) { - closeDataStores(); - Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.FINE, "SQL Exception opening databases", ex); - throw new UpdateException("Error updating the CPE/CVE data, please see the log file for more details."); - } catch (DatabaseException ex) { - closeDataStores(); - Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.FINE, "Database Exception opening databases", ex); - throw new UpdateException("Error updating the CPE/CVE data, please see the log file for more details."); - } catch (ClassNotFoundException ex) { - closeDataStores(); - Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.FINE, "Class not found exception opening databases", ex); - throw new UpdateException("Error updating the CPE/CVE data, please see the log file for more details."); - } - } - - /** - * Determines if the index needs to be updated. This is done by fetching the - * NVD CVE meta data and checking the last update date. If the data needs to - * be refreshed this method will return the NvdCveUrl for the files that - * need to be updated. - * - * @return the NvdCveUrl of the files that need to be updated. - * @throws MalformedURLException is thrown if the URL for the NVD CVE Meta - * data is incorrect. - * @throws DownloadFailedException is thrown if there is an error. - * downloading the NVD CVE download data file. - * @throws UpdateException Is thrown if there is an issue with the last - * updated properties file. - */ - private Map updateNeeded() throws MalformedURLException, DownloadFailedException, UpdateException { - - Map currentlyPublished; - try { - currentlyPublished = retrieveCurrentTimestampsFromWeb(); - } catch (InvalidDataException ex) { - final String msg = "Unable to retrieve valid timestamp from nvd cve downloads page"; - Logger.getLogger(DataStoreMetaInfo.class.getName()).log(Level.FINE, msg, ex); - throw new DownloadFailedException(msg, ex); - } catch (InvalidSettingException ex) { - Logger.getLogger(DataStoreMetaInfo.class.getName()).log(Level.FINE, "Invalid setting found when retrieving timestamps", ex); - throw new DownloadFailedException("Invalid settings", ex); - } - - if (currentlyPublished == null) { - throw new DownloadFailedException("Unable to retrieve the timestamps of the currently published NVD CVE data"); - } - -// final File cpeDataDirectory; -// try { -// cpeDataDirectory = CveDB.getDataDirectory(); -// } catch (IOException ex) { -// String msg; -// try { -// msg = String.format("Unable to create the CVE Data Directory '%s'", -// Settings.getFile(Settings.KEYS.CVE_DATA_DIRECTORY).getCanonicalPath()); -// } catch (IOException ex1) { -// msg = String.format("Unable to create the CVE Data Directory, this is likely a configuration issue: '%s%s%s'", -// Settings.getString(Settings.KEYS.DATA_DIRECTORY, ""), -// File.separator, -// Settings.getString(Settings.KEYS.CVE_DATA_DIRECTORY, "")); -// } -// throw new UpdateException(msg, ex); -// } - - if (!properties.isEmpty()) { - try { - boolean deleteAndRecreate = false; - float version; - - if (properties.getProperty("version") == null) { - deleteAndRecreate = true; - } else { - try { - version = Float.parseFloat(properties.getProperty("version")); - final float currentVersion = Float.parseFloat(CveDB.DB_SCHEMA_VERSION); - if (currentVersion > version) { - deleteAndRecreate = true; - } - } catch (NumberFormatException ex) { - deleteAndRecreate = true; - } - } - - final NvdCveInfo batchInfo = currentlyPublished.get(BATCH); - if (properties.isBatchUpdateMode() && batchInfo != null) { - final long lastUpdated = Long.parseLong(properties.getProperty(DataStoreMetaInfo.BATCH, "0")); - if (lastUpdated != batchInfo.getTimestamp()) { - deleteAndRecreate = true; - } - } - - if (deleteAndRecreate) { - setDoBatchUpdate(properties.isBatchUpdateMode()); - try { - deleteExistingData(); - } catch (IOException ex) { - final String msg = "Unable to delete existing data"; - Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.WARNING, msg); - Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.FINE, null, ex); - } - return currentlyPublished; - } - - final long lastUpdated = Long.parseLong(properties.getProperty(DataStoreMetaInfo.LAST_UPDATED, "0")); - final Date now = new Date(); - final int days = Settings.getInt(Settings.KEYS.CVE_MODIFIED_VALID_FOR_DAYS, 7); - final int start = Settings.getInt(Settings.KEYS.CVE_START_YEAR, 2002); - final int end = Calendar.getInstance().get(Calendar.YEAR); - if (lastUpdated == currentlyPublished.get(MODIFIED).getTimestamp()) { - currentlyPublished.clear(); //we don't need to update anything. - setDoBatchUpdate(properties.isBatchUpdateMode()); - } else if (withinRange(lastUpdated, now.getTime(), days)) { - currentlyPublished.get(MODIFIED).setNeedsUpdate(true); - if (properties.isBatchUpdateMode()) { - setDoBatchUpdate(false); - } else { - for (int i = start; i <= end; i++) { - currentlyPublished.get(String.valueOf(i)).setNeedsUpdate(false); - } - } - } else if (properties.isBatchUpdateMode()) { - currentlyPublished.get(MODIFIED).setNeedsUpdate(true); - setDoBatchUpdate(true); - } else { //we figure out which of the several XML files need to be downloaded. - currentlyPublished.get(MODIFIED).setNeedsUpdate(false); - for (int i = start; i <= end; i++) { - final NvdCveInfo cve = currentlyPublished.get(String.valueOf(i)); - long currentTimestamp = 0; - try { - currentTimestamp = Long.parseLong(properties.getProperty(DataStoreMetaInfo.LAST_UPDATED_BASE + String.valueOf(i), "0")); - } catch (NumberFormatException ex) { - final String msg = String.format("Error parsing '%s' '%s' from nvdcve.lastupdated", - DataStoreMetaInfo.LAST_UPDATED_BASE, String.valueOf(i)); - Logger.getLogger(DataStoreMetaInfo.class.getName()).log(Level.FINE, msg, ex); - } - if (currentTimestamp == cve.getTimestamp()) { - cve.setNeedsUpdate(false); //they default to true. - } - } - } - } catch (NumberFormatException ex) { - final String msg = "An invalid schema version or timestamp exists in the data.properties file."; - Logger.getLogger(DataStoreMetaInfo.class.getName()).log(Level.WARNING, msg); - Logger.getLogger(DataStoreMetaInfo.class.getName()).log(Level.FINE, null, ex); - setDoBatchUpdate(properties.isBatchUpdateMode()); - } - } else { - setDoBatchUpdate(properties.isBatchUpdateMode()); - } - return currentlyPublished; - } - - /** - * Determines if the epoch date is within the range specified of the - * compareTo epoch time. This takes the (compareTo-date)/1000/60/60/24 to - * get the number of days. If the calculated days is less then the range the - * date is considered valid. - * - * @param date the date to be checked. - * @param compareTo the date to compare to. - * @param range the range in days to be considered valid. - * @return whether or not the date is within the range. - */ - private boolean withinRange(long date, long compareTo, int range) { - final double differenceInDays = (compareTo - date) / 1000.0 / 60.0 / 60.0 / 24.0; - return differenceInDays < range; - } - - /** - * Retrieves the timestamps from the NVD CVE meta data file. - * - * @return the timestamp from the currently published nvdcve downloads page - * @throws MalformedURLException thrown if the URL for the NVD CCE Meta data - * is incorrect. - * @throws DownloadFailedException thrown if there is an error downloading - * the nvd cve meta data file - * @throws InvalidDataException thrown if there is an exception parsing the - * timestamps - * @throws InvalidSettingException thrown if the settings are invalid - */ - private Map retrieveCurrentTimestampsFromWeb() - throws MalformedURLException, DownloadFailedException, InvalidDataException, InvalidSettingException { - - final Map map = new TreeMap(); - String retrieveUrl = Settings.getString(Settings.KEYS.CVE_MODIFIED_20_URL); - if (retrieveUrl == null && properties.isBatchUpdateMode()) { - final NvdCveInfo item = new NvdCveInfo(); - retrieveUrl = Settings.getString(Settings.KEYS.BATCH_UPDATE_URL); - if (retrieveUrl == null) { - final String msg = "Invalid configuration - neither the modified or batch update URLs are specified in the configuration."; - Logger.getLogger(DataStoreMetaInfo.class.getName()).log(Level.SEVERE, msg); - throw new InvalidSettingException(msg); - } - item.setTimestamp(Downloader.getLastModified(new URL(retrieveUrl))); - item.setId(BATCH); - item.setNeedsUpdate(false); - map.put(BATCH, item); - } else { - NvdCveInfo item = new NvdCveInfo(); - item.setNeedsUpdate(false); //the others default to true, to make life easier later this should default to false. - item.setId(MODIFIED); - item.setUrl(retrieveUrl); - item.setOldSchemaVersionUrl(Settings.getString(Settings.KEYS.CVE_MODIFIED_12_URL)); - - item.setTimestamp(Downloader.getLastModified(new URL(retrieveUrl))); - map.put(MODIFIED, item); - - //only add these urls if we are not in batch mode - if (!properties.isBatchUpdateMode()) { - final int start = Settings.getInt(Settings.KEYS.CVE_START_YEAR); - final int end = Calendar.getInstance().get(Calendar.YEAR); - final String baseUrl20 = Settings.getString(Settings.KEYS.CVE_SCHEMA_2_0); - final String baseUrl12 = Settings.getString(Settings.KEYS.CVE_SCHEMA_1_2); - for (int i = start; i <= end; i++) { - retrieveUrl = String.format(baseUrl20, i); - item = new NvdCveInfo(); - item.setId(Integer.toString(i)); - item.setUrl(retrieveUrl); - item.setOldSchemaVersionUrl(String.format(baseUrl12, i)); - item.setTimestamp(Downloader.getLastModified(new URL(retrieveUrl))); - map.put(item.getId(), item); - } - } - } - return map; - } } diff --git a/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/StandardUpdate.java b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/StandardUpdate.java new file mode 100644 index 000000000..6cefcaa0f --- /dev/null +++ b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/StandardUpdate.java @@ -0,0 +1,292 @@ +/* + * This file is part of dependency-check-core. + * + * Dependency-check-core is free software: you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation, either version 3 of the License, or (at your option) any + * later version. + * + * Dependency-check-core is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more + * details. + * + * You should have received a copy of the GNU General Public License along with + * dependency-check-core. If not, see http://www.gnu.org/licenses/. + * + * Copyright (c) 2012 Jeremy Long. All Rights Reserved. + */ +package org.owasp.dependencycheck.data.update; + +import org.owasp.dependencycheck.data.nvdcve.InvalidDataException; +import java.io.File; +import java.io.FileNotFoundException; +import java.io.IOException; +import javax.xml.parsers.ParserConfigurationException; +import org.xml.sax.SAXException; +import java.net.MalformedURLException; +import java.net.URL; +import java.sql.SQLException; +import java.util.Calendar; +import java.util.Date; +import java.util.logging.Level; +import java.util.logging.Logger; +import org.owasp.dependencycheck.data.UpdateException; +import org.owasp.dependencycheck.data.nvdcve.CveDB; +import org.owasp.dependencycheck.utils.DownloadFailedException; +import org.owasp.dependencycheck.utils.Downloader; +import org.owasp.dependencycheck.utils.Settings; +import org.owasp.dependencycheck.data.nvdcve.DatabaseException; +import org.owasp.dependencycheck.utils.InvalidSettingException; +import static org.owasp.dependencycheck.data.update.DataStoreMetaInfo.MODIFIED; + +/** + * Class responsible for updating the CPE and NVDCVE data stores. + * + * @author Jeremy Long (jeremy.long@owasp.org) + */ +public class StandardUpdate extends AbstractUpdate { + + public StandardUpdate() throws MalformedURLException, DownloadFailedException, UpdateException { + super(); + } + + /** + *

Downloads the latest NVD CVE XML file from the web and imports it into + * the current CVE Database.

+ * + * @param updatesNeeded a collection of NvdCveInfo containing information + * about needed updates. + * @throws UpdateException is thrown if there is an error updating the + * database + */ + @Override + public void update() throws UpdateException { + try { + properties = new DataStoreMetaInfo(); + int maxUpdates = 0; + for (NvdCveInfo cve : updatesNeeded) { + if (cve.getNeedsUpdate()) { + maxUpdates += 1; + } + } + if (maxUpdates > 3) { + Logger.getLogger(StandardUpdate.class.getName()).log(Level.INFO, + "NVD CVE requires several updates; this could take a couple of minutes."); + } + if (maxUpdates > 0) { + openDataStores(); + } + + int count = 0; + for (NvdCveInfo cve : updatesNeeded) { + if (cve.getNeedsUpdate()) { + count += 1; + Logger.getLogger(StandardUpdate.class.getName()).log(Level.INFO, + "Updating NVD CVE ({0} of {1})", new Object[]{count, maxUpdates}); + URL url = new URL(cve.getUrl()); + File outputPath = null; + File outputPath12 = null; + try { + Logger.getLogger(StandardUpdate.class.getName()).log(Level.INFO, + "Downloading {0}", cve.getUrl()); + outputPath = File.createTempFile("cve" + cve.getId() + "_", ".xml"); + Downloader.fetchFile(url, outputPath); + + url = new URL(cve.getOldSchemaVersionUrl()); + outputPath12 = File.createTempFile("cve_1_2_" + cve.getId() + "_", ".xml"); + Downloader.fetchFile(url, outputPath12); + + Logger.getLogger(StandardUpdate.class.getName()).log(Level.INFO, + "Processing {0}", cve.getUrl()); + + importXML(outputPath, outputPath12); + + cveDB.commit(); + cpeIndex.commit(); + + properties.save(cve); + + Logger.getLogger(StandardUpdate.class.getName()).log(Level.INFO, + "Completed update {0} of {1}", new Object[]{count, maxUpdates}); + } catch (FileNotFoundException ex) { + throw new UpdateException(ex); + } catch (ParserConfigurationException ex) { + throw new UpdateException(ex); + } catch (SAXException ex) { + throw new UpdateException(ex); + } catch (IOException ex) { + throw new UpdateException(ex); + } catch (SQLException ex) { + throw new UpdateException(ex); + } catch (DatabaseException ex) { + throw new UpdateException(ex); + } catch (ClassNotFoundException ex) { + throw new UpdateException(ex); + } finally { + boolean deleted = false; + try { + if (outputPath != null && outputPath.exists()) { + deleted = outputPath.delete(); + } + } finally { + if (outputPath != null && (outputPath.exists() || !deleted)) { + outputPath.deleteOnExit(); + } + } + try { + deleted = false; + if (outputPath12 != null && outputPath12.exists()) { + deleted = outputPath12.delete(); + } + } finally { + if (outputPath12 != null && (outputPath12.exists() || !deleted)) { + outputPath12.deleteOnExit(); + } + } + } + } + } + if (maxUpdates >= 1) { //ensure the modified file date gets written + properties.save(updatesNeeded.get(MODIFIED)); + cveDB.cleanupDatabase(); + } + } catch (MalformedURLException ex) { + throw new UpdateException(ex); + } finally { + closeDataStores(); + } + } + + /** + * Determines if the index needs to be updated. This is done by fetching the + * NVD CVE meta data and checking the last update date. If the data needs to + * be refreshed this method will return the NvdCveUrl for the files that + * need to be updated. + * + * @return the collection of files that need to be updated + * @throws MalformedURLException is thrown if the URL for the NVD CVE Meta + * data is incorrect + * @throws DownloadFailedException is thrown if there is an error. + * downloading the NVD CVE download data file + * @throws UpdateException Is thrown if there is an issue with the last + * updated properties file + */ + @Override + protected Updateable updatesNeeded() throws MalformedURLException, DownloadFailedException, UpdateException { + Updateable updates = null; + try { + updates = retrieveCurrentTimestampsFromWeb(); + } catch (InvalidDataException ex) { + final String msg = "Unable to retrieve valid timestamp from nvd cve downloads page"; + Logger.getLogger(StandardUpdate.class.getName()).log(Level.FINE, msg, ex); + throw new DownloadFailedException(msg, ex); + } catch (InvalidSettingException ex) { + Logger.getLogger(StandardUpdate.class.getName()).log(Level.FINE, "Invalid setting found when retrieving timestamps", ex); + throw new DownloadFailedException("Invalid settings", ex); + } + + if (updates == null) { + throw new DownloadFailedException("Unable to retrieve the timestamps of the currently published NVD CVE data"); + } + + if (!properties.isEmpty()) { + try { + float version; + + if (properties.getProperty("version") == null) { + setDeleteAndRecreate(true); + } else { + try { + version = Float.parseFloat(properties.getProperty("version")); + final float currentVersion = Float.parseFloat(CveDB.DB_SCHEMA_VERSION); + if (currentVersion > version) { + setDeleteAndRecreate(true); + } + } catch (NumberFormatException ex) { + setDeleteAndRecreate(true); + } + } + + if (shouldDeleteAndRecreate()) { + return updates; + } + + final long lastUpdated = Long.parseLong(properties.getProperty(DataStoreMetaInfo.LAST_UPDATED, "0")); + final Date now = new Date(); + final int days = Settings.getInt(Settings.KEYS.CVE_MODIFIED_VALID_FOR_DAYS, 7); + final int start = Settings.getInt(Settings.KEYS.CVE_START_YEAR, 2002); + final int end = Calendar.getInstance().get(Calendar.YEAR); + if (lastUpdated == updates.getTimeStamp(MODIFIED)) { + updates.clear(); //we don't need to update anything. + } else if (withinRange(lastUpdated, now.getTime(), days)) { + for (NvdCveInfo entry : updates) { + if (MODIFIED.equals(entry.getId())) { + entry.setNeedsUpdate(true); + } else { + entry.setNeedsUpdate(false); + } + } + } else { //we figure out which of the several XML files need to be downloaded. + for (NvdCveInfo entry : updates) { + if (MODIFIED.equals(entry.getId())) { + entry.setNeedsUpdate(true); + } else { + long currentTimestamp = 0; + try { + currentTimestamp = Long.parseLong(properties.getProperty(DataStoreMetaInfo.LAST_UPDATED_BASE + entry.getId(), "0")); + } catch (NumberFormatException ex) { + final String msg = String.format("Error parsing '%s' '%s' from nvdcve.lastupdated", + DataStoreMetaInfo.LAST_UPDATED_BASE, entry.getId()); + Logger.getLogger(StandardUpdate.class.getName()).log(Level.FINE, msg, ex); + } + if (currentTimestamp == entry.getTimestamp()) { + entry.setNeedsUpdate(false); + } + } + } + } + } catch (NumberFormatException ex) { + final String msg = "An invalid schema version or timestamp exists in the data.properties file."; + Logger.getLogger(StandardUpdate.class.getName()).log(Level.WARNING, msg); + Logger.getLogger(StandardUpdate.class.getName()).log(Level.FINE, null, ex); + } + } + return updates; + } + + /** + * Retrieves the timestamps from the NVD CVE meta data file. + * + * @return the timestamp from the currently published nvdcve downloads page + * @throws MalformedURLException thrown if the URL for the NVD CCE Meta data + * is incorrect. + * @throws DownloadFailedException thrown if there is an error downloading + * the nvd cve meta data file + * @throws InvalidDataException thrown if there is an exception parsing the + * timestamps + * @throws InvalidSettingException thrown if the settings are invalid + */ + private Updateable retrieveCurrentTimestampsFromWeb() + throws MalformedURLException, DownloadFailedException, InvalidDataException, InvalidSettingException { + + Updateable updates = new Updateable(); + updates.add(MODIFIED, Settings.getString(Settings.KEYS.CVE_MODIFIED_20_URL), + Settings.getString(Settings.KEYS.CVE_MODIFIED_12_URL), + false); + + //only add these urls if we are not in batch mode + if (!properties.isBatchUpdateMode()) { + final int start = Settings.getInt(Settings.KEYS.CVE_START_YEAR); + final int end = Calendar.getInstance().get(Calendar.YEAR); + final String baseUrl20 = Settings.getString(Settings.KEYS.CVE_SCHEMA_2_0); + final String baseUrl12 = Settings.getString(Settings.KEYS.CVE_SCHEMA_1_2); + for (int i = start; i <= end; i++) { + updates.add(Integer.toString(i), String.format(baseUrl20, i), + String.format(baseUrl12, i), + true); + } + } + return updates; + } +} diff --git a/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/Updateable.java b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/Updateable.java new file mode 100644 index 000000000..02bb14c74 --- /dev/null +++ b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/Updateable.java @@ -0,0 +1,152 @@ +/* + * To change this template, choose Tools | Templates + * and open the template in the editor. + */ +package org.owasp.dependencycheck.data.update; + +import java.net.MalformedURLException; +import java.net.URL; +import java.util.Iterator; +import java.util.Map; +import java.util.Map.Entry; +import java.util.TreeMap; +import org.owasp.dependencycheck.utils.DownloadFailedException; +import org.owasp.dependencycheck.utils.Downloader; + +/** + * Contains a collection of updateable NvdCveInfo objects. This is used to + * determine which files need to be downloaded and processed. + * + * @author Jeremy Long (jeremy.long@owasp.org) + */ +public class Updateable implements java.lang.Iterable, Iterator { + + /** + * A collection of sources of data. + */ + Map collection = new TreeMap(); + + /** + * Gets whether or not an update is needed. + * + * @return true or false depending on whether an update is needed + */ + public boolean isUpdateNeeded() { + for (NvdCveInfo item : this) { + if (item.getNeedsUpdate()) { + return true; + } + } + return false; + } + + /** + * Adds a new entry of updateable information to the contained collection. + * + * @param id the key for the item to be added + * @param url the URL to download the item + * @param oldUrl the URL for the old version of the item (the NVD CVE old + * schema still contains useful data we need). + * @throws MalformedURLException thrown if the URL provided is invalid + * @throws DownloadFailedException thrown if the download fails. + */ + public void add(String id, String url, String oldUrl) throws MalformedURLException, DownloadFailedException { + add(id, url, oldUrl, false); + } + + /** + * Adds a new entry of updateable information to the contained collection. + * + * @param id the key for the item to be added + * @param url the URL to download the item + * @param oldUrl the URL for the old version of the item (the NVD CVE old + * schema still contains useful data we need). + * @param needsUpdate whether or not the data needs to be updated + * @throws MalformedURLException thrown if the URL provided is invalid + * @throws DownloadFailedException thrown if the download fails. + */ + public void add(String id, String url, String oldUrl, boolean needsUpdate) throws MalformedURLException, DownloadFailedException { + NvdCveInfo item = new NvdCveInfo(); + item.setNeedsUpdate(needsUpdate); //the others default to true, to make life easier later this should default to false. + item.setId(id); + item.setUrl(url); + item.setOldSchemaVersionUrl(oldUrl); + item.setTimestamp(Downloader.getLastModified(new URL(url))); + collection.put(id, item); + } + + /** + * Clears the contained collection of NvdCveInfo entries. + */ + public void clear() { + collection.clear(); + } + + /** + * Returns the timestamp for the given entry. + * + * @param key the key to lookup in the collection of NvdCveInfo items + * @return the timestamp for the given entry + */ + public long getTimeStamp(String key) { + return collection.get(key).getTimestamp(); + } + /** + * An internal iterator used to implement iterable. + */ + Iterator> iterableContent = null; + + /** + *

Returns an iterator for the NvdCveInfo contained.

+ *

This method is not thread safe.

+ * + * @return an NvdCveInfo Iterator + */ + @Override + public Iterator iterator() { + iterableContent = collection.entrySet().iterator(); + return this; + } + + /** + *

Returns whether or not there is another item in the collection.

+ *

This method is not thread safe.

+ * + * @return true or false depending on whether or not another item exists in + * the collection + */ + @Override + public boolean hasNext() { + return iterableContent.hasNext(); + } + + /** + *

Returns the next item in the collection.

+ *

This method is not thread safe.

+ * + * @return the next NvdCveInfo item in the collection + */ + @Override + public NvdCveInfo next() { + return iterableContent.next().getValue(); + } + + /** + *

Removes the current NvdCveInfo object from the collection.

+ *

This method is not thread safe.

+ */ + @Override + public void remove() { + iterableContent.remove(); + } + + /** + * Returns the specified item from the collection. + * + * @param key the key to lookup the return value + * @return the NvdCveInfo object stored using the specified key + */ + NvdCveInfo get(String key) { + return collection.get(key); + } +}