diff --git a/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/AbstractUpdateTask.java b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/AbstractUpdateTask.java deleted file mode 100644 index 531cd39c0..000000000 --- a/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/AbstractUpdateTask.java +++ /dev/null @@ -1,260 +0,0 @@ -/* - * This file is part of dependency-check-core. - * - * Dependency-check-core is free software: you can redistribute it and/or modify it - * under the terms of the GNU General Public License as published by the Free - * Software Foundation, either version 3 of the License, or (at your option) any - * later version. - * - * Dependency-check-core is distributed in the hope that it will be useful, but - * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more - * details. - * - * You should have received a copy of the GNU General Public License along with - * dependency-check-core. If not, see http://www.gnu.org/licenses/. - * - * Copyright (c) 2012 Jeremy Long. All Rights Reserved. - */ -package org.owasp.dependencycheck.data.update; - -import java.io.File; -import java.io.IOException; -import java.net.MalformedURLException; -import java.sql.SQLException; -import java.util.List; -import java.util.Map; -import java.util.logging.Level; -import java.util.logging.Logger; -import javax.xml.parsers.ParserConfigurationException; -import javax.xml.parsers.SAXParser; -import javax.xml.parsers.SAXParserFactory; -import org.owasp.dependencycheck.data.UpdateException; -import org.owasp.dependencycheck.data.nvdcve.CveDB; -import org.owasp.dependencycheck.utils.FileUtils; -import org.owasp.dependencycheck.utils.Settings; -import org.owasp.dependencycheck.data.nvdcve.DatabaseException; -import org.owasp.dependencycheck.data.nvdcve.NvdCve12Handler; -import org.owasp.dependencycheck.data.nvdcve.NvdCve20Handler; -import org.owasp.dependencycheck.dependency.VulnerableSoftware; -import org.owasp.dependencycheck.utils.DownloadFailedException; -import org.xml.sax.SAXException; - -/** - * Class responsible for updating the CPE and NVDCVE data stores. - * - * @author Jeremy Long (jeremy.long@owasp.org) - */ -public abstract class AbstractUpdateTask implements UpdateTask { - - /** - * Initializes the AbstractUpdateTask. - * - * @param properties information about the data store - * @throws MalformedURLException thrown if the configuration contains a - * malformed url - * @throws DownloadFailedException thrown if the timestamp on a file cannot - * be checked - * @throws UpdateException thrown if the update fails - */ - public AbstractUpdateTask(DataStoreMetaInfo properties) throws MalformedURLException, DownloadFailedException, UpdateException { - this.properties = properties; - this.updateable = updatesNeeded(); - } - /** - * A collection of updateable NVD CVE items. - */ - private Updateable updateable; - /** - * Utility to read and write meta-data about the data. - */ - private DataStoreMetaInfo properties = null; - - /** - * Returns the data store properties. - * - * @return the data store properties - */ - protected DataStoreMetaInfo getProperties() { - return properties; - } - /** - * Reference to the Cve Database. - */ - private CveDB cveDB = null; - - /** - * Returns the CveDB. - * - * @return the CveDB - */ - protected CveDB getCveDB() { - return cveDB; - } - - /** - * Gets whether or not an update is needed. - * - * @return true or false depending on whether an update is needed - */ - public boolean isUpdateNeeded() { - return updateable.isUpdateNeeded(); - } - - /** - * Gets the updateable NVD CVE Entries. - * - * @return an Updateable object containing the NVD CVE entries - */ - protected Updateable getUpdateable() { - return updateable; - } - - /** - * Determines if the index needs to be updated. - * - * @return a collection of updateable resources. - * @throws MalformedURLException is thrown if the URL for the NVD CVE Meta - * data is incorrect. - * @throws DownloadFailedException is thrown if there is an error. - * downloading the NVD CVE download data file. - * @throws UpdateException Is thrown if there is an issue with the last - * updated properties file. - */ - protected abstract Updateable updatesNeeded() throws MalformedURLException, DownloadFailedException, UpdateException; - - /** - *

Updates the data store to the latest version.

- * - * @throws UpdateException is thrown if there is an error updating the - * database - */ - public abstract void update() throws UpdateException; - /** - * A flag indicating whether or not the current data store should be - * deleted. - */ - private boolean deleteAndRecreate = false; - - /** - * Get the value of deleteAndRecreate. - * - * @return the value of deleteAndRecreate - */ - public boolean shouldDeleteAndRecreate() { - return deleteAndRecreate; - } - - /** - * Set the value of deleteAndRecreate. - * - * @param deleteAndRecreate new value of deleteAndRecreate - */ - protected void setDeleteAndRecreate(boolean deleteAndRecreate) { - this.deleteAndRecreate = deleteAndRecreate; - } - - /** - * Deletes the existing data directories. - * - * @throws IOException thrown if the directory cannot be deleted - */ - protected void deleteExistingData() throws IOException { - File data = Settings.getDataFile(Settings.KEYS.CVE_DATA_DIRECTORY); - if (data.exists()) { - FileUtils.delete(data); - } - data = DataStoreMetaInfo.getPropertiesFile(); - if (data.exists()) { - FileUtils.delete(data); - } - } - - /** - * Closes the CVE and CPE data stores. - */ - protected void closeDataStores() { - if (cveDB != null) { - try { - cveDB.close(); - } catch (Exception ignore) { - Logger.getLogger(AbstractUpdateTask.class.getName()).log(Level.FINEST, "Error closing the cveDB", ignore); - } - } - } - - /** - * Opens the CVE and CPE data stores. - * - * @throws UpdateException thrown if a data store cannot be opened - */ - protected void openDataStores() throws UpdateException { - //open the cve and cpe data stores - try { - cveDB = new CveDB(); - cveDB.open(); - } catch (IOException ex) { - closeDataStores(); - Logger.getLogger(AbstractUpdateTask.class.getName()).log(Level.FINE, "IO Error opening databases", ex); - throw new UpdateException("Error updating the CPE/CVE data, please see the log file for more details."); - } catch (SQLException ex) { - closeDataStores(); - Logger.getLogger(AbstractUpdateTask.class.getName()).log(Level.FINE, "SQL Exception opening databases", ex); - throw new UpdateException("Error updating the CPE/CVE data, please see the log file for more details."); - } catch (DatabaseException ex) { - closeDataStores(); - Logger.getLogger(AbstractUpdateTask.class.getName()).log(Level.FINE, "Database Exception opening databases", ex); - throw new UpdateException("Error updating the CPE/CVE data, please see the log file for more details."); - } catch (ClassNotFoundException ex) { - closeDataStores(); - Logger.getLogger(AbstractUpdateTask.class.getName()).log(Level.FINE, "Class not found exception opening databases", ex); - throw new UpdateException("Error updating the CPE/CVE data, please see the log file for more details."); - } - } - - /** - * Determines if the epoch date is within the range specified of the - * compareTo epoch time. This takes the (compareTo-date)/1000/60/60/24 to - * get the number of days. If the calculated days is less then the range the - * date is considered valid. - * - * @param date the date to be checked. - * @param compareTo the date to compare to. - * @param range the range in days to be considered valid. - * @return whether or not the date is within the range. - */ - protected boolean withinRange(long date, long compareTo, int range) { - final double differenceInDays = (compareTo - date) / 1000.0 / 60.0 / 60.0 / 24.0; - return differenceInDays < range; - } - - /** - * Imports the NVD CVE XML File into the Lucene Index. - * - * @param file the file containing the NVD CVE XML - * @param oldVersion contains the file containing the NVD CVE XML 1.2 - * @throws ParserConfigurationException is thrown if there is a parser - * configuration exception - * @throws SAXException is thrown if there is a SAXException - * @throws IOException is thrown if there is a IO Exception - * @throws SQLException is thrown if there is a SQL exception - * @throws DatabaseException is thrown if there is a database exception - * @throws ClassNotFoundException thrown if the h2 database driver cannot be - * loaded - */ - protected void importXML(File file, File oldVersion) throws ParserConfigurationException, - SAXException, IOException, SQLException, DatabaseException, ClassNotFoundException { - - final SAXParserFactory factory = SAXParserFactory.newInstance(); - final SAXParser saxParser = factory.newSAXParser(); - - final NvdCve12Handler cve12Handler = new NvdCve12Handler(); - saxParser.parse(oldVersion, cve12Handler); - final Map> prevVersionVulnMap = cve12Handler.getVulnerabilities(); - - final NvdCve20Handler cve20Handler = new NvdCve20Handler(); - cve20Handler.setCveDB(cveDB); - cve20Handler.setPrevVersionVulnMap(prevVersionVulnMap); - saxParser.parse(file, cve20Handler); - } -} diff --git a/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/BatchUpdateTask.java b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/BatchUpdateTask.java deleted file mode 100644 index cfc60457a..000000000 --- a/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/BatchUpdateTask.java +++ /dev/null @@ -1,272 +0,0 @@ -/* - * This file is part of dependency-check-core. - * - * Dependency-check-core is free software: you can redistribute it and/or modify it - * under the terms of the GNU General Public License as published by the Free - * Software Foundation, either version 3 of the License, or (at your option) any - * later version. - * - * Dependency-check-core is distributed in the hope that it will be useful, but - * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more - * details. - * - * You should have received a copy of the GNU General Public License along with - * dependency-check-core. If not, see http://www.gnu.org/licenses/. - * - * Copyright (c) 2012 Jeremy Long. All Rights Reserved. - */ -package org.owasp.dependencycheck.data.update; - -import org.owasp.dependencycheck.data.nvdcve.InvalidDataException; -import java.io.File; -import java.io.IOException; -import java.net.MalformedURLException; -import java.net.URISyntaxException; -import java.net.URL; -import java.util.Calendar; -import java.util.Date; -import java.util.logging.Level; -import java.util.logging.Logger; -import org.owasp.dependencycheck.data.UpdateException; -import org.owasp.dependencycheck.data.nvdcve.CveDB; -import org.owasp.dependencycheck.utils.DownloadFailedException; -import org.owasp.dependencycheck.utils.Downloader; -import org.owasp.dependencycheck.utils.FileUtils; -import org.owasp.dependencycheck.utils.Settings; -import org.owasp.dependencycheck.utils.InvalidSettingException; -import static org.owasp.dependencycheck.data.update.DataStoreMetaInfo.BATCH; -import static org.owasp.dependencycheck.data.update.DataStoreMetaInfo.MODIFIED; - -/** - * Class responsible for updating the CPE and NVDCVE data stores. - * - * @author Jeremy Long (jeremy.long@owasp.org) - * @deprecated batch update mode will no longer be supported. - */ -@Deprecated -public class BatchUpdateTask extends AbstractUpdateTask { - - /** - * Constructs a new BatchUpdateTask. - * - * @param properties information about the data store - * @throws MalformedURLException thrown if a configured URL is malformed - * @throws DownloadFailedException thrown if a timestamp cannot be checked - * on a configured URL - * @throws UpdateException thrown if there is an exception generating the - * update task - */ - public BatchUpdateTask(DataStoreMetaInfo properties) throws MalformedURLException, DownloadFailedException, UpdateException { - super(properties); - } - /** - * A flag indicating whether or not the batch update should be performed. - */ - private boolean doBatchUpdate; - - /** - * Get the value of doBatchUpdate - * - * @return the value of doBatchUpdate - */ - protected boolean isDoBatchUpdate() { - return doBatchUpdate; - } - - /** - * Set the value of doBatchUpdate - * - * @param doBatchUpdate new value of doBatchUpdate - */ - protected void setDoBatchUpdate(boolean doBatchUpdate) { - this.doBatchUpdate = doBatchUpdate; - } - - /** - *

Downloads the latest NVD CVE XML file from the web and imports it into - * the current CVE Database.

- * - * @throws UpdateException is thrown if there is an error updating the - * database - */ - @Override - public void update() throws UpdateException { - if (getProperties().isBatchUpdateMode() && doBatchUpdate) { - final String batchSrc = Settings.getString(Settings.KEYS.BATCH_UPDATE_URL); - File tmp = null; - try { - deleteExistingData(); - final File dataDirectory = CveDB.getDataDirectory().getParentFile(); - final URL batchUrl = new URL(batchSrc); - if ("file".equals(batchUrl.getProtocol())) { - try { - tmp = new File(batchUrl.toURI()); - } catch (URISyntaxException ex) { - final String msg = String.format("Invalid batch update URI: %s", batchSrc); - throw new UpdateException(msg, ex); - } - } else if ("http".equals(batchUrl.getProtocol()) - || "https".equals(batchUrl.getProtocol())) { - tmp = File.createTempFile("batch_", ".zip"); - Downloader.fetchFile(batchUrl, tmp); - } - //TODO add FTP? - FileUtils.extractFiles(tmp, dataDirectory); - - } catch (IOException ex) { - final String msg = String.format("IO Exception Occured performing batch update using: %s", batchSrc); - throw new UpdateException(msg, ex); - } finally { - if (tmp != null && !tmp.delete()) { - tmp.deleteOnExit(); - } - } - } - } - - /** - * Determines if the index needs to be updated. This is done by fetching the - * NVD CVE meta data and checking the last update date. If the data needs to - * be refreshed this method will return the NvdCveUrl for the files that - * need to be updated. - * - * @return the collection of files that need to be updated - * @throws MalformedURLException is thrown if the URL for the NVD CVE Meta - * data is incorrect - * @throws DownloadFailedException is thrown if there is an error. - * downloading the NVD CVE download data file - * @throws UpdateException Is thrown if there is an issue with the last - * updated properties file - */ - @Override - public Updateable updatesNeeded() throws MalformedURLException, DownloadFailedException, UpdateException { - Updateable updates = null; - try { - updates = retrieveCurrentTimestampsFromWeb(); - } catch (InvalidDataException ex) { - final String msg = "Unable to retrieve valid timestamp from nvd cve downloads page"; - Logger.getLogger(BatchUpdateTask.class.getName()).log(Level.FINE, msg, ex); - throw new DownloadFailedException(msg, ex); - } catch (InvalidSettingException ex) { - Logger.getLogger(BatchUpdateTask.class.getName()).log(Level.FINE, "Invalid setting found when retrieving timestamps", ex); - throw new DownloadFailedException("Invalid settings", ex); - } - - if (updates == null) { - throw new DownloadFailedException("Unable to retrieve the timestamps of the currently published NVD CVE data"); - } - final DataStoreMetaInfo properties = getProperties(); - if (!properties.isEmpty()) { - try { - boolean deleteAndRecreate = false; - float version; - - if (properties.getProperty("version") == null) { - deleteAndRecreate = true; - } else { - try { - version = Float.parseFloat(properties.getProperty("version")); - final float currentVersion = Float.parseFloat(CveDB.DB_SCHEMA_VERSION); - if (currentVersion > version) { - deleteAndRecreate = true; - } - } catch (NumberFormatException ex) { - deleteAndRecreate = true; - } - } - - final NvdCveInfo batchInfo = updates.get(BATCH); - if (properties.isBatchUpdateMode() && batchInfo != null) { - final long lastUpdated = Long.parseLong(properties.getProperty(DataStoreMetaInfo.BATCH, "0")); - if (lastUpdated != batchInfo.getTimestamp()) { - deleteAndRecreate = true; - } - } - - if (deleteAndRecreate) { - setDoBatchUpdate(properties.isBatchUpdateMode()); - try { - deleteExistingData(); - } catch (IOException ex) { - final String msg = "Unable to delete existing data"; - Logger.getLogger(BatchUpdateTask.class.getName()).log(Level.WARNING, msg); - Logger.getLogger(BatchUpdateTask.class.getName()).log(Level.FINE, null, ex); - } - return updates; - } - - final long lastUpdated = Long.parseLong(properties.getProperty(DataStoreMetaInfo.LAST_UPDATED, "0")); - final Date now = new Date(); - final int days = Settings.getInt(Settings.KEYS.CVE_MODIFIED_VALID_FOR_DAYS, 7); - final int start = Settings.getInt(Settings.KEYS.CVE_START_YEAR, 2002); - final int end = Calendar.getInstance().get(Calendar.YEAR); - if (lastUpdated == updates.get(MODIFIED).getTimestamp()) { - updates.clear(); //we don't need to update anything. - setDoBatchUpdate(properties.isBatchUpdateMode()); - } else if (withinRange(lastUpdated, now.getTime(), days)) { - updates.get(MODIFIED).setNeedsUpdate(true); - if (properties.isBatchUpdateMode()) { - setDoBatchUpdate(false); - } else { - for (int i = start; i <= end; i++) { - updates.get(String.valueOf(i)).setNeedsUpdate(false); - } - } - } else if (properties.isBatchUpdateMode()) { - updates.get(MODIFIED).setNeedsUpdate(true); - setDoBatchUpdate(true); - } else { //we figure out which of the several XML files need to be downloaded. - updates.get(MODIFIED).setNeedsUpdate(false); - for (int i = start; i <= end; i++) { - final NvdCveInfo cve = updates.get(String.valueOf(i)); - long currentTimestamp = 0; - try { - currentTimestamp = Long.parseLong(properties.getProperty(DataStoreMetaInfo.LAST_UPDATED_BASE + String.valueOf(i), "0")); - } catch (NumberFormatException ex) { - final String msg = String.format("Error parsing '%s' '%s' from nvdcve.lastupdated", - DataStoreMetaInfo.LAST_UPDATED_BASE, String.valueOf(i)); - Logger.getLogger(BatchUpdateTask.class.getName()).log(Level.FINE, msg, ex); - } - if (currentTimestamp == cve.getTimestamp()) { - cve.setNeedsUpdate(false); //they default to true. - } - } - } - } catch (NumberFormatException ex) { - final String msg = "An invalid schema version or timestamp exists in the data.properties file."; - Logger.getLogger(BatchUpdateTask.class.getName()).log(Level.WARNING, msg); - Logger.getLogger(BatchUpdateTask.class.getName()).log(Level.FINE, null, ex); - setDoBatchUpdate(properties.isBatchUpdateMode()); - } - } - return updates; - } - - /** - * Retrieves the timestamps from the NVD CVE meta data file. - * - * @return the timestamp from the currently published nvdcve downloads page - * @throws MalformedURLException thrown if the URL for the NVD CCE Meta data - * is incorrect. - * @throws DownloadFailedException thrown if there is an error downloading - * the nvd cve meta data file - * @throws InvalidDataException thrown if there is an exception parsing the - * timestamps - * @throws InvalidSettingException thrown if the settings are invalid - */ - private Updateable retrieveCurrentTimestampsFromWeb() - throws MalformedURLException, DownloadFailedException, InvalidDataException, InvalidSettingException { - final Updateable updates = new Updateable(); - updates.add(BATCH, Settings.getString(Settings.KEYS.BATCH_UPDATE_URL), - null, false); - - final String url = Settings.getString(Settings.KEYS.CVE_MODIFIED_20_URL, ""); - if (!url.isEmpty()) { - updates.add(MODIFIED, url, - Settings.getString(Settings.KEYS.CVE_MODIFIED_12_URL), - false); - } - return updates; - } -} diff --git a/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/DatabaseUpdater.java b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/DatabaseUpdater.java index 6ab7a74fa..61778eb66 100644 --- a/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/DatabaseUpdater.java +++ b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/DatabaseUpdater.java @@ -46,7 +46,7 @@ public class DatabaseUpdater implements CachedWebDataSource { @Override public void update() throws UpdateException { try { - final UpdateTask task = UpdateTaskFactory.getUpdateTask(); + final StandardUpdateTask task = new StandardUpdateTask(); if (task.isUpdateNeeded()) { if (task.shouldDeleteAndRecreate()) { try { diff --git a/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/StandardUpdateTask.java b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/StandardUpdateTask.java index 4286bef20..6197a720d 100644 --- a/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/StandardUpdateTask.java +++ b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/StandardUpdateTask.java @@ -29,6 +29,8 @@ import java.sql.SQLException; import java.util.Calendar; import java.util.Date; import java.util.HashSet; +import java.util.List; +import java.util.Map; import java.util.Set; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; @@ -36,38 +38,89 @@ import java.util.concurrent.Executors; import java.util.concurrent.Future; import java.util.logging.Level; import java.util.logging.Logger; +import javax.xml.parsers.SAXParser; +import javax.xml.parsers.SAXParserFactory; import org.owasp.dependencycheck.data.UpdateException; import org.owasp.dependencycheck.data.nvdcve.CveDB; import org.owasp.dependencycheck.utils.DownloadFailedException; import org.owasp.dependencycheck.utils.Settings; import org.owasp.dependencycheck.data.nvdcve.DatabaseException; +import org.owasp.dependencycheck.data.nvdcve.NvdCve12Handler; +import org.owasp.dependencycheck.data.nvdcve.NvdCve20Handler; import org.owasp.dependencycheck.utils.InvalidSettingException; import static org.owasp.dependencycheck.data.update.DataStoreMetaInfo.MODIFIED; +import org.owasp.dependencycheck.dependency.VulnerableSoftware; +import org.owasp.dependencycheck.utils.FileUtils; /** * Class responsible for updating the CPE and NVDCVE data stores. * * @author Jeremy Long (jeremy.long@owasp.org) */ -public class StandardUpdateTask extends AbstractUpdateTask { +public class StandardUpdateTask { /** * The max thread pool size to use when downloading files. */ public static final int MAX_THREAD_POOL_SIZE = Settings.getInt(Settings.KEYS.MAX_DOWNLOAD_THREAD_POOL_SIZE, 3); + /** + * Information about the timestamps and URLs for data that needs to be + * updated. + */ + private DataStoreMetaInfo properties; + /** + * A collection of updateable NVD CVE items. + */ + private Updateable updateable; + /** + * A flag indicating whether or not the current data store should be + * deleted. + */ + private boolean deleteAndRecreate = false; + /** + * Reference to the Cve Database. + */ + private CveDB cveDB = null; + + /** + * Gets whether or not an update is needed. + * + * @return true or false depending on whether an update is needed + */ + public boolean isUpdateNeeded() { + return updateable.isUpdateNeeded(); + } + + /** + * Set the value of deleteAndRecreate. + * + * @param deleteAndRecreate new value of deleteAndRecreate + */ + protected void setDeleteAndRecreate(boolean deleteAndRecreate) { + this.deleteAndRecreate = deleteAndRecreate; + } + + /** + * Get the value of deleteAndRecreate. + * + * @return the value of deleteAndRecreate + */ + public boolean shouldDeleteAndRecreate() { + return deleteAndRecreate; + } /** * Constructs a new Standard Update Task. * - * @param properties information about the data store * @throws MalformedURLException thrown if a configured URL is malformed * @throws DownloadFailedException thrown if a timestamp cannot be checked * on a configured URL * @throws UpdateException thrown if there is an exception generating the * update task */ - public StandardUpdateTask(DataStoreMetaInfo properties) throws MalformedURLException, DownloadFailedException, UpdateException { - super(properties); + public StandardUpdateTask() throws MalformedURLException, DownloadFailedException, UpdateException { + properties = new DataStoreMetaInfo(); + updateable = updatesNeeded(); } /** @@ -77,11 +130,10 @@ public class StandardUpdateTask extends AbstractUpdateTask { * @throws UpdateException is thrown if there is an error updating the * database */ - @Override public void update() throws UpdateException { int maxUpdates = 0; try { - for (NvdCveInfo cve : getUpdateable()) { + for (NvdCveInfo cve : updateable) { if (cve.getNeedsUpdate()) { maxUpdates += 1; } @@ -101,7 +153,7 @@ public class StandardUpdateTask extends AbstractUpdateTask { final ExecutorService executorService = Executors.newFixedThreadPool(poolSize); final Set> futures = new HashSet>(maxUpdates); - for (NvdCveInfo cve : getUpdateable()) { + for (NvdCveInfo cve : updateable) { if (cve.getNeedsUpdate()) { final File file1; final File file2; @@ -123,8 +175,8 @@ public class StandardUpdateTask extends AbstractUpdateTask { Logger.getLogger(StandardUpdateTask.class.getName()).log(Level.INFO, msg); try { importXML(filePair.getFirst(), filePair.getSecond()); - getCveDB().commit(); - getProperties().save(filePair.getNvdCveInfo()); + cveDB.commit(); + properties.save(filePair.getNvdCveInfo()); } catch (FileNotFoundException ex) { throw new UpdateException(ex); } catch (ParserConfigurationException ex) { @@ -159,8 +211,8 @@ public class StandardUpdateTask extends AbstractUpdateTask { } if (maxUpdates >= 1) { //ensure the modified file date gets written - getProperties().save(getUpdateable().get(MODIFIED)); - getCveDB().cleanupDatabase(); + properties.save(updateable.get(MODIFIED)); + cveDB.cleanupDatabase(); } } finally { closeDataStores(); @@ -279,7 +331,6 @@ public class StandardUpdateTask extends AbstractUpdateTask { * @throws UpdateException Is thrown if there is an issue with the last * updated properties file */ - @Override protected Updateable updatesNeeded() throws MalformedURLException, DownloadFailedException, UpdateException { Updateable updates = null; try { @@ -300,26 +351,25 @@ public class StandardUpdateTask extends AbstractUpdateTask { if (updates == null) { throw new DownloadFailedException("Unable to retrieve the timestamps of the currently published NVD CVE data"); } - final DataStoreMetaInfo properties = getProperties(); if (!properties.isEmpty()) { try { float version; if (properties.getProperty("version") == null) { - setDeleteAndRecreate(true); + deleteAndRecreate = true; } else { try { version = Float.parseFloat(properties.getProperty("version")); final float currentVersion = Float.parseFloat(CveDB.DB_SCHEMA_VERSION); if (currentVersion > version) { - setDeleteAndRecreate(true); + deleteAndRecreate = true; } } catch (NumberFormatException ex) { - setDeleteAndRecreate(true); + deleteAndRecreate = true; } } - if (shouldDeleteAndRecreate()) { + if (deleteAndRecreate) { return updates; } @@ -401,4 +451,108 @@ public class StandardUpdateTask extends AbstractUpdateTask { return updates; } + + /** + * Deletes the existing data directories. + * + * @throws IOException thrown if the directory cannot be deleted + */ + protected void deleteExistingData() throws IOException { + File data = Settings.getDataFile(Settings.KEYS.CVE_DATA_DIRECTORY); + if (data.exists()) { + FileUtils.delete(data); + } + data = DataStoreMetaInfo.getPropertiesFile(); + if (data.exists()) { + FileUtils.delete(data); + } + } + + /** + * Closes the CVE and CPE data stores. + */ + protected void closeDataStores() { + if (cveDB != null) { + try { + cveDB.close(); + } catch (Exception ignore) { + Logger.getLogger(StandardUpdateTask.class.getName()).log(Level.FINEST, "Error closing the cveDB", ignore); + } + } + } + + /** + * Opens the CVE and CPE data stores. + * + * @throws UpdateException thrown if a data store cannot be opened + */ + protected void openDataStores() throws UpdateException { + //open the cve and cpe data stores + try { + cveDB = new CveDB(); + cveDB.open(); + } catch (IOException ex) { + closeDataStores(); + Logger.getLogger(StandardUpdateTask.class.getName()).log(Level.FINE, "IO Error opening databases", ex); + throw new UpdateException("Error updating the CPE/CVE data, please see the log file for more details."); + } catch (SQLException ex) { + closeDataStores(); + Logger.getLogger(StandardUpdateTask.class.getName()).log(Level.FINE, "SQL Exception opening databases", ex); + throw new UpdateException("Error updating the CPE/CVE data, please see the log file for more details."); + } catch (DatabaseException ex) { + closeDataStores(); + Logger.getLogger(StandardUpdateTask.class.getName()).log(Level.FINE, "Database Exception opening databases", ex); + throw new UpdateException("Error updating the CPE/CVE data, please see the log file for more details."); + } catch (ClassNotFoundException ex) { + closeDataStores(); + Logger.getLogger(StandardUpdateTask.class.getName()).log(Level.FINE, "Class not found exception opening databases", ex); + throw new UpdateException("Error updating the CPE/CVE data, please see the log file for more details."); + } + } + + /** + * Determines if the epoch date is within the range specified of the + * compareTo epoch time. This takes the (compareTo-date)/1000/60/60/24 to + * get the number of days. If the calculated days is less then the range the + * date is considered valid. + * + * @param date the date to be checked. + * @param compareTo the date to compare to. + * @param range the range in days to be considered valid. + * @return whether or not the date is within the range. + */ + protected boolean withinRange(long date, long compareTo, int range) { + final double differenceInDays = (compareTo - date) / 1000.0 / 60.0 / 60.0 / 24.0; + return differenceInDays < range; + } + + /** + * Imports the NVD CVE XML File into the Lucene Index. + * + * @param file the file containing the NVD CVE XML + * @param oldVersion contains the file containing the NVD CVE XML 1.2 + * @throws ParserConfigurationException is thrown if there is a parser + * configuration exception + * @throws SAXException is thrown if there is a SAXException + * @throws IOException is thrown if there is a IO Exception + * @throws SQLException is thrown if there is a SQL exception + * @throws DatabaseException is thrown if there is a database exception + * @throws ClassNotFoundException thrown if the h2 database driver cannot be + * loaded + */ + protected void importXML(File file, File oldVersion) throws ParserConfigurationException, + SAXException, IOException, SQLException, DatabaseException, ClassNotFoundException { + + final SAXParserFactory factory = SAXParserFactory.newInstance(); + final SAXParser saxParser = factory.newSAXParser(); + + final NvdCve12Handler cve12Handler = new NvdCve12Handler(); + saxParser.parse(oldVersion, cve12Handler); + final Map> prevVersionVulnMap = cve12Handler.getVulnerabilities(); + + final NvdCve20Handler cve20Handler = new NvdCve20Handler(); + cve20Handler.setCveDB(cveDB); + cve20Handler.setPrevVersionVulnMap(prevVersionVulnMap); + saxParser.parse(file, cve20Handler); + } } diff --git a/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/UpdateTask.java b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/UpdateTask.java deleted file mode 100644 index 737eb25e6..000000000 --- a/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/UpdateTask.java +++ /dev/null @@ -1,51 +0,0 @@ -/* - * This file is part of dependency-check-core. - * - * Dependency-check-core is free software: you can redistribute it and/or modify it - * under the terms of the GNU General Public License as published by the Free - * Software Foundation, either version 3 of the License, or (at your option) any - * later version. - * - * Dependency-check-core is distributed in the hope that it will be useful, but - * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more - * details. - * - * You should have received a copy of the GNU General Public License along with - * dependency-check-core. If not, see http://www.gnu.org/licenses/. - * - * Copyright (c) 2012 Jeremy Long. All Rights Reserved. - */ -package org.owasp.dependencycheck.data.update; - -import org.owasp.dependencycheck.data.UpdateException; - -/** - * An interface defining an update task. - * - * @author Jeremy Long (jeremy.long@owasp.org) - */ -public interface UpdateTask { - - /** - *

Updates the data store to the latest version.

- * - * @throws UpdateException is thrown if there is an error updating the - * database - */ - void update() throws UpdateException; - - /** - * Get the value of deleteAndRecreate. - * - * @return the value of deleteAndRecreate - */ - boolean shouldDeleteAndRecreate(); - - /** - * Gets whether or not an update is needed. - * - * @return true or false depending on whether an update is needed - */ - boolean isUpdateNeeded(); -} diff --git a/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/UpdateTaskFactory.java b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/UpdateTaskFactory.java deleted file mode 100644 index eb1a4a10d..000000000 --- a/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/UpdateTaskFactory.java +++ /dev/null @@ -1,60 +0,0 @@ -/* - * This file is part of dependency-check-core. - * - * Dependency-check-core is free software: you can redistribute it and/or modify it - * under the terms of the GNU General Public License as published by the Free - * Software Foundation, either version 3 of the License, or (at your option) any - * later version. - * - * Dependency-check-core is distributed in the hope that it will be useful, but - * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more - * details. - * - * You should have received a copy of the GNU General Public License along with - * dependency-check-core. If not, see http://www.gnu.org/licenses/. - * - * Copyright (c) 2012 Jeremy Long. All Rights Reserved. - */ -package org.owasp.dependencycheck.data.update; - -import java.net.MalformedURLException; -import org.owasp.dependencycheck.data.UpdateException; -import org.owasp.dependencycheck.utils.DownloadFailedException; - -/** - * An UpdateTask Factory that instantiates the correct UpdateTask based on the - * given configuration. - * - * @author Jeremy Long (jeremy.long@owasp.org) - */ -public final class UpdateTaskFactory { - - /** - * private constructor for a utility class. - */ - private UpdateTaskFactory() { - //empty contrusctor for utility class - } - - /** - * Constructs the appropriate update task based on configuration. - * - * @return an UpdateTask - * @throws MalformedURLException thrown if a configured URL is malformed - * @throws DownloadFailedException thrown if a timestamp cannot be checked - * on a configured URL - * @throws UpdateException thrown if there is an exception generating the - * update task - */ - public static UpdateTask getUpdateTask() throws MalformedURLException, DownloadFailedException, UpdateException { - final UpdateTask task; - final DataStoreMetaInfo properties = new DataStoreMetaInfo(); -// if (properties.isBatchUpdateMode()) { -// task = new BatchUpdateTask(properties); -// } else { - task = new StandardUpdateTask(properties); -// } - return task; - } -} diff --git a/dependency-check-core/src/test/java/org/owasp/dependencycheck/data/update/AbstractUpdateTaskTest.java b/dependency-check-core/src/test/java/org/owasp/dependencycheck/data/update/AbstractUpdateTaskTest.java deleted file mode 100644 index 67db5a6bc..000000000 --- a/dependency-check-core/src/test/java/org/owasp/dependencycheck/data/update/AbstractUpdateTaskTest.java +++ /dev/null @@ -1,140 +0,0 @@ -/* - * This file is part of dependency-check-core. - * - * Dependency-check-core is free software: you can redistribute it and/or modify it - * under the terms of the GNU General Public License as published by the Free - * Software Foundation, either version 3 of the License, or (at your option) any - * later version. - * - * Dependency-check-core is distributed in the hope that it will be useful, but - * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more - * details. - * - * You should have received a copy of the GNU General Public License along with - * dependency-check-core. If not, see http://www.gnu.org/licenses/. - * - * Copyright (c) 2013 Jeremy Long. All Rights Reserved. - */ -package org.owasp.dependencycheck.data.update; - -import java.io.File; -import java.io.IOException; -import java.net.MalformedURLException; -import java.text.DateFormat; -import java.util.Calendar; -import java.util.logging.Level; -import java.util.logging.Logger; -import org.junit.After; -import org.junit.AfterClass; -import org.junit.Before; -import org.junit.BeforeClass; -import org.junit.Test; -import static org.junit.Assert.*; -import org.owasp.dependencycheck.data.UpdateException; -import org.owasp.dependencycheck.utils.DownloadFailedException; - -/** - * - * @author Jeremy Long (jeremy.long@owasp.org) - */ -public class AbstractUpdateTaskTest { - - public AbstractUpdateTaskTest() { - } - - @BeforeClass - public static void setUpClass() { - } - - @AfterClass - public static void tearDownClass() { - } - - @Before - public void setUp() { - } - - @After - public void tearDown() { - } - - public AbstractUpdateTask getAbstractUpdateImpl() throws Exception { - DataStoreMetaInfo props = new DataStoreMetaInfo(); - AbstractUpdateTask instance = new AbstractUpdateImpl(props); - return instance; - } - - /** - * Test of setDeleteAndRecreate method, of class AbstractUpdateTask. - */ - @Test - public void testSetDeleteAndRecreate() throws Exception { - boolean deleteAndRecreate = false; - boolean expResult = false; - AbstractUpdateTask instance = getAbstractUpdateImpl(); - instance.setDeleteAndRecreate(deleteAndRecreate); - boolean result = instance.shouldDeleteAndRecreate(); - assertEquals(expResult, result); - } - - /** - * Test of deleteExistingData method, of class AbstractUpdateTask. - */ - @Test - public void testDeleteExistingData() throws Exception { - AbstractUpdateTask instance = getAbstractUpdateImpl(); - Exception result = null; - try { - instance.deleteExistingData(); - } catch (IOException ex) { - result = ex; - } - assertNull(result); - } - - /** - * Test of openDataStores method, of class AbstractUpdateTask. - */ - @Test - public void testOpenDataStores() throws Exception { - AbstractUpdateTask instance = getAbstractUpdateImpl(); - instance.openDataStores(); - instance.closeDataStores(); - } - - /** - * Test of withinRange method, of class AbstractUpdateTask. - */ - @Test - public void testWithinRange() throws Exception { - Calendar c = Calendar.getInstance(); - - long current = c.getTimeInMillis(); - long lastRun = c.getTimeInMillis() - (3 * (1000 * 60 * 60 * 24)); - int range = 7; // 7 days - AbstractUpdateTask instance = getAbstractUpdateImpl(); - boolean expResult = true; - boolean result = instance.withinRange(lastRun, current, range); - assertEquals(expResult, result); - - lastRun = c.getTimeInMillis() - (8 * (1000 * 60 * 60 * 24)); - expResult = false; - result = instance.withinRange(lastRun, current, range); - assertEquals(expResult, result); - } - - public class AbstractUpdateImpl extends AbstractUpdateTask { - - public AbstractUpdateImpl(DataStoreMetaInfo props) throws Exception { - super(props); - } - - public Updateable updatesNeeded() throws MalformedURLException, DownloadFailedException, UpdateException { - return null; - } - - public void update() throws UpdateException { - } - } -} diff --git a/dependency-check-core/src/test/java/org/owasp/dependencycheck/data/update/BatchUpdateTaskTest.java b/dependency-check-core/src/test/java/org/owasp/dependencycheck/data/update/BatchUpdateTaskTest.java deleted file mode 100644 index 96457e3ec..000000000 --- a/dependency-check-core/src/test/java/org/owasp/dependencycheck/data/update/BatchUpdateTaskTest.java +++ /dev/null @@ -1,121 +0,0 @@ -/* - * This file is part of dependency-check-core. - * - * Dependency-check-core is free software: you can redistribute it and/or modify it - * under the terms of the GNU General Public License as published by the Free - * Software Foundation, either version 3 of the License, or (at your option) any - * later version. - * - * Dependency-check-core is distributed in the hope that it will be useful, but - * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more - * details. - * - * You should have received a copy of the GNU General Public License along with - * dependency-check-core. If not, see http://www.gnu.org/licenses/. - * - * Copyright (c) 2013 Jeremy Long. All Rights Reserved. - */ -package org.owasp.dependencycheck.data.update; - -import java.io.File; -import java.net.MalformedURLException; -import org.apache.commons.io.FileUtils; -import org.junit.After; -import org.junit.AfterClass; -import org.junit.Before; -import org.junit.BeforeClass; -import org.junit.Test; -import static org.junit.Assert.*; -import org.owasp.dependencycheck.data.UpdateException; -import org.owasp.dependencycheck.utils.DownloadFailedException; -import org.owasp.dependencycheck.utils.Settings; - -/** - * - * @author Jeremy Long (jeremy.long@owasp.org) - */ -public class BatchUpdateTaskTest { - - public BatchUpdateTaskTest() { - } - - @BeforeClass - public static void setUpClass() { - } - - @AfterClass - public static void tearDownClass() { - } - private String old12; - private String old20; - - @Before - public void setUp() throws Exception { - old12 = Settings.getString(Settings.KEYS.CVE_MODIFIED_12_URL); - old20 = Settings.getString(Settings.KEYS.CVE_MODIFIED_20_URL); - - File tmp = Settings.getTempDirectory(); - if (!tmp.exists()) { - tmp.mkdirs(); - } - - File dest = new File(tmp, "data.zip"); - File file = new File(this.getClass().getClassLoader().getResource("data.zip").toURI()); - FileUtils.copyFile(file, dest); - String path = "file:///" + dest.getCanonicalPath(); - Settings.setString(Settings.KEYS.BATCH_UPDATE_URL, path); - - dest = new File(tmp, "nvdcve-2012.xml"); - file = new File(this.getClass().getClassLoader().getResource("nvdcve-2012.xml").toURI()); - FileUtils.copyFile(file, dest); - path = "file:///" + dest.getCanonicalPath(); - Settings.setString(Settings.KEYS.CVE_MODIFIED_12_URL, path); - - dest = new File(tmp, "nvdcve-2.0-2012.xml"); - file = new File(this.getClass().getClassLoader().getResource("nvdcve-2.0-2012.xml").toURI()); - FileUtils.copyFile(file, dest); - path = "file:///" + dest.getCanonicalPath(); - Settings.setString(Settings.KEYS.CVE_MODIFIED_20_URL, path); - } - - @After - public void tearDown() { - Settings.setString(Settings.KEYS.CVE_MODIFIED_12_URL, old12); - Settings.setString(Settings.KEYS.CVE_MODIFIED_20_URL, old20); - Settings.setString(Settings.KEYS.BATCH_UPDATE_URL, ""); - } - - public BatchUpdateTask getBatchUpdateTask() throws MalformedURLException, DownloadFailedException, UpdateException { - DataStoreMetaInfo props = new DataStoreMetaInfo(); - BatchUpdateTask instance = new BatchUpdateTask(props); - return instance; - } - - /** - * Test of setDoBatchUpdate method, of class BatchUpdateTask. - */ - @Test - public void testSetDoBatchUpdate() throws DownloadFailedException, MalformedURLException, UpdateException { - boolean expected = false; - BatchUpdateTask instance = getBatchUpdateTask(); - instance.setDoBatchUpdate(expected); - boolean results = instance.isDoBatchUpdate(); - assertEquals(results, expected); - } - - /** - * Test of update method, of class BatchUpdateTask. - */ - @Test - public void testUpdate() throws Exception { - BatchUpdateTask instance = getBatchUpdateTask(); - - //do some setup - instance.setDoBatchUpdate(true); - instance.deleteExistingData(); - - instance.update(); //no exceptions it worked? - //todo add some actual asserts to check things. - } -} diff --git a/dependency-check-core/src/test/java/org/owasp/dependencycheck/data/update/StandardUpdateTaskIntegrationTest.java b/dependency-check-core/src/test/java/org/owasp/dependencycheck/data/update/StandardUpdateTaskIntegrationTest.java index 52fbf092c..0d2e5c1c7 100644 --- a/dependency-check-core/src/test/java/org/owasp/dependencycheck/data/update/StandardUpdateTaskIntegrationTest.java +++ b/dependency-check-core/src/test/java/org/owasp/dependencycheck/data/update/StandardUpdateTaskIntegrationTest.java @@ -18,7 +18,9 @@ */ package org.owasp.dependencycheck.data.update; +import java.io.IOException; import java.net.MalformedURLException; +import java.util.Calendar; import org.junit.After; import org.junit.AfterClass; import org.junit.Before; @@ -54,11 +56,69 @@ public class StandardUpdateTaskIntegrationTest { } public StandardUpdateTask getStandardUpdateTask() throws MalformedURLException, DownloadFailedException, UpdateException { - DataStoreMetaInfo props = new DataStoreMetaInfo(); - StandardUpdateTask instance = new StandardUpdateTask(props); + StandardUpdateTask instance = new StandardUpdateTask(); return instance; } + /** + * Test of setDeleteAndRecreate method, of class StandardUpdateTask. + */ + @Test + public void testSetDeleteAndRecreate() throws Exception { + boolean deleteAndRecreate = false; + boolean expResult = false; + StandardUpdateTask instance = getStandardUpdateTask(); + instance.setDeleteAndRecreate(deleteAndRecreate); + boolean result = instance.shouldDeleteAndRecreate(); + assertEquals(expResult, result); + } + + /** + * Test of deleteExistingData method, of class StandardUpdateTask. + */ + @Test + public void testDeleteExistingData() throws Exception { + StandardUpdateTask instance = getStandardUpdateTask(); + Exception result = null; + try { + instance.deleteExistingData(); + } catch (IOException ex) { + result = ex; + } + assertNull(result); + } + + /** + * Test of openDataStores method, of class StandardUpdateTask. + */ + @Test + public void testOpenDataStores() throws Exception { + StandardUpdateTask instance = getStandardUpdateTask(); + instance.openDataStores(); + instance.closeDataStores(); + } + + /** + * Test of withinRange method, of class StandardUpdateTask. + */ + @Test + public void testWithinRange() throws Exception { + Calendar c = Calendar.getInstance(); + + long current = c.getTimeInMillis(); + long lastRun = c.getTimeInMillis() - (3 * (1000 * 60 * 60 * 24)); + int range = 7; // 7 days + StandardUpdateTask instance = getStandardUpdateTask(); + boolean expResult = true; + boolean result = instance.withinRange(lastRun, current, range); + assertEquals(expResult, result); + + lastRun = c.getTimeInMillis() - (8 * (1000 * 60 * 60 * 24)); + expResult = false; + result = instance.withinRange(lastRun, current, range); + assertEquals(expResult, result); + } + /** * Test of update method, of class StandardUpdateTask. */