From 9daa9b6cca4b65950b2711bfc448662895603087 Mon Sep 17 00:00:00 2001 From: Jeremy Long Date: Wed, 8 Jul 2015 06:20:19 -0400 Subject: [PATCH] reorganized code, made the database file name version independent so that upgrades can be made, and implemented a CPE updates per issue #149 Former-commit-id: fdb57afa28ecffdb7ca90971851844718ecb8bb9 --- .../data/nvdcve/ConnectionFactory.java | 83 +++- .../dependencycheck/data/nvdcve/CveDB.java | 43 ++- .../data/nvdcve/DatabaseProperties.java | 33 +- .../data/update/BaseUpdater.java | 86 +++++ .../data/update/CpeUpdater.java | 196 ++++++++++ .../data/update/NvdCveUpdater.java | 228 ++++++++++- .../data/update/StandardUpdate.java | 318 --------------- .../data/update/UpdateService.java | 1 + .../data/update/cpe/CPEHandler.java | 362 ++++++++++++++++++ .../dependencycheck/data/update/cpe/Cpe.java | 114 ++++++ .../update/{task => nvd}/DownloadTask.java | 10 +- .../update/{xml => nvd}/NvdCve12Handler.java | 2 +- .../update/{xml => nvd}/NvdCve20Handler.java | 2 +- .../data/update/{ => nvd}/NvdCveInfo.java | 2 +- .../update/{task => nvd}/ProcessTask.java | 6 +- .../update/{ => nvd}/UpdateableNvdCve.java | 14 +- .../update/{xml => nvd}/package-info.java | 2 +- .../data/update/task/package-info.java | 4 - ...dencycheck.data.update.CachedWebDataSource | 3 +- .../resources/data/dbStatements.properties | 3 + .../src/main/resources/data/upgrade_2.9.sql | 6 + .../main/resources/dependencycheck.properties | 6 +- .../data/nvdcve/BaseDBTestCase.java | 2 +- .../DatabasePropertiesIntegrationTest.java | 2 +- .../data/update/BaseUpdaterTest.java | 98 +++++ .../update/CpeUpdaterIntegrationTest.java | 41 ++ .../update/NvdCveUpdaterIntegrationTest.java | 61 ++- .../update/StandardUpdateIntegrationTest.java | 67 ---- .../{task => nvd}/DownloadTaskTest.java | 6 +- .../data/update/{ => nvd}/NvdCveInfoTest.java | 3 +- .../nvd/NvdCveUpdaterIntegrationTest.java | 69 ++++ .../{xml => nvd}/NvdCve_1_2_HandlerTest.java | 3 +- .../{xml => nvd}/NvdCve_2_0_HandlerTest.java | 3 +- .../{ => nvd}/UpdateableNvdCveTest.java | 4 +- .../test/resources/dependencycheck.properties | 4 +- .../owasp/dependencycheck/utils/Settings.java | 17 +- .../dependencycheck/utils/SettingsTest.java | 12 +- .../test/resources/dependencycheck.properties | 4 +- 38 files changed, 1397 insertions(+), 523 deletions(-) create mode 100644 dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/BaseUpdater.java create mode 100644 dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/CpeUpdater.java delete mode 100644 dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/StandardUpdate.java create mode 100644 dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/cpe/CPEHandler.java create mode 100644 dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/cpe/Cpe.java rename dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/{task => nvd}/DownloadTask.java (96%) rename dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/{xml => nvd}/NvdCve12Handler.java (99%) rename dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/{xml => nvd}/NvdCve20Handler.java (99%) rename dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/{ => nvd}/NvdCveInfo.java (98%) rename dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/{task => nvd}/ProcessTask.java (97%) rename dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/{ => nvd}/UpdateableNvdCve.java (95%) rename dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/{xml => nvd}/package-info.java (89%) delete mode 100644 dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/task/package-info.java create mode 100644 dependency-check-core/src/main/resources/data/upgrade_2.9.sql create mode 100644 dependency-check-core/src/test/java/org/owasp/dependencycheck/data/update/BaseUpdaterTest.java create mode 100644 dependency-check-core/src/test/java/org/owasp/dependencycheck/data/update/CpeUpdaterIntegrationTest.java delete mode 100644 dependency-check-core/src/test/java/org/owasp/dependencycheck/data/update/StandardUpdateIntegrationTest.java rename dependency-check-core/src/test/java/org/owasp/dependencycheck/data/update/{task => nvd}/DownloadTaskTest.java (89%) rename dependency-check-core/src/test/java/org/owasp/dependencycheck/data/update/{ => nvd}/NvdCveInfoTest.java (95%) create mode 100644 dependency-check-core/src/test/java/org/owasp/dependencycheck/data/update/nvd/NvdCveUpdaterIntegrationTest.java rename dependency-check-core/src/test/java/org/owasp/dependencycheck/data/update/{xml => nvd}/NvdCve_1_2_HandlerTest.java (94%) rename dependency-check-core/src/test/java/org/owasp/dependencycheck/data/update/{xml => nvd}/NvdCve_2_0_HandlerTest.java (94%) rename dependency-check-core/src/test/java/org/owasp/dependencycheck/data/update/{ => nvd}/UpdateableNvdCveTest.java (96%) diff --git a/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/nvdcve/ConnectionFactory.java b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/nvdcve/ConnectionFactory.java index a2c05f2b2..4855190a9 100644 --- a/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/nvdcve/ConnectionFactory.java +++ b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/nvdcve/ConnectionFactory.java @@ -35,9 +35,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** - * Loads the configured database driver and returns the database connection. If the embedded H2 database is used - * obtaining a connection will ensure the database file exists and that the appropriate table structure has been - * created. + * Loads the configured database driver and returns the database connection. If the embedded H2 database is used obtaining a + * connection will ensure the database file exists and that the appropriate table structure has been created. * * @author Jeremy Long */ @@ -55,6 +54,10 @@ public final class ConnectionFactory { * Resource location for SQL file used to create the database schema. */ public static final String DB_STRUCTURE_RESOURCE = "data/initialize.sql"; + /** + * Resource location for SQL file used to create the database schema. + */ + public static final String DB_STRUCTURE_UPDATE_RESOURCE = "data/upgrade_%s.sql"; /** * The database driver used to connect to the database. */ @@ -79,8 +82,8 @@ public final class ConnectionFactory { } /** - * Initializes the connection factory. Ensuring that the appropriate drivers are loaded and that a connection can be - * made successfully. + * Initializes the connection factory. Ensuring that the appropriate drivers are loaded and that a connection can be made + * successfully. * * @throws DatabaseException thrown if we are unable to connect to the database */ @@ -114,8 +117,7 @@ public final class ConnectionFactory { try { connectionString = Settings.getConnectionString( Settings.KEYS.DB_CONNECTION_STRING, - Settings.KEYS.DB_FILE_NAME, - Settings.KEYS.DB_VERSION); + Settings.KEYS.DB_FILE_NAME); } catch (IOException ex) { LOGGER.debug( "Unable to retrieve the database connection string", ex); @@ -162,13 +164,12 @@ public final class ConnectionFactory { LOGGER.debug("", dex); throw new DatabaseException("Unable to create the database structure"); } - } else { - try { - ensureSchemaVersion(conn); - } catch (DatabaseException dex) { - LOGGER.debug("", dex); - throw new DatabaseException("Database schema does not match this version of dependency-check"); - } + } + try { + ensureSchemaVersion(conn); + } catch (DatabaseException dex) { + LOGGER.debug("", dex); + throw new DatabaseException("Database schema does not match this version of dependency-check", dex); } } finally { if (conn != null) { @@ -183,8 +184,8 @@ public final class ConnectionFactory { /** * Cleans up resources and unloads any registered database drivers. This needs to be called to ensure the driver is - * unregistered prior to the finalize method being called as during shutdown the class loader used to load the - * driver may be unloaded prior to the driver being de-registered. + * unregistered prior to the finalize method being called as during shutdown the class loader used to load the driver may be + * unloaded prior to the driver being de-registered. */ public static synchronized void cleanup() { if (driver != null) { @@ -229,8 +230,7 @@ public final class ConnectionFactory { */ private static boolean h2DataFileExists() throws IOException { final File dir = Settings.getDataDirectory(); - final String name = Settings.getString(Settings.KEYS.DB_FILE_NAME); - final String fileName = String.format(name, DB_SCHEMA_VERSION); + final String fileName = Settings.getString(Settings.KEYS.DB_FILE_NAME); final File file = new File(dir, fileName); return file.exists(); } @@ -278,6 +278,46 @@ public final class ConnectionFactory { } } + private static void updateSchema(Connection conn, String schema) throws DatabaseException { + LOGGER.debug("Updating database structure"); + InputStream is; + InputStreamReader reader; + BufferedReader in = null; + String updateFile = null; + try { + updateFile = String.format(DB_STRUCTURE_UPDATE_RESOURCE, schema); + is = ConnectionFactory.class.getClassLoader().getResourceAsStream(updateFile); + reader = new InputStreamReader(is, "UTF-8"); + in = new BufferedReader(reader); + final StringBuilder sb = new StringBuilder(2110); + String tmp; + while ((tmp = in.readLine()) != null) { + sb.append(tmp); + } + Statement statement = null; + try { + statement = conn.createStatement(); + statement.execute(sb.toString()); + } catch (SQLException ex) { + LOGGER.debug("", ex); + throw new DatabaseException("Unable to update database schema", ex); + } finally { + DBUtils.closeStatement(statement); + } + } catch (IOException ex) { + final String msg = String.format("Upgrade SQL file does not exist: %s", updateFile); + throw new DatabaseException(msg, ex); + } finally { + if (in != null) { + try { + in.close(); + } catch (IOException ex) { + LOGGER.trace("", ex); + } + } + } + } + /** * Uses the provided connection to check the specified schema version within the database. * @@ -288,12 +328,13 @@ public final class ConnectionFactory { ResultSet rs = null; CallableStatement cs = null; try { + //TODO convert this to use DatabaseProperties cs = conn.prepareCall("SELECT value FROM properties WHERE id = 'version'"); rs = cs.executeQuery(); if (rs.next()) { - final boolean isWrongSchema = !DB_SCHEMA_VERSION.equals(rs.getString(1)); - if (isWrongSchema) { - throw new DatabaseException("Incorrect database schema; unable to continue"); + if (!DB_SCHEMA_VERSION.equals(rs.getString(1))) { + LOGGER.debug("Updating from version: " + rs.getString(1)); + updateSchema(conn, rs.getString(1)); } } else { throw new DatabaseException("Database schema is missing"); diff --git a/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/nvdcve/CveDB.java b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/nvdcve/CveDB.java index 6d4a34f25..c06050160 100644 --- a/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/nvdcve/CveDB.java +++ b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/nvdcve/CveDB.java @@ -19,6 +19,7 @@ package org.owasp.dependencycheck.data.nvdcve; import java.io.IOException; import java.io.UnsupportedEncodingException; +import java.sql.CallableStatement; import java.sql.Connection; import java.sql.PreparedStatement; import java.sql.ResultSet; @@ -33,6 +34,7 @@ import java.util.Map.Entry; import java.util.Properties; import java.util.ResourceBundle; import java.util.Set; +import java.util.logging.Level; import org.owasp.dependencycheck.data.cwe.CweDB; import org.owasp.dependencycheck.dependency.Reference; import org.owasp.dependencycheck.dependency.Vulnerability; @@ -766,8 +768,8 @@ public class CveDB { DependencyVersion cpeVersion; if (cpe.getVersion() != null && !cpe.getVersion().isEmpty()) { String versionText; - if (cpe.getRevision() != null && !cpe.getRevision().isEmpty()) { - versionText = String.format("%s.%s", cpe.getVersion(), cpe.getRevision()); + if (cpe.getUpdate() != null && !cpe.getUpdate().isEmpty()) { + versionText = String.format("%s.%s", cpe.getVersion(), cpe.getUpdate()); } else { versionText = cpe.getVersion(); } @@ -777,4 +779,41 @@ public class CveDB { } return cpeVersion; } + + /** + * Deletes unused dictionary entries from the database. + */ + public void deleteUnusedCpe() { + CallableStatement cs = null; + try { + cs = getConnection().prepareCall(statementBundle.getString("DELETE_UNUSED_DICT_CPE")); + cs.executeUpdate(); + } catch (SQLException ex) { + LOGGER.error("Unable to delete CPE dictionary entries", ex); + } finally { + DBUtils.closeStatement(cs); + } + } + + /** + * Merges CPE entries into the database. + * + * @param cpe the CPE identifier + * @param vendor the CPE vendor + * @param product the CPE product + */ + public void addCpe(String cpe, String vendor, String product) { + PreparedStatement ps = null; + try { + ps = getConnection().prepareCall(statementBundle.getString("ADD_DICT_CPE")); + ps.setString(1, cpe); + ps.setString(2, vendor); + ps.setString(3, product); + ps.executeUpdate(); + } catch (SQLException ex) { + LOGGER.error("Unable to add CPE dictionary entry", ex); + } finally { + DBUtils.closeStatement(ps); + } + } } diff --git a/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/nvdcve/DatabaseProperties.java b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/nvdcve/DatabaseProperties.java index fc74931e3..f3ae9b50c 100644 --- a/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/nvdcve/DatabaseProperties.java +++ b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/nvdcve/DatabaseProperties.java @@ -24,7 +24,7 @@ import java.util.Map; import java.util.Map.Entry; import java.util.Properties; import java.util.TreeMap; -import org.owasp.dependencycheck.data.update.NvdCveInfo; +import org.owasp.dependencycheck.data.update.nvd.NvdCveInfo; import org.owasp.dependencycheck.data.update.exception.UpdateException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -41,20 +41,28 @@ public class DatabaseProperties { */ private static final Logger LOGGER = LoggerFactory.getLogger(DatabaseProperties.class); /** - * Modified key word, used as a key to store information about the modified file (i.e. the containing the last 8 - * days of updates).. + * Modified key word, used as a key to store information about the modified file (i.e. the containing the last 8 days of + * updates).. */ public static final String MODIFIED = "Modified"; /** - * The properties file key for the last updated field - used to store the last updated time of the Modified NVD CVE - * xml file. + * The properties file key for the last updated field - used to store the last updated time of the Modified NVD CVE xml file. */ public static final String LAST_UPDATED = "NVD CVE Modified"; /** - * Stores the last updated time for each of the NVD CVE files. These timestamps should be updated if we process the - * modified file within 7 days of the last update. + * Stores the last updated time for each of the NVD CVE files. These timestamps should be updated if we process the modified + * file within 7 days of the last update. */ public static final String LAST_UPDATED_BASE = "NVD CVE "; + /** + * The key for the last time the CPE data was updated. + */ + public static final String LAST_CPE_UPDATE = "LAST_CPE_UPDATE"; + /** + * The key for the database schema version. + */ + public static final String VERSION = "version"; + /** * A collection of properties about the data. */ @@ -116,8 +124,7 @@ public class DatabaseProperties { } /** - * Returns the property value for the given key. If the key is not contained in the underlying properties null is - * returned. + * Returns the property value for the given key. If the key is not contained in the underlying properties null is returned. * * @param key the property key * @return the value of the property @@ -127,8 +134,8 @@ public class DatabaseProperties { } /** - * Returns the property value for the given key. If the key is not contained in the underlying properties the - * default value is returned. + * Returns the property value for the given key. If the key is not contained in the underlying properties the default value is + * returned. * * @param key the property key * @param defaultValue the default value @@ -148,8 +155,8 @@ public class DatabaseProperties { } /** - * Returns a map of the meta data from the database properties. This primarily contains timestamps of when the NVD - * CVE information was last updated. + * Returns a map of the meta data from the database properties. This primarily contains timestamps of when the NVD CVE + * information was last updated. * * @return a map of the database meta data */ diff --git a/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/BaseUpdater.java b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/BaseUpdater.java new file mode 100644 index 000000000..e054e9303 --- /dev/null +++ b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/BaseUpdater.java @@ -0,0 +1,86 @@ +/* + * This file is part of dependency-check-core. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * Copyright (c) 2015 Jeremy Long. All Rights Reserved. + */ +package org.owasp.dependencycheck.data.update; + +import org.owasp.dependencycheck.data.nvdcve.CveDB; +import org.owasp.dependencycheck.data.nvdcve.DatabaseException; +import org.owasp.dependencycheck.data.nvdcve.DatabaseProperties; +import org.owasp.dependencycheck.data.update.exception.UpdateException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * + * @author Jeremy Long + */ +public abstract class BaseUpdater { + + /** + * Static logger. + */ + private static final Logger LOGGER = LoggerFactory.getLogger(BaseUpdater.class); + /** + * Information about the timestamps and URLs for data that needs to be updated. + */ + private DatabaseProperties properties; + /** + * Reference to the Cve Database. + */ + private CveDB cveDB = null; + + protected CveDB getCveDB() { + return cveDB; + } + + protected DatabaseProperties getProperties() { + return properties; + } + + /** + * Closes the CVE and CPE data stores. + */ + protected void closeDataStores() { + if (cveDB != null) { + try { + cveDB.close(); + } catch (Throwable ignore) { + LOGGER.trace("Error closing the database", ignore); + } + } + } + + /** + * Opens the data store. + * + * @throws UpdateException thrown if a data store cannot be opened + */ + protected final void openDataStores() throws UpdateException { + if (cveDB != null) { + return; + } + try { + cveDB = new CveDB(); + cveDB.open(); + } catch (DatabaseException ex) { + closeDataStores(); + LOGGER.debug("Database Exception opening databases", ex); + throw new UpdateException("Error updating the database, please see the log file for more details."); + } + properties = cveDB.getDatabaseProperties(); + } +} diff --git a/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/CpeUpdater.java b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/CpeUpdater.java new file mode 100644 index 000000000..386b5afdd --- /dev/null +++ b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/CpeUpdater.java @@ -0,0 +1,196 @@ +/* + * This file is part of dependency-check-core. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * Copyright (c) 2015 Jeremy Long. All Rights Reserved. + */ +package org.owasp.dependencycheck.data.update; + +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.io.IOException; +import java.net.MalformedURLException; +import java.net.URL; +import java.util.Date; +import java.util.List; +import java.util.logging.Level; +import java.util.zip.GZIPInputStream; +import javax.xml.parsers.ParserConfigurationException; +import javax.xml.parsers.SAXParser; +import javax.xml.parsers.SAXParserFactory; +import org.apache.commons.io.FileUtils; +import static org.owasp.dependencycheck.data.nvdcve.DatabaseProperties.LAST_CPE_UPDATE; +import org.owasp.dependencycheck.data.update.cpe.CPEHandler; +import org.owasp.dependencycheck.data.update.cpe.Cpe; +import org.owasp.dependencycheck.data.update.exception.UpdateException; +import org.owasp.dependencycheck.utils.DateUtil; +import org.owasp.dependencycheck.utils.DownloadFailedException; +import org.owasp.dependencycheck.utils.Downloader; +import org.owasp.dependencycheck.utils.Settings; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.xml.sax.SAXException; + +/** + * + * @author Jeremy Long + */ +public class CpeUpdater extends BaseUpdater implements CachedWebDataSource { + + /** + * Static logger. + */ + private static final Logger LOGGER = LoggerFactory.getLogger(CpeUpdater.class); + + @Override + public void update() throws UpdateException { + try { + openDataStores(); + if (updateNeeded()) { + LOGGER.info("Updating the Common Platform Enumeration (CPE)"); + final File xml = downloadCpe(); + List cpes = processXML(xml); + getCveDB().deleteUnusedCpe(); + for (Cpe cpe : cpes) { + getCveDB().addCpe(cpe.getValue(), cpe.getVendor(), cpe.getProduct()); + } + final Date now = new Date(); + getProperties().save(LAST_CPE_UPDATE, Long.toString(now.getTime())); + LOGGER.info("CPE update complete"); + } + } finally { + closeDataStores(); + } + } + + /** + * Downloads the CPE XML file. + * + * @return the file reference to the CPE.xml file + * @throws UpdateException thrown if there is an issue downloading the XML file + */ + private File downloadCpe() throws UpdateException { + File xml; + final URL url; + try { + url = new URL(Settings.getString(Settings.KEYS.CPE_URL)); + xml = File.createTempFile("cpe", ".xml", Settings.getTempDirectory()); + Downloader.fetchFile(url, xml); + if (url.toExternalForm().endsWith(".xml.gz")) { + extractGzip(xml); + } + + } catch (MalformedURLException ex) { + throw new UpdateException("Invalid CPE URL", ex); + } catch (DownloadFailedException ex) { + throw new UpdateException("Unable to download CPE XML file", ex); + } catch (IOException ex) { + throw new UpdateException("Unable to create temporary file to download CPE", ex); + } + return xml; + } + + /** + * Parses the CPE XML file to return a list of CPE entries. + * + * @param xml the CPE data file + * @return the list of CPE entries + * @throws UpdateException thrown if there is an issue with parsing the XML file + */ + private List processXML(final File xml) throws UpdateException { + try { + final SAXParserFactory factory = SAXParserFactory.newInstance(); + final SAXParser saxParser = factory.newSAXParser(); + CPEHandler handler = new CPEHandler(); + saxParser.parse(xml, handler); + return handler.getData(); + } catch (ParserConfigurationException ex) { + throw new UpdateException("Unable to parse CPE XML file due to SAX Parser Issue", ex); + } catch (SAXException ex) { + throw new UpdateException("Unable to parse CPE XML file due to SAX Parser Exception", ex); + } catch (IOException ex) { + throw new UpdateException("Unable to parse CPE XML file due to IO Failure", ex); + } + } + + /** + * Checks to find the last time the CPE data was refreshed and if it needs to be updated. + * + * @return true if the CPE data should be refreshed + */ + private boolean updateNeeded() { + final Date now = new Date(); + final int days = Settings.getInt(Settings.KEYS.CVE_MODIFIED_VALID_FOR_DAYS, 30); + long timestamp = 0; + String ts = getProperties().getProperty(LAST_CPE_UPDATE); + if (ts != null && ts.matches("^[0-9]+$")) { + timestamp = Long.parseLong(ts); + } + return !DateUtil.withinDateRange(timestamp, now.getTime(), days); + } + + /** + * Extracts the file contained in a gzip archive. The extracted file is placed in the exact same path as the file specified. + * + * @param file the archive file + * @throws FileNotFoundException thrown if the file does not exist + * @throws IOException thrown if there is an error extracting the file. + */ + private void extractGzip(File file) throws FileNotFoundException, IOException { + //TODO - move this to a util class as it is duplicative of (copy of) code in the DownloadTask + final String originalPath = file.getPath(); + final File gzip = new File(originalPath + ".gz"); + if (gzip.isFile() && !gzip.delete()) { + gzip.deleteOnExit(); + } + if (!file.renameTo(gzip)) { + throw new IOException("Unable to rename '" + file.getPath() + "'"); + } + final File newfile = new File(originalPath); + + final byte[] buffer = new byte[4096]; + + GZIPInputStream cin = null; + FileOutputStream out = null; + try { + cin = new GZIPInputStream(new FileInputStream(gzip)); + out = new FileOutputStream(newfile); + + int len; + while ((len = cin.read(buffer)) > 0) { + out.write(buffer, 0, len); + } + } finally { + if (cin != null) { + try { + cin.close(); + } catch (IOException ex) { + LOGGER.trace("ignore", ex); + } + } + if (out != null) { + try { + out.close(); + } catch (IOException ex) { + LOGGER.trace("ignore", ex); + } + } + if (gzip.isFile()) { + FileUtils.deleteQuietly(gzip); + } + } + } +} diff --git a/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/NvdCveUpdater.java b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/NvdCveUpdater.java index a6b602fd8..94d03ca89 100644 --- a/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/NvdCveUpdater.java +++ b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/NvdCveUpdater.java @@ -18,23 +18,44 @@ package org.owasp.dependencycheck.data.update; import java.net.MalformedURLException; +import java.util.Calendar; +import java.util.Date; +import java.util.HashSet; +import java.util.Set; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import org.owasp.dependencycheck.data.nvdcve.DatabaseProperties; +import static org.owasp.dependencycheck.data.nvdcve.DatabaseProperties.MODIFIED; +import org.owasp.dependencycheck.data.update.exception.InvalidDataException; import org.owasp.dependencycheck.data.update.exception.UpdateException; +import org.owasp.dependencycheck.data.update.nvd.DownloadTask; +import org.owasp.dependencycheck.data.update.nvd.NvdCveInfo; +import org.owasp.dependencycheck.data.update.nvd.ProcessTask; +import org.owasp.dependencycheck.data.update.nvd.UpdateableNvdCve; +import org.owasp.dependencycheck.utils.DateUtil; import org.owasp.dependencycheck.utils.DownloadFailedException; +import org.owasp.dependencycheck.utils.InvalidSettingException; import org.owasp.dependencycheck.utils.Settings; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** - * Class responsible for updating the NVD CVE and CPE data stores. + * Class responsible for updating the NVD CVE data. * * @author Jeremy Long */ -public class NvdCveUpdater implements CachedWebDataSource { +public class NvdCveUpdater extends BaseUpdater implements CachedWebDataSource { /** * The logger */ private static final Logger LOGGER = LoggerFactory.getLogger(NvdCveUpdater.class); + /** + * The max thread pool size to use when downloading files. + */ + public static final int MAX_THREAD_POOL_SIZE = Settings.getInt(Settings.KEYS.MAX_DOWNLOAD_THREAD_POOL_SIZE, 3); /** *

@@ -45,9 +66,10 @@ public class NvdCveUpdater implements CachedWebDataSource { @Override public void update() throws UpdateException { try { - final StandardUpdate task = new StandardUpdate(); - if (task.isUpdateNeeded()) { - task.update(); + openDataStores(); + UpdateableNvdCve updateable = getUpdatesNeeded(); + if (updateable.isUpdateNeeded()) { + performUpdate(updateable); } } catch (MalformedURLException ex) { LOGGER.warn( @@ -61,6 +83,202 @@ public class NvdCveUpdater implements CachedWebDataSource { "If you are behind a proxy you may need to configure dependency-check to use the proxy."); } LOGGER.debug("", ex); + } finally { + closeDataStores(); } } + + /** + * Downloads the latest NVD CVE XML file from the web and imports it into the current CVE Database. + * + * @param updateable a collection of NVD CVE data file references that need to be downloaded and processed to update the + * database + * @throws UpdateException is thrown if there is an error updating the database + */ + public void performUpdate(UpdateableNvdCve updateable) throws UpdateException { + int maxUpdates = 0; + try { + for (NvdCveInfo cve : updateable) { + if (cve.getNeedsUpdate()) { + maxUpdates += 1; + } + } + if (maxUpdates <= 0) { + return; + } + if (maxUpdates > 3) { + LOGGER.info( + "NVD CVE requires several updates; this could take a couple of minutes."); + } + if (maxUpdates > 0) { + openDataStores(); + } + + final int poolSize = (MAX_THREAD_POOL_SIZE < maxUpdates) ? MAX_THREAD_POOL_SIZE : maxUpdates; + + final ExecutorService downloadExecutors = Executors.newFixedThreadPool(poolSize); + final ExecutorService processExecutor = Executors.newSingleThreadExecutor(); + final Set>> downloadFutures = new HashSet>>(maxUpdates); + for (NvdCveInfo cve : updateable) { + if (cve.getNeedsUpdate()) { + final DownloadTask call = new DownloadTask(cve, processExecutor, getCveDB(), Settings.getInstance()); + downloadFutures.add(downloadExecutors.submit(call)); + } + } + downloadExecutors.shutdown(); + + //next, move the future future processTasks to just future processTasks + final Set> processFutures = new HashSet>(maxUpdates); + for (Future> future : downloadFutures) { + Future task = null; + try { + task = future.get(); + } catch (InterruptedException ex) { + downloadExecutors.shutdownNow(); + processExecutor.shutdownNow(); + + LOGGER.debug("Thread was interrupted during download", ex); + throw new UpdateException("The download was interrupted", ex); + } catch (ExecutionException ex) { + downloadExecutors.shutdownNow(); + processExecutor.shutdownNow(); + + LOGGER.debug("Thread was interrupted during download execution", ex); + throw new UpdateException("The execution of the download was interrupted", ex); + } + if (task == null) { + downloadExecutors.shutdownNow(); + processExecutor.shutdownNow(); + LOGGER.debug("Thread was interrupted during download"); + throw new UpdateException("The download was interrupted; unable to complete the update"); + } else { + processFutures.add(task); + } + } + + for (Future future : processFutures) { + try { + final ProcessTask task = future.get(); + if (task.getException() != null) { + throw task.getException(); + } + } catch (InterruptedException ex) { + processExecutor.shutdownNow(); + LOGGER.debug("Thread was interrupted during processing", ex); + throw new UpdateException(ex); + } catch (ExecutionException ex) { + processExecutor.shutdownNow(); + LOGGER.debug("Execution Exception during process", ex); + throw new UpdateException(ex); + } finally { + processExecutor.shutdown(); + } + } + + if (maxUpdates >= 1) { //ensure the modified file date gets written (we may not have actually updated it) + getProperties().save(updateable.get(MODIFIED)); + LOGGER.info("Begin database maintenance."); + getCveDB().cleanupDatabase(); + LOGGER.info("End database maintenance."); + } + } finally { + closeDataStores(); + } + } + + /** + * Determines if the index needs to be updated. This is done by fetching the NVD CVE meta data and checking the last update + * date. If the data needs to be refreshed this method will return the NvdCveUrl for the files that need to be updated. + * + * @return the collection of files that need to be updated + * @throws MalformedURLException is thrown if the URL for the NVD CVE Meta data is incorrect + * @throws DownloadFailedException is thrown if there is an error. downloading the NVD CVE download data file + * @throws UpdateException Is thrown if there is an issue with the last updated properties file + */ + protected final UpdateableNvdCve getUpdatesNeeded() throws MalformedURLException, DownloadFailedException, UpdateException { + UpdateableNvdCve updates = null; + try { + updates = retrieveCurrentTimestampsFromWeb(); + } catch (InvalidDataException ex) { + final String msg = "Unable to retrieve valid timestamp from nvd cve downloads page"; + LOGGER.debug(msg, ex); + throw new DownloadFailedException(msg, ex); + } catch (InvalidSettingException ex) { + LOGGER.debug("Invalid setting found when retrieving timestamps", ex); + throw new DownloadFailedException("Invalid settings", ex); + } + + if (updates == null) { + throw new DownloadFailedException("Unable to retrieve the timestamps of the currently published NVD CVE data"); + } + if (!getProperties().isEmpty()) { + try { + final long lastUpdated = Long.parseLong(getProperties().getProperty(DatabaseProperties.LAST_UPDATED, "0")); + final Date now = new Date(); + final int days = Settings.getInt(Settings.KEYS.CVE_MODIFIED_VALID_FOR_DAYS, 7); + if (lastUpdated == updates.getTimeStamp(MODIFIED)) { + updates.clear(); //we don't need to update anything. + } else if (DateUtil.withinDateRange(lastUpdated, now.getTime(), days)) { + for (NvdCveInfo entry : updates) { + if (MODIFIED.equals(entry.getId())) { + entry.setNeedsUpdate(true); + } else { + entry.setNeedsUpdate(false); + } + } + } else { //we figure out which of the several XML files need to be downloaded. + for (NvdCveInfo entry : updates) { + if (MODIFIED.equals(entry.getId())) { + entry.setNeedsUpdate(true); + } else { + long currentTimestamp = 0; + try { + currentTimestamp = Long.parseLong(getProperties().getProperty(DatabaseProperties.LAST_UPDATED_BASE + entry.getId(), "0")); + } catch (NumberFormatException ex) { + LOGGER.debug("Error parsing '{}' '{}' from nvdcve.lastupdated", + DatabaseProperties.LAST_UPDATED_BASE, entry.getId(), ex); + } + if (currentTimestamp == entry.getTimestamp()) { + entry.setNeedsUpdate(false); + } + } + } + } + } catch (NumberFormatException ex) { + LOGGER.warn("An invalid schema version or timestamp exists in the data.properties file."); + LOGGER.debug("", ex); + } + } + return updates; + } + + /** + * Retrieves the timestamps from the NVD CVE meta data file. + * + * @return the timestamp from the currently published nvdcve downloads page + * @throws MalformedURLException thrown if the URL for the NVD CCE Meta data is incorrect. + * @throws DownloadFailedException thrown if there is an error downloading the nvd cve meta data file + * @throws InvalidDataException thrown if there is an exception parsing the timestamps + * @throws InvalidSettingException thrown if the settings are invalid + */ + private UpdateableNvdCve retrieveCurrentTimestampsFromWeb() + throws MalformedURLException, DownloadFailedException, InvalidDataException, InvalidSettingException { + + final UpdateableNvdCve updates = new UpdateableNvdCve(); + updates.add(MODIFIED, Settings.getString(Settings.KEYS.CVE_MODIFIED_20_URL), + Settings.getString(Settings.KEYS.CVE_MODIFIED_12_URL), + false); + + final int start = Settings.getInt(Settings.KEYS.CVE_START_YEAR); + final int end = Calendar.getInstance().get(Calendar.YEAR); + final String baseUrl20 = Settings.getString(Settings.KEYS.CVE_SCHEMA_2_0); + final String baseUrl12 = Settings.getString(Settings.KEYS.CVE_SCHEMA_1_2); + for (int i = start; i <= end; i++) { + updates.add(Integer.toString(i), String.format(baseUrl20, i), + String.format(baseUrl12, i), + true); + } + return updates; + } + } diff --git a/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/StandardUpdate.java b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/StandardUpdate.java deleted file mode 100644 index d5f1c2c22..000000000 --- a/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/StandardUpdate.java +++ /dev/null @@ -1,318 +0,0 @@ -/* - * This file is part of dependency-check-core. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * Copyright (c) 2012 Jeremy Long. All Rights Reserved. - */ -package org.owasp.dependencycheck.data.update; - -import java.net.MalformedURLException; -import java.util.Calendar; -import java.util.Date; -import java.util.HashSet; -import java.util.Set; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.Future; -import org.owasp.dependencycheck.data.nvdcve.CveDB; -import org.owasp.dependencycheck.data.nvdcve.DatabaseException; -import org.owasp.dependencycheck.data.nvdcve.DatabaseProperties; -import static org.owasp.dependencycheck.data.nvdcve.DatabaseProperties.MODIFIED; -import org.owasp.dependencycheck.data.update.exception.InvalidDataException; -import org.owasp.dependencycheck.data.update.exception.UpdateException; -import org.owasp.dependencycheck.data.update.task.DownloadTask; -import org.owasp.dependencycheck.data.update.task.ProcessTask; -import org.owasp.dependencycheck.utils.DateUtil; -import org.owasp.dependencycheck.utils.DownloadFailedException; -import org.owasp.dependencycheck.utils.InvalidSettingException; -import org.owasp.dependencycheck.utils.Settings; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * Class responsible for updating the NVDCVE data store. - * - * @author Jeremy Long - */ -public class StandardUpdate { - - /** - * Static logger. - */ - private static final Logger LOGGER = LoggerFactory.getLogger(StandardUpdate.class); - /** - * The max thread pool size to use when downloading files. - */ - public static final int MAX_THREAD_POOL_SIZE = Settings.getInt(Settings.KEYS.MAX_DOWNLOAD_THREAD_POOL_SIZE, 3); - /** - * Information about the timestamps and URLs for data that needs to be updated. - */ - private DatabaseProperties properties; - /** - * A collection of updateable NVD CVE items. - */ - private UpdateableNvdCve updateable; - /** - * Reference to the Cve Database. - */ - private CveDB cveDB = null; - - /** - * Gets whether or not an update is needed. - * - * @return true or false depending on whether an update is needed - */ - public boolean isUpdateNeeded() { - return updateable.isUpdateNeeded(); - } - - /** - * Constructs a new Standard Update Task. - * - * @throws MalformedURLException thrown if a configured URL is malformed - * @throws DownloadFailedException thrown if a timestamp cannot be checked on a configured URL - * @throws UpdateException thrown if there is an exception generating the update task - */ - public StandardUpdate() throws MalformedURLException, DownloadFailedException, UpdateException { - openDataStores(); - properties = cveDB.getDatabaseProperties(); - updateable = updatesNeeded(); - } - - /** - *

- * Downloads the latest NVD CVE XML file from the web and imports it into the current CVE Database.

- * - * @throws UpdateException is thrown if there is an error updating the database - */ - public void update() throws UpdateException { - int maxUpdates = 0; - try { - for (NvdCveInfo cve : updateable) { - if (cve.getNeedsUpdate()) { - maxUpdates += 1; - } - } - if (maxUpdates <= 0) { - return; - } - if (maxUpdates > 3) { - LOGGER.info( - "NVD CVE requires several updates; this could take a couple of minutes."); - } - if (maxUpdates > 0) { - openDataStores(); - } - - final int poolSize = (MAX_THREAD_POOL_SIZE < maxUpdates) ? MAX_THREAD_POOL_SIZE : maxUpdates; - - final ExecutorService downloadExecutors = Executors.newFixedThreadPool(poolSize); - final ExecutorService processExecutor = Executors.newSingleThreadExecutor(); - final Set>> downloadFutures = new HashSet>>(maxUpdates); - for (NvdCveInfo cve : updateable) { - if (cve.getNeedsUpdate()) { - final DownloadTask call = new DownloadTask(cve, processExecutor, cveDB, Settings.getInstance()); - downloadFutures.add(downloadExecutors.submit(call)); - } - } - downloadExecutors.shutdown(); - - //next, move the future future processTasks to just future processTasks - final Set> processFutures = new HashSet>(maxUpdates); - for (Future> future : downloadFutures) { - Future task = null; - try { - task = future.get(); - } catch (InterruptedException ex) { - downloadExecutors.shutdownNow(); - processExecutor.shutdownNow(); - - LOGGER.debug("Thread was interrupted during download", ex); - throw new UpdateException("The download was interrupted", ex); - } catch (ExecutionException ex) { - downloadExecutors.shutdownNow(); - processExecutor.shutdownNow(); - - LOGGER.debug("Thread was interrupted during download execution", ex); - throw new UpdateException("The execution of the download was interrupted", ex); - } - if (task == null) { - downloadExecutors.shutdownNow(); - processExecutor.shutdownNow(); - LOGGER.debug("Thread was interrupted during download"); - throw new UpdateException("The download was interrupted; unable to complete the update"); - } else { - processFutures.add(task); - } - } - - for (Future future : processFutures) { - try { - final ProcessTask task = future.get(); - if (task.getException() != null) { - throw task.getException(); - } - } catch (InterruptedException ex) { - processExecutor.shutdownNow(); - LOGGER.debug("Thread was interrupted during processing", ex); - throw new UpdateException(ex); - } catch (ExecutionException ex) { - processExecutor.shutdownNow(); - LOGGER.debug("Execution Exception during process", ex); - throw new UpdateException(ex); - } finally { - processExecutor.shutdown(); - } - } - - if (maxUpdates >= 1) { //ensure the modified file date gets written (we may not have actually updated it) - properties.save(updateable.get(MODIFIED)); - LOGGER.info("Begin database maintenance."); - cveDB.cleanupDatabase(); - LOGGER.info("End database maintenance."); - } - } finally { - closeDataStores(); - } - } - - /** - * Determines if the index needs to be updated. This is done by fetching the NVD CVE meta data and checking the last update - * date. If the data needs to be refreshed this method will return the NvdCveUrl for the files that need to be updated. - * - * @return the collection of files that need to be updated - * @throws MalformedURLException is thrown if the URL for the NVD CVE Meta data is incorrect - * @throws DownloadFailedException is thrown if there is an error. downloading the NVD CVE download data file - * @throws UpdateException Is thrown if there is an issue with the last updated properties file - */ - protected final UpdateableNvdCve updatesNeeded() throws MalformedURLException, DownloadFailedException, UpdateException { - UpdateableNvdCve updates = null; - try { - updates = retrieveCurrentTimestampsFromWeb(); - } catch (InvalidDataException ex) { - final String msg = "Unable to retrieve valid timestamp from nvd cve downloads page"; - LOGGER.debug(msg, ex); - throw new DownloadFailedException(msg, ex); - } catch (InvalidSettingException ex) { - LOGGER.debug("Invalid setting found when retrieving timestamps", ex); - throw new DownloadFailedException("Invalid settings", ex); - } - - if (updates == null) { - throw new DownloadFailedException("Unable to retrieve the timestamps of the currently published NVD CVE data"); - } - if (!properties.isEmpty()) { - try { - final long lastUpdated = Long.parseLong(properties.getProperty(DatabaseProperties.LAST_UPDATED, "0")); - final Date now = new Date(); - final int days = Settings.getInt(Settings.KEYS.CVE_MODIFIED_VALID_FOR_DAYS, 7); - if (lastUpdated == updates.getTimeStamp(MODIFIED)) { - updates.clear(); //we don't need to update anything. - } else if (DateUtil.withinDateRange(lastUpdated, now.getTime(), days)) { - for (NvdCveInfo entry : updates) { - if (MODIFIED.equals(entry.getId())) { - entry.setNeedsUpdate(true); - } else { - entry.setNeedsUpdate(false); - } - } - } else { //we figure out which of the several XML files need to be downloaded. - for (NvdCveInfo entry : updates) { - if (MODIFIED.equals(entry.getId())) { - entry.setNeedsUpdate(true); - } else { - long currentTimestamp = 0; - try { - currentTimestamp = Long.parseLong(properties.getProperty(DatabaseProperties.LAST_UPDATED_BASE + entry.getId(), "0")); - } catch (NumberFormatException ex) { - LOGGER.debug("Error parsing '{}' '{}' from nvdcve.lastupdated", - DatabaseProperties.LAST_UPDATED_BASE, entry.getId(), ex); - } - if (currentTimestamp == entry.getTimestamp()) { - entry.setNeedsUpdate(false); - } - } - } - } - } catch (NumberFormatException ex) { - LOGGER.warn("An invalid schema version or timestamp exists in the data.properties file."); - LOGGER.debug("", ex); - } - } - return updates; - } - - /** - * Retrieves the timestamps from the NVD CVE meta data file. - * - * @return the timestamp from the currently published nvdcve downloads page - * @throws MalformedURLException thrown if the URL for the NVD CCE Meta data is incorrect. - * @throws DownloadFailedException thrown if there is an error downloading the nvd cve meta data file - * @throws InvalidDataException thrown if there is an exception parsing the timestamps - * @throws InvalidSettingException thrown if the settings are invalid - */ - private UpdateableNvdCve retrieveCurrentTimestampsFromWeb() - throws MalformedURLException, DownloadFailedException, InvalidDataException, InvalidSettingException { - - final UpdateableNvdCve updates = new UpdateableNvdCve(); - updates.add(MODIFIED, Settings.getString(Settings.KEYS.CVE_MODIFIED_20_URL), - Settings.getString(Settings.KEYS.CVE_MODIFIED_12_URL), - false); - - final int start = Settings.getInt(Settings.KEYS.CVE_START_YEAR); - final int end = Calendar.getInstance().get(Calendar.YEAR); - final String baseUrl20 = Settings.getString(Settings.KEYS.CVE_SCHEMA_2_0); - final String baseUrl12 = Settings.getString(Settings.KEYS.CVE_SCHEMA_1_2); - for (int i = start; i <= end; i++) { - updates.add(Integer.toString(i), String.format(baseUrl20, i), - String.format(baseUrl12, i), - true); - } - - return updates; - } - - /** - * Closes the CVE and CPE data stores. - */ - protected void closeDataStores() { - if (cveDB != null) { - try { - cveDB.close(); - } catch (Throwable ignore) { - LOGGER.trace("Error closing the cveDB", ignore); - } - } - } - - /** - * Opens the CVE and CPE data stores. - * - * @throws UpdateException thrown if a data store cannot be opened - */ - protected final void openDataStores() throws UpdateException { - if (cveDB != null) { - return; - } - try { - cveDB = new CveDB(); - cveDB.open(); - } catch (DatabaseException ex) { - closeDataStores(); - LOGGER.debug("Database Exception opening databases", ex); - throw new UpdateException("Error updating the CPE/CVE data, please see the log file for more details."); - } - } -} diff --git a/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/UpdateService.java b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/UpdateService.java index 8720b3539..96fbda0d1 100644 --- a/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/UpdateService.java +++ b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/UpdateService.java @@ -19,6 +19,7 @@ package org.owasp.dependencycheck.data.update; import java.util.Iterator; import java.util.ServiceLoader; +import org.owasp.dependencycheck.data.update.CachedWebDataSource; /** * The CachedWebDataSource Service Loader. This class loads all services that implement diff --git a/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/cpe/CPEHandler.java b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/cpe/CPEHandler.java new file mode 100644 index 000000000..2f7461881 --- /dev/null +++ b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/cpe/CPEHandler.java @@ -0,0 +1,362 @@ +/* + * This file is part of dependency-check-core. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * Copyright (c) 2015 Jeremy Long. All Rights Reserved. + */ +package org.owasp.dependencycheck.data.update.cpe; + +import java.io.UnsupportedEncodingException; +import java.net.URLDecoder; +import java.util.ArrayList; +import java.util.List; +import org.owasp.dependencycheck.data.update.NvdCveUpdater; +import org.owasp.dependencycheck.data.update.exception.InvalidDataException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.xml.sax.Attributes; +import org.xml.sax.SAXException; +import org.xml.sax.helpers.DefaultHandler; + +/** + * A SAX Handler that will parse the CPE XML and load it into the databse. + * + * @author Jeremy Long + */ +public class CPEHandler extends DefaultHandler { + + /** + * The current CPE schema. + */ + private static final String CURRENT_SCHEMA_VERSION = "2.3"; + /** + * The text content of the node being processed. This can be used during the end element event. + */ + StringBuilder nodeText = null; + /** + * A reference to the current element. + */ + Element current = new Element(); + /** + * The logger. + */ + private static final Logger LOGGER = LoggerFactory.getLogger(NvdCveUpdater.class); + /** + * The list of CPE values. + */ + List data = new ArrayList(); + + /** + * Returns the list of CPE values. + * + * @return the list of CPE values + */ + public List getData() { + return data; + } + + /** + * Handles the start element event + * + * @param uri the elements uri + * @param localName the local name + * @param qName the qualified name + * @param attributes the attributes + * @throws SAXException thrown if there is an exception processing the element + */ + @Override + public void startElement(String uri, String localName, String qName, Attributes attributes) throws SAXException { + nodeText = null; + current.setNode(qName); + if (current.isCpeItemNode()) { + String temp = attributes.getValue("deprecated"); + String value = attributes.getValue("name"); + boolean delete = (temp != null && temp.equalsIgnoreCase("true")); + if (!delete && value.startsWith("cpe:/a:") && value.length() > 7) { + try { + Cpe cpe = new Cpe(value); + data.add(cpe); + } catch (UnsupportedEncodingException ex) { + LOGGER.debug("Unable to parse the CPE", ex); + } catch (InvalidDataException ex) { + LOGGER.debug("CPE is not the correct format", ex); + } + } + } else if (current.isSchemaVersionNode()) { + nodeText = new StringBuilder(3); + } +// } else if (current.isTitleNode()) { +// //do nothing +// } else if (current.isMetaNode()) { +// //do nothing +// } else if (current.isTimestampNode()) { +// //do nothing +// } else if (current.isCpeListNode()) { +// //do nothing +// } else if (current.isNotesNode()) { +// //do nothing +// } else if (current.isNoteNode()) { +// //do nothing +// } else if (current.isCheckNode()) { +// //do nothing +// } else if (current.isGeneratorNode()) { +// //do nothing +// } else if (current.isProductNameNode()) { +// //do nothing +// } else if (current.isProductVersionNode()) { +// //do nothing + } + + /** + * Reads the characters in the current node. + * + * @param ch the char array + * @param start the start position of the data read + * @param length the length of the data read + * @throws SAXException thrown if there is an exception processing the characters + */ + @Override + public void characters(char[] ch, int start, int length) throws SAXException { + if (nodeText != null) { + nodeText.append(ch, start, length); + } + } + + /** + * Handles the end element event. Stores the CPE data in the Cve Database if the cpe item node is ending. + * + * @param uri the element's uri + * @param localName the local name + * @param qName the qualified name + * @throws SAXException thrown if there is an exception processing the element + */ + @Override + public void endElement(String uri, String localName, String qName) throws SAXException { + current.setNode(qName); + if (current.isSchemaVersionNode() && !CURRENT_SCHEMA_VERSION.equals(nodeText.toString())) { + throw new SAXException("ERROR: Unexpecgted CPE Schema Version, expected: " + + CURRENT_SCHEMA_VERSION + ", file is: " + nodeText); + + } +// } else if (current.isCpeItemNode()) { +// //do nothing +// } else if (current.isTitleNode()) { +// //do nothing +// } else if (current.isCpeListNode()) { +// //do nothing +// } else if (current.isMetaNode()) { +// //do nothing +// } else if (current.isNotesNode()) { +// //do nothing +// } else if (current.isNoteNode()) { +// //do nothing +// } else if (current.isCheckNode()) { +// //do nothing +// } else if (current.isGeneratorNode()) { +// //do nothing +// } else if (current.isProductNameNode()) { +// //do nothing +// } else if (current.isProductVersionNode()) { +// //do nothing +// else if (current.isTimestampNode()) { +// //do nothing +// } else { +// throw new SAXException("ERROR STATE: Unexpected qName '" + qName + "'"); +// } + } + + // + /** + * A simple class to maintain information about the current element while parsing the CPE XML. + */ + protected class Element { + + /** + * A node type in the CPE Schema 2.2 + */ + public static final String CPE_LIST = "cpe-list"; + /** + * A node type in the CPE Schema 2.2 + */ + public static final String CPE_ITEM = "cpe-item"; + /** + * A node type in the CPE Schema 2.2 + */ + public static final String TITLE = "title"; + /** + * A node type in the CPE Schema 2.2 + */ + public static final String NOTES = "notes"; + /** + * A node type in the CPE Schema 2.2 + */ + public static final String NOTE = "note"; + /** + * A node type in the CPE Schema 2.2 + */ + public static final String CHECK = "check"; + /** + * A node type in the CPE Schema 2.2 + */ + public static final String META = "meta:item-metadata"; + /** + * A node type in the CPE Schema 2.2 + */ + public static final String GENERATOR = "generator"; + /** + * A node type in the CPE Schema 2.2 + */ + public static final String PRODUCT_NAME = "product_name"; + /** + * A node type in the CPE Schema 2.2 + */ + public static final String PRODUCT_VERSION = "product_version"; + /** + * A node type in the CPE Schema 2.2 + */ + public static final String SCHEMA_VERSION = "schema_version"; + /** + * A node type in the CPE Schema 2.2 + */ + public static final String TIMESTAMP = "timestamp"; + private String node = null; + + /** + * Gets the value of node + * + * @return the value of node + */ + public String getNode() { + return this.node; + } + + /** + * Sets the value of node + * + * @param node new value of node + */ + public void setNode(String node) { + this.node = node; + } + + /** + * Checks if the handler is at the CPE_LIST node + * + * @return true or false + */ + public boolean isCpeListNode() { + return CPE_LIST.equals(node); + } + + /** + * Checks if the handler is at the CPE_ITEM node + * + * @return true or false + */ + public boolean isCpeItemNode() { + return CPE_ITEM.equals(node); + } + + /** + * Checks if the handler is at the TITLE node + * + * @return true or false + */ + public boolean isTitleNode() { + return TITLE.equals(node); + } + + /** + * Checks if the handler is at the NOTES node + * + * @return true or false + */ + public boolean isNotesNode() { + return NOTES.equals(node); + } + + /** + * Checks if the handler is at the NOTE node + * + * @return true or false + */ + public boolean isNoteNode() { + return NOTE.equals(node); + } + + /** + * Checks if the handler is at the CHECK node + * + * @return true or false + */ + public boolean isCheckNode() { + return CHECK.equals(node); + } + + /** + * Checks if the handler is at the META node + * + * @return true or false + */ + public boolean isMetaNode() { + return META.equals(node); + } + + /** + * Checks if the handler is at the GENERATOR node + * + * @return true or false + */ + public boolean isGeneratorNode() { + return GENERATOR.equals(node); + } + + /** + * Checks if the handler is at the PRODUCT_NAME node + * + * @return true or false + */ + public boolean isProductNameNode() { + return PRODUCT_NAME.equals(node); + } + + /** + * Checks if the handler is at the PRODUCT_VERSION node + * + * @return true or false + */ + public boolean isProductVersionNode() { + return PRODUCT_VERSION.equals(node); + } + + /** + * Checks if the handler is at the SCHEMA_VERSION node + * + * @return true or false + */ + public boolean isSchemaVersionNode() { + return SCHEMA_VERSION.equals(node); + } + + /** + * Checks if the handler is at the TIMESTAMP node + * + * @return true or false + */ + public boolean isTimestampNode() { + return TIMESTAMP.equals(node); + } + } + // +} diff --git a/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/cpe/Cpe.java b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/cpe/Cpe.java new file mode 100644 index 000000000..4309d2914 --- /dev/null +++ b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/cpe/Cpe.java @@ -0,0 +1,114 @@ +/* + * This file is part of dependency-check-core. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * Copyright (c) 2015 Jeremy Long. All Rights Reserved. + */ +package org.owasp.dependencycheck.data.update.cpe; + +import java.io.UnsupportedEncodingException; +import java.net.URLDecoder; +import org.owasp.dependencycheck.data.update.exception.InvalidDataException; + +/** + * + * @author Jeremy Long + */ +public class Cpe { + + public Cpe(String value) throws UnsupportedEncodingException, InvalidDataException { + this.value = value; + final String[] data = value.substring(7).split(":"); + if (data.length >= 2) { + vendor = URLDecoder.decode(data[0].replace("+", "%2B"), "UTF-8"); + product = URLDecoder.decode(data[1].replace("+", "%2B"), "UTF-8"); + } else { + throw new InvalidDataException(String.format("CPE has an invalid format: %s", value)); + } + } + + /** + * The CPE identifier string (cpe:/a:vendor:product:version). + */ + private String value; + + /** + * Get the value of value. + * + * @return the value of value + */ + public String getValue() { + return value; + } + + /** + * Set the value of value. + * + * @param value new value of value + */ + public void setValue(String value) { + this.value = value; + } + /** + * The vendor portion of the identifier. + */ + private String vendor; + + /** + * Get the value of vendor + * + * @return the value of vendor + */ + public String getVendor() { + return vendor; + } + + /** + * Set the value of vendor + * + * @param vendor new value of vendor + */ + public void setVendor(String vendor) { + this.vendor = vendor; + } + + /** + * The product portion of the identifier. + */ + private String product; + + /** + * Get the value of product + * + * @return the value of product + */ + public String getProduct() { + return product; + } + + /** + * Set the value of product + * + * @param product new value of product + */ + public void setProduct(String product) { + this.product = product; + } + + @Override + public String toString() { + return value; + } + +} diff --git a/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/task/DownloadTask.java b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/nvd/DownloadTask.java similarity index 96% rename from dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/task/DownloadTask.java rename to dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/nvd/DownloadTask.java index 3365c179b..ed1ab22bc 100644 --- a/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/task/DownloadTask.java +++ b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/nvd/DownloadTask.java @@ -15,7 +15,7 @@ * * Copyright (c) 2013 Jeremy Long. All Rights Reserved. */ -package org.owasp.dependencycheck.data.update.task; +package org.owasp.dependencycheck.data.update.nvd; import java.io.File; import java.io.FileInputStream; @@ -29,7 +29,6 @@ import java.util.concurrent.Future; import java.util.zip.GZIPInputStream; import org.apache.commons.io.FileUtils; import org.owasp.dependencycheck.data.nvdcve.CveDB; -import org.owasp.dependencycheck.data.update.NvdCveInfo; import org.owasp.dependencycheck.data.update.exception.UpdateException; import org.owasp.dependencycheck.utils.DownloadFailedException; import org.owasp.dependencycheck.utils.Downloader; @@ -55,8 +54,8 @@ public class DownloadTask implements Callable> { * @param nvdCveInfo the NVD CVE info * @param processor the processor service to submit the downloaded files to * @param cveDB the CVE DB to use to store the vulnerability data - * @param settings a reference to the global settings object; this is necessary so that when the thread is started - * the dependencies have a correct reference to the global settings. + * @param settings a reference to the global settings object; this is necessary so that when the thread is started the + * dependencies have a correct reference to the global settings. * @throws UpdateException thrown if temporary files could not be created */ public DownloadTask(NvdCveInfo nvdCveInfo, ExecutorService processor, CveDB cveDB, Settings settings) throws UpdateException { @@ -248,8 +247,7 @@ public class DownloadTask implements Callable> { } /** - * Extracts the file contained in a gzip archive. The extracted file is placed in the exact same path as the file - * specified. + * Extracts the file contained in a gzip archive. The extracted file is placed in the exact same path as the file specified. * * @param file the archive file * @throws FileNotFoundException thrown if the file does not exist diff --git a/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/xml/NvdCve12Handler.java b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/nvd/NvdCve12Handler.java similarity index 99% rename from dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/xml/NvdCve12Handler.java rename to dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/nvd/NvdCve12Handler.java index 453f35b6b..26811ae7b 100644 --- a/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/xml/NvdCve12Handler.java +++ b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/nvd/NvdCve12Handler.java @@ -15,7 +15,7 @@ * * Copyright (c) 2012 Jeremy Long. All Rights Reserved. */ -package org.owasp.dependencycheck.data.update.xml; +package org.owasp.dependencycheck.data.update.nvd; import java.util.ArrayList; import java.util.HashMap; diff --git a/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/xml/NvdCve20Handler.java b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/nvd/NvdCve20Handler.java similarity index 99% rename from dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/xml/NvdCve20Handler.java rename to dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/nvd/NvdCve20Handler.java index 1ecabd763..e2b60db7a 100644 --- a/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/xml/NvdCve20Handler.java +++ b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/nvd/NvdCve20Handler.java @@ -15,7 +15,7 @@ * * Copyright (c) 2012 Jeremy Long. All Rights Reserved. */ -package org.owasp.dependencycheck.data.update.xml; +package org.owasp.dependencycheck.data.update.nvd; import java.io.IOException; import java.util.List; diff --git a/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/NvdCveInfo.java b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/nvd/NvdCveInfo.java similarity index 98% rename from dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/NvdCveInfo.java rename to dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/nvd/NvdCveInfo.java index 45d366eb6..a143000a8 100644 --- a/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/NvdCveInfo.java +++ b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/nvd/NvdCveInfo.java @@ -15,7 +15,7 @@ * * Copyright (c) 2013 Jeremy Long. All Rights Reserved. */ -package org.owasp.dependencycheck.data.update; +package org.owasp.dependencycheck.data.update.nvd; /** * A pojo that contains the Url and timestamp of the current NvdCve XML files. diff --git a/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/task/ProcessTask.java b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/nvd/ProcessTask.java similarity index 97% rename from dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/task/ProcessTask.java rename to dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/nvd/ProcessTask.java index bb0583ad7..df8c52819 100644 --- a/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/task/ProcessTask.java +++ b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/nvd/ProcessTask.java @@ -15,7 +15,7 @@ * * Copyright (c) 2013 Jeremy Long. All Rights Reserved. */ -package org.owasp.dependencycheck.data.update.task; +package org.owasp.dependencycheck.data.update.nvd; import java.io.File; import java.io.FileNotFoundException; @@ -31,8 +31,8 @@ import org.owasp.dependencycheck.data.nvdcve.CveDB; import org.owasp.dependencycheck.data.nvdcve.DatabaseException; import org.owasp.dependencycheck.data.nvdcve.DatabaseProperties; import org.owasp.dependencycheck.data.update.exception.UpdateException; -import org.owasp.dependencycheck.data.update.xml.NvdCve12Handler; -import org.owasp.dependencycheck.data.update.xml.NvdCve20Handler; +import org.owasp.dependencycheck.data.update.nvd.NvdCve12Handler; +import org.owasp.dependencycheck.data.update.nvd.NvdCve20Handler; import org.owasp.dependencycheck.dependency.VulnerableSoftware; import org.owasp.dependencycheck.utils.Settings; import org.slf4j.Logger; diff --git a/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/UpdateableNvdCve.java b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/nvd/UpdateableNvdCve.java similarity index 95% rename from dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/UpdateableNvdCve.java rename to dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/nvd/UpdateableNvdCve.java index 6fc48d0be..2b2cf22b3 100644 --- a/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/UpdateableNvdCve.java +++ b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/nvd/UpdateableNvdCve.java @@ -15,7 +15,7 @@ * * Copyright (c) 2012 Jeremy Long. All Rights Reserved. */ -package org.owasp.dependencycheck.data.update; +package org.owasp.dependencycheck.data.update.nvd; import java.net.MalformedURLException; import java.net.URL; @@ -27,8 +27,8 @@ import org.owasp.dependencycheck.utils.DownloadFailedException; import org.owasp.dependencycheck.utils.Downloader; /** - * Contains a collection of updateable NvdCveInfo objects. This is used to determine which files need to be downloaded - * and processed. + * Contains a collection of updateable NvdCveInfo objects. This is used to determine which files need to be downloaded and + * processed. * * @author Jeremy Long */ @@ -67,8 +67,7 @@ public class UpdateableNvdCve implements java.lang.Iterable, Iterato * * @param id the key for the item to be added * @param url the URL to download the item - * @param oldUrl the URL for the old version of the item (the NVD CVE old schema still contains useful data we - * need). + * @param oldUrl the URL for the old version of the item (the NVD CVE old schema still contains useful data we need). * @throws MalformedURLException thrown if the URL provided is invalid * @throws DownloadFailedException thrown if the download fails. */ @@ -81,8 +80,7 @@ public class UpdateableNvdCve implements java.lang.Iterable, Iterato * * @param id the key for the item to be added * @param url the URL to download the item - * @param oldUrl the URL for the old version of the item (the NVD CVE old schema still contains useful data we - * need). + * @param oldUrl the URL for the old version of the item (the NVD CVE old schema still contains useful data we need). * @param needsUpdate whether or not the data needs to be updated * @throws MalformedURLException thrown if the URL provided is invalid * @throws DownloadFailedException thrown if the download fails. @@ -175,7 +173,7 @@ public class UpdateableNvdCve implements java.lang.Iterable, Iterato * @param key the key to lookup the return value * @return the NvdCveInfo object stored using the specified key */ - NvdCveInfo get(String key) { + public NvdCveInfo get(String key) { return collection.get(key); } diff --git a/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/xml/package-info.java b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/nvd/package-info.java similarity index 89% rename from dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/xml/package-info.java rename to dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/nvd/package-info.java index 8dbc168c1..c1438046c 100644 --- a/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/xml/package-info.java +++ b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/nvd/package-info.java @@ -5,4 +5,4 @@ * extends Index). The Indexer creates a partial-unmarshalling SAX parser (implemented in the NvdCveXmlFilter) that extracts * VulnerabilityTypes (aka Entry) from the NVD CVE data file and stores these into a Lucene Index. */ -package org.owasp.dependencycheck.data.update.xml; +package org.owasp.dependencycheck.data.update.nvd; diff --git a/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/task/package-info.java b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/task/package-info.java deleted file mode 100644 index 0d9385edd..000000000 --- a/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/task/package-info.java +++ /dev/null @@ -1,4 +0,0 @@ -/** - * A collection of callable/runnable tasks used to speed up the update process. - */ -package org.owasp.dependencycheck.data.update.task; diff --git a/dependency-check-core/src/main/resources/META-INF/services/org.owasp.dependencycheck.data.update.CachedWebDataSource b/dependency-check-core/src/main/resources/META-INF/services/org.owasp.dependencycheck.data.update.CachedWebDataSource index 73ca93754..915a3a73d 100644 --- a/dependency-check-core/src/main/resources/META-INF/services/org.owasp.dependencycheck.data.update.CachedWebDataSource +++ b/dependency-check-core/src/main/resources/META-INF/services/org.owasp.dependencycheck.data.update.CachedWebDataSource @@ -1,2 +1,3 @@ org.owasp.dependencycheck.data.update.NvdCveUpdater -org.owasp.dependencycheck.data.update.EngineVersionCheck \ No newline at end of file +org.owasp.dependencycheck.data.update.EngineVersionCheck +org.owasp.dependencycheck.data.update.CpeUpdater \ No newline at end of file diff --git a/dependency-check-core/src/main/resources/data/dbStatements.properties b/dependency-check-core/src/main/resources/data/dbStatements.properties index 5245a03f0..e612f259e 100644 --- a/dependency-check-core/src/main/resources/data/dbStatements.properties +++ b/dependency-check-core/src/main/resources/data/dbStatements.properties @@ -34,3 +34,6 @@ SELECT_PROPERTY=SELECT id, value FROM properties WHERE id = ? INSERT_PROPERTY=INSERT INTO properties (id, value) VALUES (?, ?) UPDATE_PROPERTY=UPDATE properties SET value = ? WHERE id = ? DELETE_PROPERTY=DELETE FROM properties WHERE id = ? + +DELETE_UNUSED_DICT_CPE=DELETE FROM cpeEntry WHERE dictionaryEntry=true AND id NOT IN (SELECT cpeEntryId FROM software) +ADD_DICT_CPE=MERGE INTO cpeEntry (cpe, vendor, product, dictionaryEntry) KEY(cpe) VALUES(?,?,?,true) diff --git a/dependency-check-core/src/main/resources/data/upgrade_2.9.sql b/dependency-check-core/src/main/resources/data/upgrade_2.9.sql new file mode 100644 index 000000000..175da176a --- /dev/null +++ b/dependency-check-core/src/main/resources/data/upgrade_2.9.sql @@ -0,0 +1,6 @@ + +ALTER TABLE cpeEntry ADD COLUMN IF NOT EXISTS dictionaryEntry BOOLEAN; +ALTER TABLE cpeEntry ALTER COLUMN dictionaryEntry SET DEFAULT FALSE; +UPDATE cpeEntry SET dictionaryEntry=false; + +UPDATE Properties SET value='3.0' WHERE ID='version'; \ No newline at end of file diff --git a/dependency-check-core/src/main/resources/dependencycheck.properties b/dependency-check-core/src/main/resources/dependencycheck.properties index 6bbba2235..1096e2d4d 100644 --- a/dependency-check-core/src/main/resources/dependencycheck.properties +++ b/dependency-check-core/src/main/resources/dependencycheck.properties @@ -17,8 +17,8 @@ engine.version.url=http://jeremylong.github.io/DependencyCheck/current.txt # below contains a %s then the data.directory will replace the %s. data.directory=[JAR]/data #if the filename has a %s it will be replaced with the current expected version -data.file_name=cve.%s.h2.db -data.version=2.9 +data.file_name=dc.h2.db +data.version=3.0 data.connection_string=jdbc:h2:file:%s;FILE_LOCK=SERIALIZED;AUTOCOMMIT=ON; #data.connection_string=jdbc:mysql://localhost:3306/dependencycheck @@ -53,6 +53,8 @@ cve.url-1.2.base=https://nvd.nist.gov/download/nvdcve-%d.xml.gz cve.url-2.0.base=https://nvd.nist.gov/feeds/xml/cve/nvdcve-2.0-%d.xml.gz #cve.url-2.0.base=http://static.nvd.nist.gov/feeds/xml/cve/nvdcve-2.0-%d.xml +cpe.validfordays=30 +cpe.url=http://static.nvd.nist.gov/feeds/xml/cpe/dictionary/official-cpe-dictionary_v2.3.xml.gz # file type analyzer settings: analyzer.archive.enabled=true diff --git a/dependency-check-core/src/test/java/org/owasp/dependencycheck/data/nvdcve/BaseDBTestCase.java b/dependency-check-core/src/test/java/org/owasp/dependencycheck/data/nvdcve/BaseDBTestCase.java index 242971ff4..76fe5cecd 100644 --- a/dependency-check-core/src/test/java/org/owasp/dependencycheck/data/nvdcve/BaseDBTestCase.java +++ b/dependency-check-core/src/test/java/org/owasp/dependencycheck/data/nvdcve/BaseDBTestCase.java @@ -45,7 +45,7 @@ public abstract class BaseDBTestCase extends BaseTest { public static void ensureDBExists() throws Exception { java.io.File dataPath = Settings.getDataDirectory(); - String fileName = String.format(Settings.getString(Settings.KEYS.DB_FILE_NAME), Settings.getString(Settings.KEYS.DB_VERSION)); + String fileName = Settings.getString(Settings.KEYS.DB_FILE_NAME); java.io.File dataFile = new File(dataPath, fileName); if (!dataPath.exists() || !dataFile.exists()) { dataPath.mkdirs(); diff --git a/dependency-check-core/src/test/java/org/owasp/dependencycheck/data/nvdcve/DatabasePropertiesIntegrationTest.java b/dependency-check-core/src/test/java/org/owasp/dependencycheck/data/nvdcve/DatabasePropertiesIntegrationTest.java index e657f11ab..0d5d4112f 100644 --- a/dependency-check-core/src/test/java/org/owasp/dependencycheck/data/nvdcve/DatabasePropertiesIntegrationTest.java +++ b/dependency-check-core/src/test/java/org/owasp/dependencycheck/data/nvdcve/DatabasePropertiesIntegrationTest.java @@ -21,7 +21,7 @@ import java.util.Properties; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; import org.junit.Test; -import org.owasp.dependencycheck.data.update.NvdCveInfo; +import org.owasp.dependencycheck.data.update.nvd.NvdCveInfo; /** * diff --git a/dependency-check-core/src/test/java/org/owasp/dependencycheck/data/update/BaseUpdaterTest.java b/dependency-check-core/src/test/java/org/owasp/dependencycheck/data/update/BaseUpdaterTest.java new file mode 100644 index 000000000..71d00d34e --- /dev/null +++ b/dependency-check-core/src/test/java/org/owasp/dependencycheck/data/update/BaseUpdaterTest.java @@ -0,0 +1,98 @@ +/* + * This file is part of dependency-check-core. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * Copyright (c) 2015 Jeremy Long. All Rights Reserved. + */ +package org.owasp.dependencycheck.data.update; + +import org.junit.Test; +import static org.junit.Assert.*; +import org.owasp.dependencycheck.BaseTest; +import org.owasp.dependencycheck.data.nvdcve.CveDB; +import org.owasp.dependencycheck.data.nvdcve.DatabaseProperties; +import org.owasp.dependencycheck.data.update.exception.UpdateException; + +/** + * + * @author Jeremy Long + */ +public class BaseUpdaterTest extends BaseTest { + + /** + * Test of getCveDB method, of class BaseUpdater. + */ + @Test + public void testGetCveDB() { + BaseUpdater instance = new BaseUpdaterImpl(); + CveDB expResult = null; + CveDB result = instance.getCveDB(); + assertEquals(expResult, result); + } + + /** + * Test of getProperties method, of class BaseUpdater. + */ + @Test + public void testGetProperties() throws UpdateException { + BaseUpdater instance = null; + try { + instance = new BaseUpdaterImpl(); + instance.openDataStores(); + + DatabaseProperties result = instance.getProperties(); + assertTrue(result.getProperties().keySet().size() > 1); + } finally { + if (instance != null) { + instance.closeDataStores(); + } + } + } + + /** + * Test of closeDataStores method, of class BaseUpdater. + */ + @Test + public void testCloseDataStores() throws UpdateException { + BaseUpdater instance = null; + try { + instance = new BaseUpdaterImpl(); + instance.openDataStores(); + } finally { + if (instance != null) { + instance.closeDataStores(); + } + } + } + + /** + * Test of openDataStores method, of class BaseUpdater. + */ + @Test + public void testOpenDataStores() throws Exception { + BaseUpdater instance = null; + try { + instance = new BaseUpdaterImpl(); + instance.openDataStores(); + } finally { + if (instance != null) { + instance.closeDataStores(); + } + } + } + + public class BaseUpdaterImpl extends BaseUpdater { + } + +} diff --git a/dependency-check-core/src/test/java/org/owasp/dependencycheck/data/update/CpeUpdaterIntegrationTest.java b/dependency-check-core/src/test/java/org/owasp/dependencycheck/data/update/CpeUpdaterIntegrationTest.java new file mode 100644 index 000000000..32879b12d --- /dev/null +++ b/dependency-check-core/src/test/java/org/owasp/dependencycheck/data/update/CpeUpdaterIntegrationTest.java @@ -0,0 +1,41 @@ +/* + * Copyright 2015 OWASP. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.owasp.dependencycheck.data.update; + +import org.junit.After; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; +import static org.junit.Assert.*; +import org.owasp.dependencycheck.BaseTest; + +/** + * + * @author jeremy + */ +public class CpeUpdaterIntegrationTest extends BaseTest { + + /** + * Test of update method, of class CpeUpdater. + */ + @Test + public void testUpdate() throws Exception { + CpeUpdater instance = new CpeUpdater(); + instance.update(); + } + +} diff --git a/dependency-check-core/src/test/java/org/owasp/dependencycheck/data/update/NvdCveUpdaterIntegrationTest.java b/dependency-check-core/src/test/java/org/owasp/dependencycheck/data/update/NvdCveUpdaterIntegrationTest.java index 3cf0554e7..8f43f1a09 100644 --- a/dependency-check-core/src/test/java/org/owasp/dependencycheck/data/update/NvdCveUpdaterIntegrationTest.java +++ b/dependency-check-core/src/test/java/org/owasp/dependencycheck/data/update/NvdCveUpdaterIntegrationTest.java @@ -17,12 +17,13 @@ */ package org.owasp.dependencycheck.data.update; -import java.io.File; -import java.util.Calendar; -import org.junit.Before; +import java.net.MalformedURLException; +import static org.junit.Assert.assertNotNull; import org.junit.Test; import org.owasp.dependencycheck.BaseTest; -import org.owasp.dependencycheck.utils.Settings; +import org.owasp.dependencycheck.data.update.exception.UpdateException; +import org.owasp.dependencycheck.data.update.nvd.UpdateableNvdCve; +import org.owasp.dependencycheck.utils.DownloadFailedException; /** * @@ -30,39 +31,33 @@ import org.owasp.dependencycheck.utils.Settings; */ public class NvdCveUpdaterIntegrationTest extends BaseTest { - @Before - public void setUp() throws Exception { - int year = Calendar.getInstance().get(Calendar.YEAR); - if (year <= 2014) { - //File f = new File(NvdCveUpdaterIntegrationTest.class.getClassLoader().getResource("nvdcve-2.0-2014.xml").getPath()); - File f = BaseTest.getResourceAsFile(this, "nvdcve-2.0-2014.xml"); - String baseURL = f.toURI().toURL().toString(); - String modified12 = baseURL.replace("nvdcve-2.0-2014.xml", "nvdcve-modified.xml"); - String modified20 = baseURL.replace("nvdcve-2.0-2014.xml", "nvdcve-2.0-modified.xml"); - String full12 = baseURL.replace("nvdcve-2.0-2014.xml", "nvdcve-%d.xml"); - String full20 = baseURL.replace("nvdcve-2.0-2014.xml", "nvdcve-2.0-%d.xml"); -// cve.url-1.2.modified=http://nvd.nist.gov/download/nvdcve-modified.xml -// cve.url-2.0.modified=http://static.nvd.nist.gov/feeds/xml/cve/nvdcve-2.0-modified.xml -// cve.startyear=2014 -// cve.url-2.0.base=http://static.nvd.nist.gov/feeds/xml/cve/nvdcve-2.0-%d.xml -// cve.url-1.2.base=http://nvd.nist.gov/download/nvdcve-%d.xml - - Settings.setString(Settings.KEYS.CVE_MODIFIED_12_URL, modified12); - Settings.setString(Settings.KEYS.CVE_MODIFIED_20_URL, modified20); - Settings.setString(Settings.KEYS.CVE_SCHEMA_1_2, full12); - Settings.setString(Settings.KEYS.CVE_SCHEMA_2_0, full20); - Settings.setString(Settings.KEYS.CVE_START_YEAR, "2014"); - } else { - System.err.println("Consider updating the local data files to make the NvdCveUpdaterIntegrationTest perform faster"); - } + public NvdCveUpdater getUpdater() throws MalformedURLException, DownloadFailedException, UpdateException { + NvdCveUpdater instance = new NvdCveUpdater(); + return instance; } +// test removed as it is duplicative of the EngineIntegrationTest and the NvdCveUpdaterIntergraionTest +// /** +// * Test of update method, of class StandardUpdate. +// */ +// @Test +// public void testUpdate() throws Exception { +// StandardUpdate instance = getStandardUpdateTask(); +// instance.update(); +// //TODO make this an actual test +// } /** - * Test of update method, of class NvdCveUpdater. + * Test of updatesNeeded method, of class StandardUpdate. */ @Test - public void testUpdate() throws Exception { - NvdCveUpdater instance = new NvdCveUpdater(); - instance.update(); + public void testUpdatesNeeded() throws Exception { + NvdCveUpdater instance = getUpdater(); + try { + instance.openDataStores(); + UpdateableNvdCve result = instance.getUpdatesNeeded(); + assertNotNull(result); + } finally { + instance.closeDataStores(); + } } } diff --git a/dependency-check-core/src/test/java/org/owasp/dependencycheck/data/update/StandardUpdateIntegrationTest.java b/dependency-check-core/src/test/java/org/owasp/dependencycheck/data/update/StandardUpdateIntegrationTest.java deleted file mode 100644 index 71516bbaf..000000000 --- a/dependency-check-core/src/test/java/org/owasp/dependencycheck/data/update/StandardUpdateIntegrationTest.java +++ /dev/null @@ -1,67 +0,0 @@ -/* - * This file is part of dependency-check-core. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * Copyright (c) 2013 Jeremy Long. All Rights Reserved. - */ -package org.owasp.dependencycheck.data.update; - -import java.net.MalformedURLException; -import static org.junit.Assert.assertNotNull; -import org.junit.Test; -import org.owasp.dependencycheck.BaseTest; -import org.owasp.dependencycheck.data.update.exception.UpdateException; -import org.owasp.dependencycheck.utils.DownloadFailedException; - -/** - * - * @author Jeremy Long - */ -public class StandardUpdateIntegrationTest extends BaseTest { - - public StandardUpdate getStandardUpdateTask() throws MalformedURLException, DownloadFailedException, UpdateException { - StandardUpdate instance = new StandardUpdate(); - return instance; - } - - /** - * Test of openDataStores method, of class StandardUpdate. - */ - @Test - public void testOpenDataStores() throws Exception { - StandardUpdate instance = getStandardUpdateTask(); - instance.openDataStores(); - instance.closeDataStores(); - } - -// test removed as it is duplicative of the EngineIntegrationTest and the NvdCveUpdaterIntergraionTest -// /** -// * Test of update method, of class StandardUpdate. -// */ -// @Test -// public void testUpdate() throws Exception { -// StandardUpdate instance = getStandardUpdateTask(); -// instance.update(); -// //TODO make this an actual test -// } - /** - * Test of updatesNeeded method, of class StandardUpdate. - */ - @Test - public void testUpdatesNeeded() throws Exception { - StandardUpdate instance = getStandardUpdateTask(); - UpdateableNvdCve result = instance.updatesNeeded(); - assertNotNull(result); - } -} diff --git a/dependency-check-core/src/test/java/org/owasp/dependencycheck/data/update/task/DownloadTaskTest.java b/dependency-check-core/src/test/java/org/owasp/dependencycheck/data/update/nvd/DownloadTaskTest.java similarity index 89% rename from dependency-check-core/src/test/java/org/owasp/dependencycheck/data/update/task/DownloadTaskTest.java rename to dependency-check-core/src/test/java/org/owasp/dependencycheck/data/update/nvd/DownloadTaskTest.java index bdc676e18..573f0739e 100644 --- a/dependency-check-core/src/test/java/org/owasp/dependencycheck/data/update/task/DownloadTaskTest.java +++ b/dependency-check-core/src/test/java/org/owasp/dependencycheck/data/update/nvd/DownloadTaskTest.java @@ -15,8 +15,10 @@ * * Copyright (c) 2014 Jeremy Long. All Rights Reserved. */ -package org.owasp.dependencycheck.data.update.task; +package org.owasp.dependencycheck.data.update.nvd; +import org.owasp.dependencycheck.data.update.nvd.ProcessTask; +import org.owasp.dependencycheck.data.update.nvd.DownloadTask; import java.util.concurrent.ExecutorService; import java.util.concurrent.Future; import org.junit.After; @@ -26,7 +28,7 @@ import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; import org.owasp.dependencycheck.data.nvdcve.CveDB; -import org.owasp.dependencycheck.data.update.NvdCveInfo; +import org.owasp.dependencycheck.data.update.nvd.NvdCveInfo; import org.owasp.dependencycheck.utils.Settings; /** diff --git a/dependency-check-core/src/test/java/org/owasp/dependencycheck/data/update/NvdCveInfoTest.java b/dependency-check-core/src/test/java/org/owasp/dependencycheck/data/update/nvd/NvdCveInfoTest.java similarity index 95% rename from dependency-check-core/src/test/java/org/owasp/dependencycheck/data/update/NvdCveInfoTest.java rename to dependency-check-core/src/test/java/org/owasp/dependencycheck/data/update/nvd/NvdCveInfoTest.java index 84a5e54fc..81250cd52 100644 --- a/dependency-check-core/src/test/java/org/owasp/dependencycheck/data/update/NvdCveInfoTest.java +++ b/dependency-check-core/src/test/java/org/owasp/dependencycheck/data/update/nvd/NvdCveInfoTest.java @@ -15,8 +15,9 @@ * * Copyright (c) 2013 Jeremy Long. All Rights Reserved. */ -package org.owasp.dependencycheck.data.update; +package org.owasp.dependencycheck.data.update.nvd; +import org.owasp.dependencycheck.data.update.nvd.NvdCveInfo; import static org.junit.Assert.assertEquals; import org.junit.Test; import org.owasp.dependencycheck.BaseTest; diff --git a/dependency-check-core/src/test/java/org/owasp/dependencycheck/data/update/nvd/NvdCveUpdaterIntegrationTest.java b/dependency-check-core/src/test/java/org/owasp/dependencycheck/data/update/nvd/NvdCveUpdaterIntegrationTest.java new file mode 100644 index 000000000..d93fbce6b --- /dev/null +++ b/dependency-check-core/src/test/java/org/owasp/dependencycheck/data/update/nvd/NvdCveUpdaterIntegrationTest.java @@ -0,0 +1,69 @@ +/* + * This file is part of dependency-check-core. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * Copyright (c) 2013 Jeremy Long. All Rights Reserved. + */ +package org.owasp.dependencycheck.data.update.nvd; + +import java.io.File; +import java.util.Calendar; +import org.junit.Before; +import org.junit.Test; +import org.owasp.dependencycheck.BaseTest; +import org.owasp.dependencycheck.data.update.NvdCveUpdater; +import org.owasp.dependencycheck.utils.Settings; + +/** + * + * @author Jeremy Long + */ +public class NvdCveUpdaterIntegrationTest extends BaseTest { + + @Before + public void setUp() throws Exception { + int year = Calendar.getInstance().get(Calendar.YEAR); + if (year <= 2014) { + //File f = new File(NvdCveUpdaterIntegrationTest.class.getClassLoader().getResource("nvdcve-2.0-2014.xml").getPath()); + File f = BaseTest.getResourceAsFile(this, "nvdcve-2.0-2014.xml"); + String baseURL = f.toURI().toURL().toString(); + String modified12 = baseURL.replace("nvdcve-2.0-2014.xml", "nvdcve-modified.xml"); + String modified20 = baseURL.replace("nvdcve-2.0-2014.xml", "nvdcve-2.0-modified.xml"); + String full12 = baseURL.replace("nvdcve-2.0-2014.xml", "nvdcve-%d.xml"); + String full20 = baseURL.replace("nvdcve-2.0-2014.xml", "nvdcve-2.0-%d.xml"); +// cve.url-1.2.modified=http://nvd.nist.gov/download/nvdcve-modified.xml +// cve.url-2.0.modified=http://static.nvd.nist.gov/feeds/xml/cve/nvdcve-2.0-modified.xml +// cve.startyear=2014 +// cve.url-2.0.base=http://static.nvd.nist.gov/feeds/xml/cve/nvdcve-2.0-%d.xml +// cve.url-1.2.base=http://nvd.nist.gov/download/nvdcve-%d.xml + + Settings.setString(Settings.KEYS.CVE_MODIFIED_12_URL, modified12); + Settings.setString(Settings.KEYS.CVE_MODIFIED_20_URL, modified20); + Settings.setString(Settings.KEYS.CVE_SCHEMA_1_2, full12); + Settings.setString(Settings.KEYS.CVE_SCHEMA_2_0, full20); + Settings.setString(Settings.KEYS.CVE_START_YEAR, "2014"); + } else { + System.err.println("Consider updating the local data files to make the NvdCveUpdaterIntegrationTest perform faster"); + } + } + + /** + * Test of update method, of class NvdCveUpdater. + */ + @Test + public void testUpdate() throws Exception { + NvdCveUpdater instance = new NvdCveUpdater(); + instance.update(); + } +} diff --git a/dependency-check-core/src/test/java/org/owasp/dependencycheck/data/update/xml/NvdCve_1_2_HandlerTest.java b/dependency-check-core/src/test/java/org/owasp/dependencycheck/data/update/nvd/NvdCve_1_2_HandlerTest.java similarity index 94% rename from dependency-check-core/src/test/java/org/owasp/dependencycheck/data/update/xml/NvdCve_1_2_HandlerTest.java rename to dependency-check-core/src/test/java/org/owasp/dependencycheck/data/update/nvd/NvdCve_1_2_HandlerTest.java index 2ab59b245..fa00e849f 100644 --- a/dependency-check-core/src/test/java/org/owasp/dependencycheck/data/update/xml/NvdCve_1_2_HandlerTest.java +++ b/dependency-check-core/src/test/java/org/owasp/dependencycheck/data/update/nvd/NvdCve_1_2_HandlerTest.java @@ -15,8 +15,9 @@ * * Copyright (c) 2012 Jeremy Long. All Rights Reserved. */ -package org.owasp.dependencycheck.data.update.xml; +package org.owasp.dependencycheck.data.update.nvd; +import org.owasp.dependencycheck.data.update.nvd.NvdCve12Handler; import java.io.File; import java.util.List; import java.util.Map; diff --git a/dependency-check-core/src/test/java/org/owasp/dependencycheck/data/update/xml/NvdCve_2_0_HandlerTest.java b/dependency-check-core/src/test/java/org/owasp/dependencycheck/data/update/nvd/NvdCve_2_0_HandlerTest.java similarity index 94% rename from dependency-check-core/src/test/java/org/owasp/dependencycheck/data/update/xml/NvdCve_2_0_HandlerTest.java rename to dependency-check-core/src/test/java/org/owasp/dependencycheck/data/update/nvd/NvdCve_2_0_HandlerTest.java index 403bd6394..ea1c147ad 100644 --- a/dependency-check-core/src/test/java/org/owasp/dependencycheck/data/update/xml/NvdCve_2_0_HandlerTest.java +++ b/dependency-check-core/src/test/java/org/owasp/dependencycheck/data/update/nvd/NvdCve_2_0_HandlerTest.java @@ -15,8 +15,9 @@ * * Copyright (c) 2012 Jeremy Long. All Rights Reserved. */ -package org.owasp.dependencycheck.data.update.xml; +package org.owasp.dependencycheck.data.update.nvd; +import org.owasp.dependencycheck.data.update.nvd.NvdCve20Handler; import java.io.File; import javax.xml.parsers.SAXParser; import javax.xml.parsers.SAXParserFactory; diff --git a/dependency-check-core/src/test/java/org/owasp/dependencycheck/data/update/UpdateableNvdCveTest.java b/dependency-check-core/src/test/java/org/owasp/dependencycheck/data/update/nvd/UpdateableNvdCveTest.java similarity index 96% rename from dependency-check-core/src/test/java/org/owasp/dependencycheck/data/update/UpdateableNvdCveTest.java rename to dependency-check-core/src/test/java/org/owasp/dependencycheck/data/update/nvd/UpdateableNvdCveTest.java index d8e979321..6aee2790e 100644 --- a/dependency-check-core/src/test/java/org/owasp/dependencycheck/data/update/UpdateableNvdCveTest.java +++ b/dependency-check-core/src/test/java/org/owasp/dependencycheck/data/update/nvd/UpdateableNvdCveTest.java @@ -15,8 +15,10 @@ * * Copyright (c) 2013 Jeremy Long. All Rights Reserved. */ -package org.owasp.dependencycheck.data.update; +package org.owasp.dependencycheck.data.update.nvd; +import org.owasp.dependencycheck.data.update.nvd.UpdateableNvdCve; +import org.owasp.dependencycheck.data.update.nvd.NvdCveInfo; import java.io.File; import java.io.IOException; import java.net.MalformedURLException; diff --git a/dependency-check-core/src/test/resources/dependencycheck.properties b/dependency-check-core/src/test/resources/dependencycheck.properties index 15f1ce0a8..541ce329d 100644 --- a/dependency-check-core/src/test/resources/dependencycheck.properties +++ b/dependency-check-core/src/test/resources/dependencycheck.properties @@ -19,8 +19,8 @@ data.directory=[JAR]/data # if the filename has a %s it will be replaced with the current expected version. For file # based databases the below filename will be added to the data directory above and then # if the connection string has a %s it will be replaced by the directory/filename path. -data.file_name=cve.%s.h2.db -data.version=2.9 +data.file_name=dc.h2.db +data.version=3.0 data.connection_string=jdbc:h2:file:%s;FILE_LOCK=SERIALIZED;AUTOCOMMIT=ON; #data.connection_string=jdbc:mysql://localhost:3306/dependencycheck diff --git a/dependency-check-utils/src/main/java/org/owasp/dependencycheck/utils/Settings.java b/dependency-check-utils/src/main/java/org/owasp/dependencycheck/utils/Settings.java index 0da632208..487ccc4a3 100644 --- a/dependency-check-utils/src/main/java/org/owasp/dependencycheck/utils/Settings.java +++ b/dependency-check-utils/src/main/java/org/owasp/dependencycheck/utils/Settings.java @@ -736,16 +736,15 @@ public final class Settings { * * @param connectionStringKey the property file key for the connection string * @param dbFileNameKey the settings key for the db filename - * @param dbVersionKey the settings key for the dbVersion * @return the connection string * @throws IOException thrown the data directory cannot be created * @throws InvalidSettingException thrown if there is an invalid setting */ - public static String getConnectionString(String connectionStringKey, String dbFileNameKey, String dbVersionKey) + public static String getConnectionString(String connectionStringKey, String dbFileNameKey) throws IOException, InvalidSettingException { final String connStr = Settings.getString(connectionStringKey); if (connStr == null) { - final String msg = String.format("Invalid properties file to get the connection string; '%s' must be defined.", + final String msg = String.format("Invalid properties file; data.connection_string is missing.", connectionStringKey); throw new InvalidSettingException(msg); } @@ -760,18 +759,6 @@ public final class Settings { dbFileNameKey); throw new InvalidSettingException(msg); } - if (fileName.contains("%s")) { - String version = null; - if (dbVersionKey != null) { - version = Settings.getString(dbVersionKey); - } - if (version == null) { - final String msg = String.format("Invalid properties file to get a file based connection string; '%s' must be defined.", - dbFileNameKey); - throw new InvalidSettingException(msg); - } - fileName = String.format(fileName, version); - } if (connStr.startsWith("jdbc:h2:file:") && fileName.endsWith(".h2.db")) { fileName = fileName.substring(0, fileName.length() - 6); } diff --git a/dependency-check-utils/src/test/java/org/owasp/dependencycheck/utils/SettingsTest.java b/dependency-check-utils/src/test/java/org/owasp/dependencycheck/utils/SettingsTest.java index 4c914d791..63caba59a 100644 --- a/dependency-check-utils/src/test/java/org/owasp/dependencycheck/utils/SettingsTest.java +++ b/dependency-check-utils/src/test/java/org/owasp/dependencycheck/utils/SettingsTest.java @@ -163,21 +163,15 @@ public class SettingsTest extends BaseTest { */ @Test public void testGetConnectionString() throws Exception { - String value = Settings.getConnectionString(Settings.KEYS.DB_CONNECTION_STRING, Settings.KEYS.DB_FILE_NAME, Settings.KEYS.DB_VERSION); + String value = Settings.getConnectionString(Settings.KEYS.DB_CONNECTION_STRING, Settings.KEYS.DB_FILE_NAME); Assert.assertNotNull(value); String msg = null; try { - value = Settings.getConnectionString(Settings.KEYS.DB_CONNECTION_STRING, Settings.KEYS.DB_FILE_NAME, null); + value = Settings.getConnectionString("invalidKey", null); } catch (InvalidSettingException e) { msg = e.getMessage(); } - Assert.assertNotNull(msg, msg); - try { - value = Settings.getConnectionString("invalidKey", null, null); - } catch (InvalidSettingException e) { - msg = e.getMessage(); - } - Assert.assertNotNull(msg, msg); + Assert.assertNotNull(msg); } /** diff --git a/dependency-check-utils/src/test/resources/dependencycheck.properties b/dependency-check-utils/src/test/resources/dependencycheck.properties index 2b9b6fcf2..447247e30 100644 --- a/dependency-check-utils/src/test/resources/dependencycheck.properties +++ b/dependency-check-utils/src/test/resources/dependencycheck.properties @@ -16,8 +16,8 @@ engine.version.url=http://jeremylong.github.io/DependencyCheck/current.txt # will not be used. The data.directory will be resolved and if the connection string # below contains a %s then the data.directory will replace the %s. data.directory=[JAR]/data -data.file_name=cve.%s.h2.db -data.version=2.9 +data.file_name=dc.h2.db +data.version=3.0 data.connection_string=jdbc:h2:file:%s;FILE_LOCK=SERIALIZED;AUTOCOMMIT=ON; #data.connection_string=jdbc:h2:file:%s;AUTO_SERVER=TRUE;AUTOCOMMIT=ON; #data.connection_string=jdbc:mysql://localhost:3306/dependencycheck