reorganized code, made the database file name version independent so that upgrades can be made, and implemented a CPE updates per issue #149

Former-commit-id: fdb57afa28ecffdb7ca90971851844718ecb8bb9

dependency-check-core/src/main/java/org/owasp/dependencycheck/data/nvdcve/ConnectionFactory.java

@@ -35,9 +35,8 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 /**
- * Loads the configured database driver and returns the database connection. If the embedded H2 database is used
+ * Loads the configured database driver and returns the database connection. If the embedded H2 database is used obtaining a
- * obtaining a connection will ensure the database file exists and that the appropriate table structure has been
+ * connection will ensure the database file exists and that the appropriate table structure has been created.
- * created.
  *
  * @author Jeremy Long
  */
@@ -55,6 +54,10 @@ public final class ConnectionFactory {
      * Resource location for SQL file used to create the database schema.
      */
     public static final String DB_STRUCTURE_RESOURCE = "data/initialize.sql";
+    /**
+     * Resource location for SQL file used to create the database schema.
+     */
+    public static final String DB_STRUCTURE_UPDATE_RESOURCE = "data/upgrade_%s.sql";
     /**
      * The database driver used to connect to the database.
      */
@@ -79,8 +82,8 @@ public final class ConnectionFactory {
     }
 
     /**
-     * Initializes the connection factory. Ensuring that the appropriate drivers are loaded and that a connection can be
+     * Initializes the connection factory. Ensuring that the appropriate drivers are loaded and that a connection can be made
-     * made successfully.
+     * successfully.
      *
      * @throws DatabaseException thrown if we are unable to connect to the database
      */
@@ -114,8 +117,7 @@ public final class ConnectionFactory {
             try {
                 connectionString = Settings.getConnectionString(
                         Settings.KEYS.DB_CONNECTION_STRING,
-                        Settings.KEYS.DB_FILE_NAME,
+                        Settings.KEYS.DB_FILE_NAME);
-                        Settings.KEYS.DB_VERSION);
             } catch (IOException ex) {
                 LOGGER.debug(
                         "Unable to retrieve the database connection string", ex);
@@ -162,13 +164,12 @@ public final class ConnectionFactory {
                     LOGGER.debug("", dex);
                     throw new DatabaseException("Unable to create the database structure");
                 }
-            } else {
+            }
-                try {
+            try {
-                    ensureSchemaVersion(conn);
+                ensureSchemaVersion(conn);
-                } catch (DatabaseException dex) {
+            } catch (DatabaseException dex) {
-                    LOGGER.debug("", dex);
+                LOGGER.debug("", dex);
-                    throw new DatabaseException("Database schema does not match this version of dependency-check");
+                throw new DatabaseException("Database schema does not match this version of dependency-check", dex);
-                }
             }
         } finally {
             if (conn != null) {
@@ -183,8 +184,8 @@ public final class ConnectionFactory {
 
     /**
      * Cleans up resources and unloads any registered database drivers. This needs to be called to ensure the driver is
-     * unregistered prior to the finalize method being called as during shutdown the class loader used to load the
+     * unregistered prior to the finalize method being called as during shutdown the class loader used to load the driver may be
-     * driver may be unloaded prior to the driver being de-registered.
+     * unloaded prior to the driver being de-registered.
      */
     public static synchronized void cleanup() {
         if (driver != null) {
@@ -229,8 +230,7 @@ public final class ConnectionFactory {
      */
     private static boolean h2DataFileExists() throws IOException {
         final File dir = Settings.getDataDirectory();
-        final String name = Settings.getString(Settings.KEYS.DB_FILE_NAME);
+        final String fileName = Settings.getString(Settings.KEYS.DB_FILE_NAME);
-        final String fileName = String.format(name, DB_SCHEMA_VERSION);
         final File file = new File(dir, fileName);
         return file.exists();
     }
@@ -278,6 +278,46 @@ public final class ConnectionFactory {
         }
     }
 
+    private static void updateSchema(Connection conn, String schema) throws DatabaseException {
+        LOGGER.debug("Updating database structure");
+        InputStream is;
+        InputStreamReader reader;
+        BufferedReader in = null;
+        String updateFile = null;
+        try {
+            updateFile = String.format(DB_STRUCTURE_UPDATE_RESOURCE, schema);
+            is = ConnectionFactory.class.getClassLoader().getResourceAsStream(updateFile);
+            reader = new InputStreamReader(is, "UTF-8");
+            in = new BufferedReader(reader);
+            final StringBuilder sb = new StringBuilder(2110);
+            String tmp;
+            while ((tmp = in.readLine()) != null) {
+                sb.append(tmp);
+            }
+            Statement statement = null;
+            try {
+                statement = conn.createStatement();
+                statement.execute(sb.toString());
+            } catch (SQLException ex) {
+                LOGGER.debug("", ex);
+                throw new DatabaseException("Unable to update database schema", ex);
+            } finally {
+                DBUtils.closeStatement(statement);
+            }
+        } catch (IOException ex) {
+            final String msg = String.format("Upgrade SQL file does not exist: %s", updateFile);
+            throw new DatabaseException(msg, ex);
+        } finally {
+            if (in != null) {
+                try {
+                    in.close();
+                } catch (IOException ex) {
+                    LOGGER.trace("", ex);
+                }
+            }
+        }
+    }
+
     /**
      * Uses the provided connection to check the specified schema version within the database.
      *
@@ -288,12 +328,13 @@ public final class ConnectionFactory {
         ResultSet rs = null;
         CallableStatement cs = null;
         try {
+            //TODO convert this to use DatabaseProperties
             cs = conn.prepareCall("SELECT value FROM properties WHERE id = 'version'");
             rs = cs.executeQuery();
             if (rs.next()) {
-                final boolean isWrongSchema = !DB_SCHEMA_VERSION.equals(rs.getString(1));
+                if (!DB_SCHEMA_VERSION.equals(rs.getString(1))) {
-                if (isWrongSchema) {
+                    LOGGER.debug("Updating from version: " + rs.getString(1));
-                    throw new DatabaseException("Incorrect database schema; unable to continue");
+                    updateSchema(conn, rs.getString(1));
                 }
             } else {
                 throw new DatabaseException("Database schema is missing");

dependency-check-core/src/main/java/org/owasp/dependencycheck/data/nvdcve/CveDB.java

@@ -19,6 +19,7 @@ package org.owasp.dependencycheck.data.nvdcve;
 
 import java.io.IOException;
 import java.io.UnsupportedEncodingException;
+import java.sql.CallableStatement;
 import java.sql.Connection;
 import java.sql.PreparedStatement;
 import java.sql.ResultSet;
@@ -33,6 +34,7 @@ import java.util.Map.Entry;
 import java.util.Properties;
 import java.util.ResourceBundle;
 import java.util.Set;
+import java.util.logging.Level;
 import org.owasp.dependencycheck.data.cwe.CweDB;
 import org.owasp.dependencycheck.dependency.Reference;
 import org.owasp.dependencycheck.dependency.Vulnerability;
@@ -766,8 +768,8 @@ public class CveDB {
         DependencyVersion cpeVersion;
         if (cpe.getVersion() != null && !cpe.getVersion().isEmpty()) {
             String versionText;
-            if (cpe.getRevision() != null && !cpe.getRevision().isEmpty()) {
+            if (cpe.getUpdate() != null && !cpe.getUpdate().isEmpty()) {
-                versionText = String.format("%s.%s", cpe.getVersion(), cpe.getRevision());
+                versionText = String.format("%s.%s", cpe.getVersion(), cpe.getUpdate());
             } else {
                 versionText = cpe.getVersion();
             }
@@ -777,4 +779,41 @@ public class CveDB {
         }
         return cpeVersion;
     }
+
+    /**
+     * Deletes unused dictionary entries from the database.
+     */
+    public void deleteUnusedCpe() {
+        CallableStatement cs = null;
+        try {
+            cs = getConnection().prepareCall(statementBundle.getString("DELETE_UNUSED_DICT_CPE"));
+            cs.executeUpdate();
+        } catch (SQLException ex) {
+            LOGGER.error("Unable to delete CPE dictionary entries", ex);
+        } finally {
+            DBUtils.closeStatement(cs);
+        }
+    }
+
+    /**
+     * Merges CPE entries into the database.
+     *
+     * @param cpe the CPE identifier
+     * @param vendor the CPE vendor
+     * @param product the CPE product
+     */
+    public void addCpe(String cpe, String vendor, String product) {
+        PreparedStatement ps = null;
+        try {
+            ps = getConnection().prepareCall(statementBundle.getString("ADD_DICT_CPE"));
+            ps.setString(1, cpe);
+            ps.setString(2, vendor);
+            ps.setString(3, product);
+            ps.executeUpdate();
+        } catch (SQLException ex) {
+            LOGGER.error("Unable to add CPE dictionary entry", ex);
+        } finally {
+            DBUtils.closeStatement(ps);
+        }
+    }
 }

dependency-check-core/src/main/java/org/owasp/dependencycheck/data/nvdcve/DatabaseProperties.java

@@ -24,7 +24,7 @@ import java.util.Map;
 import java.util.Map.Entry;
 import java.util.Properties;
 import java.util.TreeMap;
-import org.owasp.dependencycheck.data.update.NvdCveInfo;
+import org.owasp.dependencycheck.data.update.nvd.NvdCveInfo;
 import org.owasp.dependencycheck.data.update.exception.UpdateException;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -41,20 +41,28 @@ public class DatabaseProperties {
      */
     private static final Logger LOGGER = LoggerFactory.getLogger(DatabaseProperties.class);
     /**
-     * Modified key word, used as a key to store information about the modified file (i.e. the containing the last 8
+     * Modified key word, used as a key to store information about the modified file (i.e. the containing the last 8 days of
-     * days of updates)..
+     * updates)..
      */
     public static final String MODIFIED = "Modified";
     /**
-     * The properties file key for the last updated field - used to store the last updated time of the Modified NVD CVE
+     * The properties file key for the last updated field - used to store the last updated time of the Modified NVD CVE xml file.
-     * xml file.
      */
     public static final String LAST_UPDATED = "NVD CVE Modified";
     /**
-     * Stores the last updated time for each of the NVD CVE files. These timestamps should be updated if we process the
+     * Stores the last updated time for each of the NVD CVE files. These timestamps should be updated if we process the modified
-     * modified file within 7 days of the last update.
+     * file within 7 days of the last update.
      */
     public static final String LAST_UPDATED_BASE = "NVD CVE ";
+    /**
+     * The key for the last time the CPE data was updated.
+     */
+    public static final String LAST_CPE_UPDATE = "LAST_CPE_UPDATE";
+    /**
+     * The key for the database schema version.
+     */
+    public static final String VERSION = "version";
+
     /**
      * A collection of properties about the data.
      */
@@ -116,8 +124,7 @@ public class DatabaseProperties {
     }
 
     /**
-     * Returns the property value for the given key. If the key is not contained in the underlying properties null is
+     * Returns the property value for the given key. If the key is not contained in the underlying properties null is returned.
-     * returned.
      *
      * @param key the property key
      * @return the value of the property
@@ -127,8 +134,8 @@ public class DatabaseProperties {
     }
 
     /**
-     * Returns the property value for the given key. If the key is not contained in the underlying properties the
+     * Returns the property value for the given key. If the key is not contained in the underlying properties the default value is
-     * default value is returned.
+     * returned.
      *
      * @param key the property key
      * @param defaultValue the default value
@@ -148,8 +155,8 @@ public class DatabaseProperties {
     }
 
     /**
-     * Returns a map of the meta data from the database properties. This primarily contains timestamps of when the NVD
+     * Returns a map of the meta data from the database properties. This primarily contains timestamps of when the NVD CVE
-     * CVE information was last updated.
+     * information was last updated.
      *
      * @return a map of the database meta data
      */

dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/BaseUpdater.java

@@ -0,0 +1,86 @@
+/*
+ * This file is part of dependency-check-core.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * Copyright (c) 2015 Jeremy Long. All Rights Reserved.
+ */
+package org.owasp.dependencycheck.data.update;
+
+import org.owasp.dependencycheck.data.nvdcve.CveDB;
+import org.owasp.dependencycheck.data.nvdcve.DatabaseException;
+import org.owasp.dependencycheck.data.nvdcve.DatabaseProperties;
+import org.owasp.dependencycheck.data.update.exception.UpdateException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ *
+ * @author Jeremy Long
+ */
+public abstract class BaseUpdater {
+
+    /**
+     * Static logger.
+     */
+    private static final Logger LOGGER = LoggerFactory.getLogger(BaseUpdater.class);
+    /**
+     * Information about the timestamps and URLs for data that needs to be updated.
+     */
+    private DatabaseProperties properties;
+    /**
+     * Reference to the Cve Database.
+     */
+    private CveDB cveDB = null;
+
+    protected CveDB getCveDB() {
+        return cveDB;
+    }
+
+    protected DatabaseProperties getProperties() {
+        return properties;
+    }
+
+    /**
+     * Closes the CVE and CPE data stores.
+     */
+    protected void closeDataStores() {
+        if (cveDB != null) {
+            try {
+                cveDB.close();
+            } catch (Throwable ignore) {
+                LOGGER.trace("Error closing the database", ignore);
+            }
+        }
+    }
+
+    /**
+     * Opens the data store.
+     *
+     * @throws UpdateException thrown if a data store cannot be opened
+     */
+    protected final void openDataStores() throws UpdateException {
+        if (cveDB != null) {
+            return;
+        }
+        try {
+            cveDB = new CveDB();
+            cveDB.open();
+        } catch (DatabaseException ex) {
+            closeDataStores();
+            LOGGER.debug("Database Exception opening databases", ex);
+            throw new UpdateException("Error updating the database, please see the log file for more details.");
+        }
+        properties = cveDB.getDatabaseProperties();
+    }
+}

dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/CpeUpdater.java

@@ -0,0 +1,196 @@
+/*
+ * This file is part of dependency-check-core.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * Copyright (c) 2015 Jeremy Long. All Rights Reserved.
+ */
+package org.owasp.dependencycheck.data.update;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.util.Date;
+import java.util.List;
+import java.util.logging.Level;
+import java.util.zip.GZIPInputStream;
+import javax.xml.parsers.ParserConfigurationException;
+import javax.xml.parsers.SAXParser;
+import javax.xml.parsers.SAXParserFactory;
+import org.apache.commons.io.FileUtils;
+import static org.owasp.dependencycheck.data.nvdcve.DatabaseProperties.LAST_CPE_UPDATE;
+import org.owasp.dependencycheck.data.update.cpe.CPEHandler;
+import org.owasp.dependencycheck.data.update.cpe.Cpe;
+import org.owasp.dependencycheck.data.update.exception.UpdateException;
+import org.owasp.dependencycheck.utils.DateUtil;
+import org.owasp.dependencycheck.utils.DownloadFailedException;
+import org.owasp.dependencycheck.utils.Downloader;
+import org.owasp.dependencycheck.utils.Settings;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.xml.sax.SAXException;
+
+/**
+ *
+ * @author Jeremy Long
+ */
+public class CpeUpdater extends BaseUpdater implements CachedWebDataSource {
+
+    /**
+     * Static logger.
+     */
+    private static final Logger LOGGER = LoggerFactory.getLogger(CpeUpdater.class);
+
+    @Override
+    public void update() throws UpdateException {
+        try {
+            openDataStores();
+            if (updateNeeded()) {
+                LOGGER.info("Updating the Common Platform Enumeration (CPE)");
+                final File xml = downloadCpe();
+                List<Cpe> cpes = processXML(xml);
+                getCveDB().deleteUnusedCpe();
+                for (Cpe cpe : cpes) {
+                    getCveDB().addCpe(cpe.getValue(), cpe.getVendor(), cpe.getProduct());
+                }
+                final Date now = new Date();
+                getProperties().save(LAST_CPE_UPDATE, Long.toString(now.getTime()));
+                LOGGER.info("CPE update complete");
+            }
+        } finally {
+            closeDataStores();
+        }
+    }
+
+    /**
+     * Downloads the CPE XML file.
+     *
+     * @return the file reference to the CPE.xml file
+     * @throws UpdateException thrown if there is an issue downloading the XML file
+     */
+    private File downloadCpe() throws UpdateException {
+        File xml;
+        final URL url;
+        try {
+            url = new URL(Settings.getString(Settings.KEYS.CPE_URL));
+            xml = File.createTempFile("cpe", ".xml", Settings.getTempDirectory());
+            Downloader.fetchFile(url, xml);
+            if (url.toExternalForm().endsWith(".xml.gz")) {
+                extractGzip(xml);
+            }
+
+        } catch (MalformedURLException ex) {
+            throw new UpdateException("Invalid CPE URL", ex);
+        } catch (DownloadFailedException ex) {
+            throw new UpdateException("Unable to download CPE XML file", ex);
+        } catch (IOException ex) {
+            throw new UpdateException("Unable to create temporary file to download CPE", ex);
+        }
+        return xml;
+    }
+
+    /**
+     * Parses the CPE XML file to return a list of CPE entries.
+     *
+     * @param xml the CPE data file
+     * @return the list of CPE entries
+     * @throws UpdateException thrown if there is an issue with parsing the XML file
+     */
+    private List<Cpe> processXML(final File xml) throws UpdateException {
+        try {
+            final SAXParserFactory factory = SAXParserFactory.newInstance();
+            final SAXParser saxParser = factory.newSAXParser();
+            CPEHandler handler = new CPEHandler();
+            saxParser.parse(xml, handler);
+            return handler.getData();
+        } catch (ParserConfigurationException ex) {
+            throw new UpdateException("Unable to parse CPE XML file due to SAX Parser Issue", ex);
+        } catch (SAXException ex) {
+            throw new UpdateException("Unable to parse CPE XML file due to SAX Parser Exception", ex);
+        } catch (IOException ex) {
+            throw new UpdateException("Unable to parse CPE XML file due to IO Failure", ex);
+        }
+    }
+
+    /**
+     * Checks to find the last time the CPE data was refreshed and if it needs to be updated.
+     *
+     * @return true if the CPE data should be refreshed
+     */
+    private boolean updateNeeded() {
+        final Date now = new Date();
+        final int days = Settings.getInt(Settings.KEYS.CVE_MODIFIED_VALID_FOR_DAYS, 30);
+        long timestamp = 0;
+        String ts = getProperties().getProperty(LAST_CPE_UPDATE);
+        if (ts != null && ts.matches("^[0-9]+$")) {
+            timestamp = Long.parseLong(ts);
+        }
+        return !DateUtil.withinDateRange(timestamp, now.getTime(), days);
+    }
+
+    /**
+     * Extracts the file contained in a gzip archive. The extracted file is placed in the exact same path as the file specified.
+     *
+     * @param file the archive file
+     * @throws FileNotFoundException thrown if the file does not exist
+     * @throws IOException thrown if there is an error extracting the file.
+     */
+    private void extractGzip(File file) throws FileNotFoundException, IOException {
+        //TODO - move this to a util class as it is duplicative of (copy of) code in the DownloadTask
+        final String originalPath = file.getPath();
+        final File gzip = new File(originalPath + ".gz");
+        if (gzip.isFile() && !gzip.delete()) {
+            gzip.deleteOnExit();
+        }
+        if (!file.renameTo(gzip)) {
+            throw new IOException("Unable to rename '" + file.getPath() + "'");
+        }
+        final File newfile = new File(originalPath);
+
+        final byte[] buffer = new byte[4096];
+
+        GZIPInputStream cin = null;
+        FileOutputStream out = null;
+        try {
+            cin = new GZIPInputStream(new FileInputStream(gzip));
+            out = new FileOutputStream(newfile);
+
+            int len;
+            while ((len = cin.read(buffer)) > 0) {
+                out.write(buffer, 0, len);
+            }
+        } finally {
+            if (cin != null) {
+                try {
+                    cin.close();
+                } catch (IOException ex) {
+                    LOGGER.trace("ignore", ex);
+                }
+            }
+            if (out != null) {
+                try {
+                    out.close();
+                } catch (IOException ex) {
+                    LOGGER.trace("ignore", ex);
+                }
+            }
+            if (gzip.isFile()) {
+                FileUtils.deleteQuietly(gzip);
+            }
+        }
+    }
+}

dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/NvdCveUpdater.java

@@ -18,23 +18,44 @@
 package org.owasp.dependencycheck.data.update;
 
 import java.net.MalformedURLException;
+import java.util.Calendar;
+import java.util.Date;
+import java.util.HashSet;
+import java.util.Set;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import org.owasp.dependencycheck.data.nvdcve.DatabaseProperties;
+import static org.owasp.dependencycheck.data.nvdcve.DatabaseProperties.MODIFIED;
+import org.owasp.dependencycheck.data.update.exception.InvalidDataException;
 import org.owasp.dependencycheck.data.update.exception.UpdateException;
+import org.owasp.dependencycheck.data.update.nvd.DownloadTask;
+import org.owasp.dependencycheck.data.update.nvd.NvdCveInfo;
+import org.owasp.dependencycheck.data.update.nvd.ProcessTask;
+import org.owasp.dependencycheck.data.update.nvd.UpdateableNvdCve;
+import org.owasp.dependencycheck.utils.DateUtil;
 import org.owasp.dependencycheck.utils.DownloadFailedException;
+import org.owasp.dependencycheck.utils.InvalidSettingException;
 import org.owasp.dependencycheck.utils.Settings;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 /**
- * Class responsible for updating the NVD CVE and CPE data stores.
+ * Class responsible for updating the NVD CVE data.
  *
  * @author Jeremy Long
  */
-public class NvdCveUpdater implements CachedWebDataSource {
+public class NvdCveUpdater extends BaseUpdater implements CachedWebDataSource {
 
     /**
      * The logger
      */
     private static final Logger LOGGER = LoggerFactory.getLogger(NvdCveUpdater.class);
+    /**
+     * The max thread pool size to use when downloading files.
+     */
+    public static final int MAX_THREAD_POOL_SIZE = Settings.getInt(Settings.KEYS.MAX_DOWNLOAD_THREAD_POOL_SIZE, 3);
 
     /**
      * <p>
@@ -45,9 +66,10 @@ public class NvdCveUpdater implements CachedWebDataSource {
     @Override
     public void update() throws UpdateException {
         try {
-            final StandardUpdate task = new StandardUpdate();
+            openDataStores();
-            if (task.isUpdateNeeded()) {
+            UpdateableNvdCve updateable = getUpdatesNeeded();
-                task.update();
+            if (updateable.isUpdateNeeded()) {
+                performUpdate(updateable);
             }
         } catch (MalformedURLException ex) {
             LOGGER.warn(
@@ -61,6 +83,202 @@ public class NvdCveUpdater implements CachedWebDataSource {
                         "If you are behind a proxy you may need to configure dependency-check to use the proxy.");
             }
             LOGGER.debug("", ex);
+        } finally {
+            closeDataStores();
         }
     }
+
+    /**
+     * Downloads the latest NVD CVE XML file from the web and imports it into the current CVE Database.
+     *
+     * @param updateable a collection of NVD CVE data file references that need to be downloaded and processed to update the
+     * database
+     * @throws UpdateException is thrown if there is an error updating the database
+     */
+    public void performUpdate(UpdateableNvdCve updateable) throws UpdateException {
+        int maxUpdates = 0;
+        try {
+            for (NvdCveInfo cve : updateable) {
+                if (cve.getNeedsUpdate()) {
+                    maxUpdates += 1;
+                }
+            }
+            if (maxUpdates <= 0) {
+                return;
+            }
+            if (maxUpdates > 3) {
+                LOGGER.info(
+                        "NVD CVE requires several updates; this could take a couple of minutes.");
+            }
+            if (maxUpdates > 0) {
+                openDataStores();
+            }
+
+            final int poolSize = (MAX_THREAD_POOL_SIZE < maxUpdates) ? MAX_THREAD_POOL_SIZE : maxUpdates;
+
+            final ExecutorService downloadExecutors = Executors.newFixedThreadPool(poolSize);
+            final ExecutorService processExecutor = Executors.newSingleThreadExecutor();
+            final Set<Future<Future<ProcessTask>>> downloadFutures = new HashSet<Future<Future<ProcessTask>>>(maxUpdates);
+            for (NvdCveInfo cve : updateable) {
+                if (cve.getNeedsUpdate()) {
+                    final DownloadTask call = new DownloadTask(cve, processExecutor, getCveDB(), Settings.getInstance());
+                    downloadFutures.add(downloadExecutors.submit(call));
+                }
+            }
+            downloadExecutors.shutdown();
+
+            //next, move the future future processTasks to just future processTasks
+            final Set<Future<ProcessTask>> processFutures = new HashSet<Future<ProcessTask>>(maxUpdates);
+            for (Future<Future<ProcessTask>> future : downloadFutures) {
+                Future<ProcessTask> task = null;
+                try {
+                    task = future.get();
+                } catch (InterruptedException ex) {
+                    downloadExecutors.shutdownNow();
+                    processExecutor.shutdownNow();
+
+                    LOGGER.debug("Thread was interrupted during download", ex);
+                    throw new UpdateException("The download was interrupted", ex);
+                } catch (ExecutionException ex) {
+                    downloadExecutors.shutdownNow();
+                    processExecutor.shutdownNow();
+
+                    LOGGER.debug("Thread was interrupted during download execution", ex);
+                    throw new UpdateException("The execution of the download was interrupted", ex);
+                }
+                if (task == null) {
+                    downloadExecutors.shutdownNow();
+                    processExecutor.shutdownNow();
+                    LOGGER.debug("Thread was interrupted during download");
+                    throw new UpdateException("The download was interrupted; unable to complete the update");
+                } else {
+                    processFutures.add(task);
+                }
+            }
+
+            for (Future<ProcessTask> future : processFutures) {
+                try {
+                    final ProcessTask task = future.get();
+                    if (task.getException() != null) {
+                        throw task.getException();
+                    }
+                } catch (InterruptedException ex) {
+                    processExecutor.shutdownNow();
+                    LOGGER.debug("Thread was interrupted during processing", ex);
+                    throw new UpdateException(ex);
+                } catch (ExecutionException ex) {
+                    processExecutor.shutdownNow();
+                    LOGGER.debug("Execution Exception during process", ex);
+                    throw new UpdateException(ex);
+                } finally {
+                    processExecutor.shutdown();
+                }
+            }
+
+            if (maxUpdates >= 1) { //ensure the modified file date gets written (we may not have actually updated it)
+                getProperties().save(updateable.get(MODIFIED));
+                LOGGER.info("Begin database maintenance.");
+                getCveDB().cleanupDatabase();
+                LOGGER.info("End database maintenance.");
+            }
+        } finally {
+            closeDataStores();
+        }
+    }
+
+    /**
+     * Determines if the index needs to be updated. This is done by fetching the NVD CVE meta data and checking the last update
+     * date. If the data needs to be refreshed this method will return the NvdCveUrl for the files that need to be updated.
+     *
+     * @return the collection of files that need to be updated
+     * @throws MalformedURLException is thrown if the URL for the NVD CVE Meta data is incorrect
+     * @throws DownloadFailedException is thrown if there is an error. downloading the NVD CVE download data file
+     * @throws UpdateException Is thrown if there is an issue with the last updated properties file
+     */
+    protected final UpdateableNvdCve getUpdatesNeeded() throws MalformedURLException, DownloadFailedException, UpdateException {
+        UpdateableNvdCve updates = null;
+        try {
+            updates = retrieveCurrentTimestampsFromWeb();
+        } catch (InvalidDataException ex) {
+            final String msg = "Unable to retrieve valid timestamp from nvd cve downloads page";
+            LOGGER.debug(msg, ex);
+            throw new DownloadFailedException(msg, ex);
+        } catch (InvalidSettingException ex) {
+            LOGGER.debug("Invalid setting found when retrieving timestamps", ex);
+            throw new DownloadFailedException("Invalid settings", ex);
+        }
+
+        if (updates == null) {
+            throw new DownloadFailedException("Unable to retrieve the timestamps of the currently published NVD CVE data");
+        }
+        if (!getProperties().isEmpty()) {
+            try {
+                final long lastUpdated = Long.parseLong(getProperties().getProperty(DatabaseProperties.LAST_UPDATED, "0"));
+                final Date now = new Date();
+                final int days = Settings.getInt(Settings.KEYS.CVE_MODIFIED_VALID_FOR_DAYS, 7);
+                if (lastUpdated == updates.getTimeStamp(MODIFIED)) {
+                    updates.clear(); //we don't need to update anything.
+                } else if (DateUtil.withinDateRange(lastUpdated, now.getTime(), days)) {
+                    for (NvdCveInfo entry : updates) {
+                        if (MODIFIED.equals(entry.getId())) {
+                            entry.setNeedsUpdate(true);
+                        } else {
+                            entry.setNeedsUpdate(false);
+                        }
+                    }
+                } else { //we figure out which of the several XML files need to be downloaded.
+                    for (NvdCveInfo entry : updates) {
+                        if (MODIFIED.equals(entry.getId())) {
+                            entry.setNeedsUpdate(true);
+                        } else {
+                            long currentTimestamp = 0;
+                            try {
+                                currentTimestamp = Long.parseLong(getProperties().getProperty(DatabaseProperties.LAST_UPDATED_BASE + entry.getId(), "0"));
+                            } catch (NumberFormatException ex) {
+                                LOGGER.debug("Error parsing '{}' '{}' from nvdcve.lastupdated",
+                                        DatabaseProperties.LAST_UPDATED_BASE, entry.getId(), ex);
+                            }
+                            if (currentTimestamp == entry.getTimestamp()) {
+                                entry.setNeedsUpdate(false);
+                            }
+                        }
+                    }
+                }
+            } catch (NumberFormatException ex) {
+                LOGGER.warn("An invalid schema version or timestamp exists in the data.properties file.");
+                LOGGER.debug("", ex);
+            }
+        }
+        return updates;
+    }
+
+    /**
+     * Retrieves the timestamps from the NVD CVE meta data file.
+     *
+     * @return the timestamp from the currently published nvdcve downloads page
+     * @throws MalformedURLException thrown if the URL for the NVD CCE Meta data is incorrect.
+     * @throws DownloadFailedException thrown if there is an error downloading the nvd cve meta data file
+     * @throws InvalidDataException thrown if there is an exception parsing the timestamps
+     * @throws InvalidSettingException thrown if the settings are invalid
+     */
+    private UpdateableNvdCve retrieveCurrentTimestampsFromWeb()
+            throws MalformedURLException, DownloadFailedException, InvalidDataException, InvalidSettingException {
+
+        final UpdateableNvdCve updates = new UpdateableNvdCve();
+        updates.add(MODIFIED, Settings.getString(Settings.KEYS.CVE_MODIFIED_20_URL),
+                Settings.getString(Settings.KEYS.CVE_MODIFIED_12_URL),
+                false);
+
+        final int start = Settings.getInt(Settings.KEYS.CVE_START_YEAR);
+        final int end = Calendar.getInstance().get(Calendar.YEAR);
+        final String baseUrl20 = Settings.getString(Settings.KEYS.CVE_SCHEMA_2_0);
+        final String baseUrl12 = Settings.getString(Settings.KEYS.CVE_SCHEMA_1_2);
+        for (int i = start; i <= end; i++) {
+            updates.add(Integer.toString(i), String.format(baseUrl20, i),
+                    String.format(baseUrl12, i),
+                    true);
+        }
+        return updates;
+    }
+
 }

dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/StandardUpdate.java

@@ -1,318 +0,0 @@
-/*
- * This file is part of dependency-check-core.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- * Copyright (c) 2012 Jeremy Long. All Rights Reserved.
- */
-package org.owasp.dependencycheck.data.update;
-
-import java.net.MalformedURLException;
-import java.util.Calendar;
-import java.util.Date;
-import java.util.HashSet;
-import java.util.Set;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.Future;
-import org.owasp.dependencycheck.data.nvdcve.CveDB;
-import org.owasp.dependencycheck.data.nvdcve.DatabaseException;
-import org.owasp.dependencycheck.data.nvdcve.DatabaseProperties;
-import static org.owasp.dependencycheck.data.nvdcve.DatabaseProperties.MODIFIED;
-import org.owasp.dependencycheck.data.update.exception.InvalidDataException;
-import org.owasp.dependencycheck.data.update.exception.UpdateException;
-import org.owasp.dependencycheck.data.update.task.DownloadTask;
-import org.owasp.dependencycheck.data.update.task.ProcessTask;
-import org.owasp.dependencycheck.utils.DateUtil;
-import org.owasp.dependencycheck.utils.DownloadFailedException;
-import org.owasp.dependencycheck.utils.InvalidSettingException;
-import org.owasp.dependencycheck.utils.Settings;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Class responsible for updating the NVDCVE data store.
- *
- * @author Jeremy Long
- */
-public class StandardUpdate {
-
-    /**
-     * Static logger.
-     */
-    private static final Logger LOGGER = LoggerFactory.getLogger(StandardUpdate.class);
-    /**
-     * The max thread pool size to use when downloading files.
-     */
-    public static final int MAX_THREAD_POOL_SIZE = Settings.getInt(Settings.KEYS.MAX_DOWNLOAD_THREAD_POOL_SIZE, 3);
-    /**
-     * Information about the timestamps and URLs for data that needs to be updated.
-     */
-    private DatabaseProperties properties;
-    /**
-     * A collection of updateable NVD CVE items.
-     */
-    private UpdateableNvdCve updateable;
-    /**
-     * Reference to the Cve Database.
-     */
-    private CveDB cveDB = null;
-
-    /**
-     * Gets whether or not an update is needed.
-     *
-     * @return true or false depending on whether an update is needed
-     */
-    public boolean isUpdateNeeded() {
-        return updateable.isUpdateNeeded();
-    }
-
-    /**
-     * Constructs a new Standard Update Task.
-     *
-     * @throws MalformedURLException thrown if a configured URL is malformed
-     * @throws DownloadFailedException thrown if a timestamp cannot be checked on a configured URL
-     * @throws UpdateException thrown if there is an exception generating the update task
-     */
-    public StandardUpdate() throws MalformedURLException, DownloadFailedException, UpdateException {
-        openDataStores();
-        properties = cveDB.getDatabaseProperties();
-        updateable = updatesNeeded();
-    }
-
-    /**
-     * <p>
-     * Downloads the latest NVD CVE XML file from the web and imports it into the current CVE Database.</p>
-     *
-     * @throws UpdateException is thrown if there is an error updating the database
-     */
-    public void update() throws UpdateException {
-        int maxUpdates = 0;
-        try {
-            for (NvdCveInfo cve : updateable) {
-                if (cve.getNeedsUpdate()) {
-                    maxUpdates += 1;
-                }
-            }
-            if (maxUpdates <= 0) {
-                return;
-            }
-            if (maxUpdates > 3) {
-                LOGGER.info(
-                        "NVD CVE requires several updates; this could take a couple of minutes.");
-            }
-            if (maxUpdates > 0) {
-                openDataStores();
-            }
-
-            final int poolSize = (MAX_THREAD_POOL_SIZE < maxUpdates) ? MAX_THREAD_POOL_SIZE : maxUpdates;
-
-            final ExecutorService downloadExecutors = Executors.newFixedThreadPool(poolSize);
-            final ExecutorService processExecutor = Executors.newSingleThreadExecutor();
-            final Set<Future<Future<ProcessTask>>> downloadFutures = new HashSet<Future<Future<ProcessTask>>>(maxUpdates);
-            for (NvdCveInfo cve : updateable) {
-                if (cve.getNeedsUpdate()) {
-                    final DownloadTask call = new DownloadTask(cve, processExecutor, cveDB, Settings.getInstance());
-                    downloadFutures.add(downloadExecutors.submit(call));
-                }
-            }
-            downloadExecutors.shutdown();
-
-            //next, move the future future processTasks to just future processTasks
-            final Set<Future<ProcessTask>> processFutures = new HashSet<Future<ProcessTask>>(maxUpdates);
-            for (Future<Future<ProcessTask>> future : downloadFutures) {
-                Future<ProcessTask> task = null;
-                try {
-                    task = future.get();
-                } catch (InterruptedException ex) {
-                    downloadExecutors.shutdownNow();
-                    processExecutor.shutdownNow();
-
-                    LOGGER.debug("Thread was interrupted during download", ex);
-                    throw new UpdateException("The download was interrupted", ex);
-                } catch (ExecutionException ex) {
-                    downloadExecutors.shutdownNow();
-                    processExecutor.shutdownNow();
-
-                    LOGGER.debug("Thread was interrupted during download execution", ex);
-                    throw new UpdateException("The execution of the download was interrupted", ex);
-                }
-                if (task == null) {
-                    downloadExecutors.shutdownNow();
-                    processExecutor.shutdownNow();
-                    LOGGER.debug("Thread was interrupted during download");
-                    throw new UpdateException("The download was interrupted; unable to complete the update");
-                } else {
-                    processFutures.add(task);
-                }
-            }
-
-            for (Future<ProcessTask> future : processFutures) {
-                try {
-                    final ProcessTask task = future.get();
-                    if (task.getException() != null) {
-                        throw task.getException();
-                    }
-                } catch (InterruptedException ex) {
-                    processExecutor.shutdownNow();
-                    LOGGER.debug("Thread was interrupted during processing", ex);
-                    throw new UpdateException(ex);
-                } catch (ExecutionException ex) {
-                    processExecutor.shutdownNow();
-                    LOGGER.debug("Execution Exception during process", ex);
-                    throw new UpdateException(ex);
-                } finally {
-                    processExecutor.shutdown();
-                }
-            }
-
-            if (maxUpdates >= 1) { //ensure the modified file date gets written (we may not have actually updated it)
-                properties.save(updateable.get(MODIFIED));
-                LOGGER.info("Begin database maintenance.");
-                cveDB.cleanupDatabase();
-                LOGGER.info("End database maintenance.");
-            }
-        } finally {
-            closeDataStores();
-        }
-    }
-
-    /**
-     * Determines if the index needs to be updated. This is done by fetching the NVD CVE meta data and checking the last update
-     * date. If the data needs to be refreshed this method will return the NvdCveUrl for the files that need to be updated.
-     *
-     * @return the collection of files that need to be updated
-     * @throws MalformedURLException is thrown if the URL for the NVD CVE Meta data is incorrect
-     * @throws DownloadFailedException is thrown if there is an error. downloading the NVD CVE download data file
-     * @throws UpdateException Is thrown if there is an issue with the last updated properties file
-     */
-    protected final UpdateableNvdCve updatesNeeded() throws MalformedURLException, DownloadFailedException, UpdateException {
-        UpdateableNvdCve updates = null;
-        try {
-            updates = retrieveCurrentTimestampsFromWeb();
-        } catch (InvalidDataException ex) {
-            final String msg = "Unable to retrieve valid timestamp from nvd cve downloads page";
-            LOGGER.debug(msg, ex);
-            throw new DownloadFailedException(msg, ex);
-        } catch (InvalidSettingException ex) {
-            LOGGER.debug("Invalid setting found when retrieving timestamps", ex);
-            throw new DownloadFailedException("Invalid settings", ex);
-        }
-
-        if (updates == null) {
-            throw new DownloadFailedException("Unable to retrieve the timestamps of the currently published NVD CVE data");
-        }
-        if (!properties.isEmpty()) {
-            try {
-                final long lastUpdated = Long.parseLong(properties.getProperty(DatabaseProperties.LAST_UPDATED, "0"));
-                final Date now = new Date();
-                final int days = Settings.getInt(Settings.KEYS.CVE_MODIFIED_VALID_FOR_DAYS, 7);
-                if (lastUpdated == updates.getTimeStamp(MODIFIED)) {
-                    updates.clear(); //we don't need to update anything.
-                } else if (DateUtil.withinDateRange(lastUpdated, now.getTime(), days)) {
-                    for (NvdCveInfo entry : updates) {
-                        if (MODIFIED.equals(entry.getId())) {
-                            entry.setNeedsUpdate(true);
-                        } else {
-                            entry.setNeedsUpdate(false);
-                        }
-                    }
-                } else { //we figure out which of the several XML files need to be downloaded.
-                    for (NvdCveInfo entry : updates) {
-                        if (MODIFIED.equals(entry.getId())) {
-                            entry.setNeedsUpdate(true);
-                        } else {
-                            long currentTimestamp = 0;
-                            try {
-                                currentTimestamp = Long.parseLong(properties.getProperty(DatabaseProperties.LAST_UPDATED_BASE + entry.getId(), "0"));
-                            } catch (NumberFormatException ex) {
-                                LOGGER.debug("Error parsing '{}' '{}' from nvdcve.lastupdated",
-                                        DatabaseProperties.LAST_UPDATED_BASE, entry.getId(), ex);
-                            }
-                            if (currentTimestamp == entry.getTimestamp()) {
-                                entry.setNeedsUpdate(false);
-                            }
-                        }
-                    }
-                }
-            } catch (NumberFormatException ex) {
-                LOGGER.warn("An invalid schema version or timestamp exists in the data.properties file.");
-                LOGGER.debug("", ex);
-            }
-        }
-        return updates;
-    }
-
-    /**
-     * Retrieves the timestamps from the NVD CVE meta data file.
-     *
-     * @return the timestamp from the currently published nvdcve downloads page
-     * @throws MalformedURLException thrown if the URL for the NVD CCE Meta data is incorrect.
-     * @throws DownloadFailedException thrown if there is an error downloading the nvd cve meta data file
-     * @throws InvalidDataException thrown if there is an exception parsing the timestamps
-     * @throws InvalidSettingException thrown if the settings are invalid
-     */
-    private UpdateableNvdCve retrieveCurrentTimestampsFromWeb()
-            throws MalformedURLException, DownloadFailedException, InvalidDataException, InvalidSettingException {
-
-        final UpdateableNvdCve updates = new UpdateableNvdCve();
-        updates.add(MODIFIED, Settings.getString(Settings.KEYS.CVE_MODIFIED_20_URL),
-                Settings.getString(Settings.KEYS.CVE_MODIFIED_12_URL),
-                false);
-
-        final int start = Settings.getInt(Settings.KEYS.CVE_START_YEAR);
-        final int end = Calendar.getInstance().get(Calendar.YEAR);
-        final String baseUrl20 = Settings.getString(Settings.KEYS.CVE_SCHEMA_2_0);
-        final String baseUrl12 = Settings.getString(Settings.KEYS.CVE_SCHEMA_1_2);
-        for (int i = start; i <= end; i++) {
-            updates.add(Integer.toString(i), String.format(baseUrl20, i),
-                    String.format(baseUrl12, i),
-                    true);
-        }
-
-        return updates;
-    }
-
-    /**
-     * Closes the CVE and CPE data stores.
-     */
-    protected void closeDataStores() {
-        if (cveDB != null) {
-            try {
-                cveDB.close();
-            } catch (Throwable ignore) {
-                LOGGER.trace("Error closing the cveDB", ignore);
-            }
-        }
-    }
-
-    /**
-     * Opens the CVE and CPE data stores.
-     *
-     * @throws UpdateException thrown if a data store cannot be opened
-     */
-    protected final void openDataStores() throws UpdateException {
-        if (cveDB != null) {
-            return;
-        }
-        try {
-            cveDB = new CveDB();
-            cveDB.open();
-        } catch (DatabaseException ex) {
-            closeDataStores();
-            LOGGER.debug("Database Exception opening databases", ex);
-            throw new UpdateException("Error updating the CPE/CVE data, please see the log file for more details.");
-        }
-    }
-}

dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/UpdateService.java

@@ -19,6 +19,7 @@ package org.owasp.dependencycheck.data.update;
 
 import java.util.Iterator;
 import java.util.ServiceLoader;
+import org.owasp.dependencycheck.data.update.CachedWebDataSource;
 
 /**
  * The CachedWebDataSource Service Loader. This class loads all services that implement

dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/cpe/CPEHandler.java

@@ -0,0 +1,362 @@
+/*
+ * This file is part of dependency-check-core.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * Copyright (c) 2015 Jeremy Long. All Rights Reserved.
+ */
+package org.owasp.dependencycheck.data.update.cpe;
+
+import java.io.UnsupportedEncodingException;
+import java.net.URLDecoder;
+import java.util.ArrayList;
+import java.util.List;
+import org.owasp.dependencycheck.data.update.NvdCveUpdater;
+import org.owasp.dependencycheck.data.update.exception.InvalidDataException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.xml.sax.Attributes;
+import org.xml.sax.SAXException;
+import org.xml.sax.helpers.DefaultHandler;
+
+/**
+ * A SAX Handler that will parse the CPE XML and load it into the databse.
+ *
+ * @author Jeremy Long
+ */
+public class CPEHandler extends DefaultHandler {
+
+    /**
+     * The current CPE schema.
+     */
+    private static final String CURRENT_SCHEMA_VERSION = "2.3";
+    /**
+     * The text content of the node being processed. This can be used during the end element event.
+     */
+    StringBuilder nodeText = null;
+    /**
+     * A reference to the current element.
+     */
+    Element current = new Element();
+    /**
+     * The logger.
+     */
+    private static final Logger LOGGER = LoggerFactory.getLogger(NvdCveUpdater.class);
+    /**
+     * The list of CPE values.
+     */
+    List<Cpe> data = new ArrayList<Cpe>();
+
+    /**
+     * Returns the list of CPE values.
+     *
+     * @return the list of CPE values
+     */
+    public List<Cpe> getData() {
+        return data;
+    }
+
+    /**
+     * Handles the start element event
+     *
+     * @param uri the elements uri
+     * @param localName the local name
+     * @param qName the qualified name
+     * @param attributes the attributes
+     * @throws SAXException thrown if there is an exception processing the element
+     */
+    @Override
+    public void startElement(String uri, String localName, String qName, Attributes attributes) throws SAXException {
+        nodeText = null;
+        current.setNode(qName);
+        if (current.isCpeItemNode()) {
+            String temp = attributes.getValue("deprecated");
+            String value = attributes.getValue("name");
+            boolean delete = (temp != null && temp.equalsIgnoreCase("true"));
+            if (!delete && value.startsWith("cpe:/a:") && value.length() > 7) {
+                try {
+                    Cpe cpe = new Cpe(value);
+                    data.add(cpe);
+                } catch (UnsupportedEncodingException ex) {
+                    LOGGER.debug("Unable to parse the CPE", ex);
+                } catch (InvalidDataException ex) {
+                    LOGGER.debug("CPE is not the correct format", ex);
+                }
+            }
+        } else if (current.isSchemaVersionNode()) {
+            nodeText = new StringBuilder(3);
+        }
+//        } else if (current.isTitleNode()) {
+//            //do nothing
+//        } else if (current.isMetaNode()) {
+//            //do nothing
+//        } else if (current.isTimestampNode()) {
+//            //do nothing
+//        } else if (current.isCpeListNode()) {
+//            //do nothing
+//        } else if (current.isNotesNode()) {
+//            //do nothing
+//        } else if (current.isNoteNode()) {
+//            //do nothing
+//        } else if (current.isCheckNode()) {
+//            //do nothing
+//        } else if (current.isGeneratorNode()) {
+//            //do nothing
+//        } else if (current.isProductNameNode()) {
+//            //do nothing
+//        } else if (current.isProductVersionNode()) {
+//            //do nothing
+    }
+
+    /**
+     * Reads the characters in the current node.
+     *
+     * @param ch the char array
+     * @param start the start position of the data read
+     * @param length the length of the data read
+     * @throws SAXException thrown if there is an exception processing the characters
+     */
+    @Override
+    public void characters(char[] ch, int start, int length) throws SAXException {
+        if (nodeText != null) {
+            nodeText.append(ch, start, length);
+        }
+    }
+
+    /**
+     * Handles the end element event. Stores the CPE data in the Cve Database if the cpe item node is ending.
+     *
+     * @param uri the element's uri
+     * @param localName the local name
+     * @param qName the qualified name
+     * @throws SAXException thrown if there is an exception processing the element
+     */
+    @Override
+    public void endElement(String uri, String localName, String qName) throws SAXException {
+        current.setNode(qName);
+        if (current.isSchemaVersionNode() && !CURRENT_SCHEMA_VERSION.equals(nodeText.toString())) {
+            throw new SAXException("ERROR: Unexpecgted CPE Schema Version, expected: "
+                    + CURRENT_SCHEMA_VERSION + ", file is: " + nodeText);
+
+        }
+//        } else if (current.isCpeItemNode()) {
+//            //do nothing
+//        } else if (current.isTitleNode()) {
+//            //do nothing
+//        } else if (current.isCpeListNode()) {
+//            //do nothing
+//        } else if (current.isMetaNode()) {
+//            //do nothing
+//        } else if (current.isNotesNode()) {
+//            //do nothing
+//        } else if (current.isNoteNode()) {
+//            //do nothing
+//        } else if (current.isCheckNode()) {
+//            //do nothing
+//        } else if (current.isGeneratorNode()) {
+//            //do nothing
+//        } else if (current.isProductNameNode()) {
+//            //do nothing
+//        } else if (current.isProductVersionNode()) {
+//            //do nothing
+//        else if (current.isTimestampNode()) {
+//            //do nothing
+//        } else {
+//            throw new SAXException("ERROR STATE: Unexpected qName '" + qName + "'");
+//        }
+    }
+
+    // <editor-fold defaultstate="collapsed" desc="The Element Class that maintains state information about the current node">
+    /**
+     * A simple class to maintain information about the current element while parsing the CPE XML.
+     */
+    protected class Element {
+
+        /**
+         * A node type in the CPE Schema 2.2
+         */
+        public static final String CPE_LIST = "cpe-list";
+        /**
+         * A node type in the CPE Schema 2.2
+         */
+        public static final String CPE_ITEM = "cpe-item";
+        /**
+         * A node type in the CPE Schema 2.2
+         */
+        public static final String TITLE = "title";
+        /**
+         * A node type in the CPE Schema 2.2
+         */
+        public static final String NOTES = "notes";
+        /**
+         * A node type in the CPE Schema 2.2
+         */
+        public static final String NOTE = "note";
+        /**
+         * A node type in the CPE Schema 2.2
+         */
+        public static final String CHECK = "check";
+        /**
+         * A node type in the CPE Schema 2.2
+         */
+        public static final String META = "meta:item-metadata";
+        /**
+         * A node type in the CPE Schema 2.2
+         */
+        public static final String GENERATOR = "generator";
+        /**
+         * A node type in the CPE Schema 2.2
+         */
+        public static final String PRODUCT_NAME = "product_name";
+        /**
+         * A node type in the CPE Schema 2.2
+         */
+        public static final String PRODUCT_VERSION = "product_version";
+        /**
+         * A node type in the CPE Schema 2.2
+         */
+        public static final String SCHEMA_VERSION = "schema_version";
+        /**
+         * A node type in the CPE Schema 2.2
+         */
+        public static final String TIMESTAMP = "timestamp";
+        private String node = null;
+
+        /**
+         * Gets the value of node
+         *
+         * @return the value of node
+         */
+        public String getNode() {
+            return this.node;
+        }
+
+        /**
+         * Sets the value of node
+         *
+         * @param node new value of node
+         */
+        public void setNode(String node) {
+            this.node = node;
+        }
+
+        /**
+         * Checks if the handler is at the CPE_LIST node
+         *
+         * @return true or false
+         */
+        public boolean isCpeListNode() {
+            return CPE_LIST.equals(node);
+        }
+
+        /**
+         * Checks if the handler is at the CPE_ITEM node
+         *
+         * @return true or false
+         */
+        public boolean isCpeItemNode() {
+            return CPE_ITEM.equals(node);
+        }
+
+        /**
+         * Checks if the handler is at the TITLE node
+         *
+         * @return true or false
+         */
+        public boolean isTitleNode() {
+            return TITLE.equals(node);
+        }
+
+        /**
+         * Checks if the handler is at the NOTES node
+         *
+         * @return true or false
+         */
+        public boolean isNotesNode() {
+            return NOTES.equals(node);
+        }
+
+        /**
+         * Checks if the handler is at the NOTE node
+         *
+         * @return true or false
+         */
+        public boolean isNoteNode() {
+            return NOTE.equals(node);
+        }
+
+        /**
+         * Checks if the handler is at the CHECK node
+         *
+         * @return true or false
+         */
+        public boolean isCheckNode() {
+            return CHECK.equals(node);
+        }
+
+        /**
+         * Checks if the handler is at the META node
+         *
+         * @return true or false
+         */
+        public boolean isMetaNode() {
+            return META.equals(node);
+        }
+
+        /**
+         * Checks if the handler is at the GENERATOR node
+         *
+         * @return true or false
+         */
+        public boolean isGeneratorNode() {
+            return GENERATOR.equals(node);
+        }
+
+        /**
+         * Checks if the handler is at the PRODUCT_NAME node
+         *
+         * @return true or false
+         */
+        public boolean isProductNameNode() {
+            return PRODUCT_NAME.equals(node);
+        }
+
+        /**
+         * Checks if the handler is at the PRODUCT_VERSION node
+         *
+         * @return true or false
+         */
+        public boolean isProductVersionNode() {
+            return PRODUCT_VERSION.equals(node);
+        }
+
+        /**
+         * Checks if the handler is at the SCHEMA_VERSION node
+         *
+         * @return true or false
+         */
+        public boolean isSchemaVersionNode() {
+            return SCHEMA_VERSION.equals(node);
+        }
+
+        /**
+         * Checks if the handler is at the TIMESTAMP node
+         *
+         * @return true or false
+         */
+        public boolean isTimestampNode() {
+            return TIMESTAMP.equals(node);
+        }
+    }
+    // </editor-fold>
+}

dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/cpe/Cpe.java

@@ -0,0 +1,114 @@
+/*
+ * This file is part of dependency-check-core.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * Copyright (c) 2015 Jeremy Long. All Rights Reserved.
+ */
+package org.owasp.dependencycheck.data.update.cpe;
+
+import java.io.UnsupportedEncodingException;
+import java.net.URLDecoder;
+import org.owasp.dependencycheck.data.update.exception.InvalidDataException;
+
+/**
+ *
+ * @author Jeremy Long
+ */
+public class Cpe {
+
+    public Cpe(String value) throws UnsupportedEncodingException, InvalidDataException {
+        this.value = value;
+        final String[] data = value.substring(7).split(":");
+        if (data.length >= 2) {
+            vendor = URLDecoder.decode(data[0].replace("+", "%2B"), "UTF-8");
+            product = URLDecoder.decode(data[1].replace("+", "%2B"), "UTF-8");
+        } else {
+            throw new InvalidDataException(String.format("CPE has an invalid format: %s", value));
+        }
+    }
+
+    /**
+     * The CPE identifier string (cpe:/a:vendor:product:version).
+     */
+    private String value;
+
+    /**
+     * Get the value of value.
+     *
+     * @return the value of value
+     */
+    public String getValue() {
+        return value;
+    }
+
+    /**
+     * Set the value of value.
+     *
+     * @param value new value of value
+     */
+    public void setValue(String value) {
+        this.value = value;
+    }
+    /**
+     * The vendor portion of the identifier.
+     */
+    private String vendor;
+
+    /**
+     * Get the value of vendor
+     *
+     * @return the value of vendor
+     */
+    public String getVendor() {
+        return vendor;
+    }
+
+    /**
+     * Set the value of vendor
+     *
+     * @param vendor new value of vendor
+     */
+    public void setVendor(String vendor) {
+        this.vendor = vendor;
+    }
+
+    /**
+     * The product portion of the identifier.
+     */
+    private String product;
+
+    /**
+     * Get the value of product
+     *
+     * @return the value of product
+     */
+    public String getProduct() {
+        return product;
+    }
+
+    /**
+     * Set the value of product
+     *
+     * @param product new value of product
+     */
+    public void setProduct(String product) {
+        this.product = product;
+    }
+
+    @Override
+    public String toString() {
+        return value;
+    }
+
+}

dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/nvd/DownloadTask.java

@@ -15,7 +15,7 @@
  *
  * Copyright (c) 2013 Jeremy Long. All Rights Reserved.
  */
-package org.owasp.dependencycheck.data.update.task;
+package org.owasp.dependencycheck.data.update.nvd;
 
 import java.io.File;
 import java.io.FileInputStream;
@@ -29,7 +29,6 @@ import java.util.concurrent.Future;
 import java.util.zip.GZIPInputStream;
 import org.apache.commons.io.FileUtils;
 import org.owasp.dependencycheck.data.nvdcve.CveDB;
-import org.owasp.dependencycheck.data.update.NvdCveInfo;
 import org.owasp.dependencycheck.data.update.exception.UpdateException;
 import org.owasp.dependencycheck.utils.DownloadFailedException;
 import org.owasp.dependencycheck.utils.Downloader;
@@ -55,8 +54,8 @@ public class DownloadTask implements Callable<Future<ProcessTask>> {
      * @param nvdCveInfo the NVD CVE info
      * @param processor the processor service to submit the downloaded files to
      * @param cveDB the CVE DB to use to store the vulnerability data
-     * @param settings a reference to the global settings object; this is necessary so that when the thread is started
+     * @param settings a reference to the global settings object; this is necessary so that when the thread is started the
-     * the dependencies have a correct reference to the global settings.
+     * dependencies have a correct reference to the global settings.
      * @throws UpdateException thrown if temporary files could not be created
      */
     public DownloadTask(NvdCveInfo nvdCveInfo, ExecutorService processor, CveDB cveDB, Settings settings) throws UpdateException {
@@ -248,8 +247,7 @@ public class DownloadTask implements Callable<Future<ProcessTask>> {
     }
 
     /**
-     * Extracts the file contained in a gzip archive. The extracted file is placed in the exact same path as the file
+     * Extracts the file contained in a gzip archive. The extracted file is placed in the exact same path as the file specified.
-     * specified.
      *
      * @param file the archive file
      * @throws FileNotFoundException thrown if the file does not exist

dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/nvd/NvdCve12Handler.java

@@ -15,7 +15,7 @@
  *
  * Copyright (c) 2012 Jeremy Long. All Rights Reserved.
  */
-package org.owasp.dependencycheck.data.update.xml;
+package org.owasp.dependencycheck.data.update.nvd;
 
 import java.util.ArrayList;
 import java.util.HashMap;

dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/nvd/NvdCve20Handler.java

@@ -15,7 +15,7 @@
  *
  * Copyright (c) 2012 Jeremy Long. All Rights Reserved.
  */
-package org.owasp.dependencycheck.data.update.xml;
+package org.owasp.dependencycheck.data.update.nvd;
 
 import java.io.IOException;
 import java.util.List;

dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/nvd/NvdCveInfo.java

@@ -15,7 +15,7 @@
  *
  * Copyright (c) 2013 Jeremy Long. All Rights Reserved.
  */
-package org.owasp.dependencycheck.data.update;
+package org.owasp.dependencycheck.data.update.nvd;
 
 /**
  * A pojo that contains the Url and timestamp of the current NvdCve XML files.

dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/nvd/ProcessTask.java

@@ -15,7 +15,7 @@
  *
  * Copyright (c) 2013 Jeremy Long. All Rights Reserved.
  */
-package org.owasp.dependencycheck.data.update.task;
+package org.owasp.dependencycheck.data.update.nvd;
 
 import java.io.File;
 import java.io.FileNotFoundException;
@@ -31,8 +31,8 @@ import org.owasp.dependencycheck.data.nvdcve.CveDB;
 import org.owasp.dependencycheck.data.nvdcve.DatabaseException;
 import org.owasp.dependencycheck.data.nvdcve.DatabaseProperties;
 import org.owasp.dependencycheck.data.update.exception.UpdateException;
-import org.owasp.dependencycheck.data.update.xml.NvdCve12Handler;
+import org.owasp.dependencycheck.data.update.nvd.NvdCve12Handler;
-import org.owasp.dependencycheck.data.update.xml.NvdCve20Handler;
+import org.owasp.dependencycheck.data.update.nvd.NvdCve20Handler;
 import org.owasp.dependencycheck.dependency.VulnerableSoftware;
 import org.owasp.dependencycheck.utils.Settings;
 import org.slf4j.Logger;

dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/nvd/UpdateableNvdCve.java

@@ -15,7 +15,7 @@
  *
  * Copyright (c) 2012 Jeremy Long. All Rights Reserved.
  */
-package org.owasp.dependencycheck.data.update;
+package org.owasp.dependencycheck.data.update.nvd;
 
 import java.net.MalformedURLException;
 import java.net.URL;
@@ -27,8 +27,8 @@ import org.owasp.dependencycheck.utils.DownloadFailedException;
 import org.owasp.dependencycheck.utils.Downloader;
 
 /**
- * Contains a collection of updateable NvdCveInfo objects. This is used to determine which files need to be downloaded
+ * Contains a collection of updateable NvdCveInfo objects. This is used to determine which files need to be downloaded and
- * and processed.
+ * processed.
  *
  * @author Jeremy Long
  */
@@ -67,8 +67,7 @@ public class UpdateableNvdCve implements java.lang.Iterable<NvdCveInfo>, Iterato
      *
      * @param id the key for the item to be added
      * @param url the URL to download the item
-     * @param oldUrl the URL for the old version of the item (the NVD CVE old schema still contains useful data we
+     * @param oldUrl the URL for the old version of the item (the NVD CVE old schema still contains useful data we need).
-     * need).
      * @throws MalformedURLException thrown if the URL provided is invalid
      * @throws DownloadFailedException thrown if the download fails.
      */
@@ -81,8 +80,7 @@ public class UpdateableNvdCve implements java.lang.Iterable<NvdCveInfo>, Iterato
      *
      * @param id the key for the item to be added
      * @param url the URL to download the item
-     * @param oldUrl the URL for the old version of the item (the NVD CVE old schema still contains useful data we
+     * @param oldUrl the URL for the old version of the item (the NVD CVE old schema still contains useful data we need).
-     * need).
      * @param needsUpdate whether or not the data needs to be updated
      * @throws MalformedURLException thrown if the URL provided is invalid
      * @throws DownloadFailedException thrown if the download fails.
@@ -175,7 +173,7 @@ public class UpdateableNvdCve implements java.lang.Iterable<NvdCveInfo>, Iterato
      * @param key the key to lookup the return value
      * @return the NvdCveInfo object stored using the specified key
      */
-    NvdCveInfo get(String key) {
+    public NvdCveInfo get(String key) {
         return collection.get(key);
     }
 

dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/nvd/package-info.java

@@ -5,4 +5,4 @@
  * extends Index). The Indexer creates a partial-unmarshalling SAX parser (implemented in the NvdCveXmlFilter) that extracts
  * VulnerabilityTypes (aka Entry) from the NVD CVE data file and stores these into a Lucene Index.
  */
-package org.owasp.dependencycheck.data.update.xml;
+package org.owasp.dependencycheck.data.update.nvd;

dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/task/package-info.java

@@ -1,4 +0,0 @@
-/**
- * A collection of callable/runnable tasks used to speed up the update process.
- */
-package org.owasp.dependencycheck.data.update.task;

dependency-check-core/src/main/resources/META-INF/services/org.owasp.dependencycheck.data.update.CachedWebDataSource

@@ -1,2 +1,3 @@
 org.owasp.dependencycheck.data.update.NvdCveUpdater
 org.owasp.dependencycheck.data.update.EngineVersionCheck
+org.owasp.dependencycheck.data.update.CpeUpdater

dependency-check-core/src/main/resources/data/dbStatements.properties

@@ -34,3 +34,6 @@ SELECT_PROPERTY=SELECT id, value FROM properties WHERE id = ?
 INSERT_PROPERTY=INSERT INTO properties (id, value) VALUES (?, ?)
 UPDATE_PROPERTY=UPDATE properties SET value = ? WHERE id = ?
 DELETE_PROPERTY=DELETE FROM properties WHERE id = ?
+
+DELETE_UNUSED_DICT_CPE=DELETE FROM cpeEntry WHERE dictionaryEntry=true AND id NOT IN (SELECT cpeEntryId FROM software)
+ADD_DICT_CPE=MERGE INTO cpeEntry (cpe, vendor, product, dictionaryEntry) KEY(cpe) VALUES(?,?,?,true)

dependency-check-core/src/main/resources/data/upgrade_2.9.sql

@@ -0,0 +1,6 @@
+
+ALTER TABLE cpeEntry ADD COLUMN IF NOT EXISTS dictionaryEntry BOOLEAN;
+ALTER TABLE cpeEntry ALTER COLUMN dictionaryEntry  SET DEFAULT FALSE;
+UPDATE cpeEntry SET dictionaryEntry=false;
+
+UPDATE Properties SET value='3.0' WHERE ID='version';

dependency-check-core/src/main/resources/dependencycheck.properties

@@ -17,8 +17,8 @@ engine.version.url=http://jeremylong.github.io/DependencyCheck/current.txt
 # below contains a %s then the data.directory will replace the %s.
 data.directory=[JAR]/data
 #if the filename has a %s it will be replaced with the current expected version
-data.file_name=cve.%s.h2.db
+data.file_name=dc.h2.db
-data.version=2.9
+data.version=3.0
 data.connection_string=jdbc:h2:file:%s;FILE_LOCK=SERIALIZED;AUTOCOMMIT=ON;
 #data.connection_string=jdbc:mysql://localhost:3306/dependencycheck
 
@@ -53,6 +53,8 @@ cve.url-1.2.base=https://nvd.nist.gov/download/nvdcve-%d.xml.gz
 cve.url-2.0.base=https://nvd.nist.gov/feeds/xml/cve/nvdcve-2.0-%d.xml.gz
 #cve.url-2.0.base=http://static.nvd.nist.gov/feeds/xml/cve/nvdcve-2.0-%d.xml
 
+cpe.validfordays=30
+cpe.url=http://static.nvd.nist.gov/feeds/xml/cpe/dictionary/official-cpe-dictionary_v2.3.xml.gz
 
 # file type analyzer settings:
 analyzer.archive.enabled=true

dependency-check-core/src/test/java/org/owasp/dependencycheck/data/nvdcve/BaseDBTestCase.java

@@ -45,7 +45,7 @@ public abstract class BaseDBTestCase extends BaseTest {
     public static void ensureDBExists() throws Exception {
 
         java.io.File dataPath = Settings.getDataDirectory();
-        String fileName = String.format(Settings.getString(Settings.KEYS.DB_FILE_NAME), Settings.getString(Settings.KEYS.DB_VERSION));
+        String fileName = Settings.getString(Settings.KEYS.DB_FILE_NAME);
         java.io.File dataFile = new File(dataPath, fileName);
         if (!dataPath.exists() || !dataFile.exists()) {
             dataPath.mkdirs();

dependency-check-core/src/test/java/org/owasp/dependencycheck/data/nvdcve/DatabasePropertiesIntegrationTest.java

@@ -21,7 +21,7 @@ import java.util.Properties;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;
 import org.junit.Test;
-import org.owasp.dependencycheck.data.update.NvdCveInfo;
+import org.owasp.dependencycheck.data.update.nvd.NvdCveInfo;
 
 /**
  *

dependency-check-core/src/test/java/org/owasp/dependencycheck/data/update/BaseUpdaterTest.java

@@ -0,0 +1,98 @@
+/*
+ * This file is part of dependency-check-core.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * Copyright (c) 2015 Jeremy Long. All Rights Reserved.
+ */
+package org.owasp.dependencycheck.data.update;
+
+import org.junit.Test;
+import static org.junit.Assert.*;
+import org.owasp.dependencycheck.BaseTest;
+import org.owasp.dependencycheck.data.nvdcve.CveDB;
+import org.owasp.dependencycheck.data.nvdcve.DatabaseProperties;
+import org.owasp.dependencycheck.data.update.exception.UpdateException;
+
+/**
+ *
+ * @author Jeremy Long
+ */
+public class BaseUpdaterTest extends BaseTest {
+
+    /**
+     * Test of getCveDB method, of class BaseUpdater.
+     */
+    @Test
+    public void testGetCveDB() {
+        BaseUpdater instance = new BaseUpdaterImpl();
+        CveDB expResult = null;
+        CveDB result = instance.getCveDB();
+        assertEquals(expResult, result);
+    }
+
+    /**
+     * Test of getProperties method, of class BaseUpdater.
+     */
+    @Test
+    public void testGetProperties() throws UpdateException {
+        BaseUpdater instance = null;
+        try {
+            instance = new BaseUpdaterImpl();
+            instance.openDataStores();
+
+            DatabaseProperties result = instance.getProperties();
+            assertTrue(result.getProperties().keySet().size() > 1);
+        } finally {
+            if (instance != null) {
+                instance.closeDataStores();
+            }
+        }
+    }
+
+    /**
+     * Test of closeDataStores method, of class BaseUpdater.
+     */
+    @Test
+    public void testCloseDataStores() throws UpdateException {
+        BaseUpdater instance = null;
+        try {
+            instance = new BaseUpdaterImpl();
+            instance.openDataStores();
+        } finally {
+            if (instance != null) {
+                instance.closeDataStores();
+            }
+        }
+    }
+
+    /**
+     * Test of openDataStores method, of class BaseUpdater.
+     */
+    @Test
+    public void testOpenDataStores() throws Exception {
+        BaseUpdater instance = null;
+        try {
+            instance = new BaseUpdaterImpl();
+            instance.openDataStores();
+        } finally {
+            if (instance != null) {
+                instance.closeDataStores();
+            }
+        }
+    }
+
+    public class BaseUpdaterImpl extends BaseUpdater {
+    }
+
+}

dependency-check-core/src/test/java/org/owasp/dependencycheck/data/update/CpeUpdaterIntegrationTest.java

@@ -0,0 +1,41 @@
+/*
+ * Copyright 2015 OWASP.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.owasp.dependencycheck.data.update;
+
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import static org.junit.Assert.*;
+import org.owasp.dependencycheck.BaseTest;
+
+/**
+ *
+ * @author jeremy
+ */
+public class CpeUpdaterIntegrationTest extends BaseTest {
+
+    /**
+     * Test of update method, of class CpeUpdater.
+     */
+    @Test
+    public void testUpdate() throws Exception {
+        CpeUpdater instance = new CpeUpdater();
+        instance.update();
+    }
+
+}

dependency-check-core/src/test/java/org/owasp/dependencycheck/data/update/NvdCveUpdaterIntegrationTest.java

@@ -17,12 +17,13 @@
  */
 package org.owasp.dependencycheck.data.update;
 
-import java.io.File;
+import java.net.MalformedURLException;
-import java.util.Calendar;
+import static org.junit.Assert.assertNotNull;
-import org.junit.Before;
 import org.junit.Test;
 import org.owasp.dependencycheck.BaseTest;
-import org.owasp.dependencycheck.utils.Settings;
+import org.owasp.dependencycheck.data.update.exception.UpdateException;
+import org.owasp.dependencycheck.data.update.nvd.UpdateableNvdCve;
+import org.owasp.dependencycheck.utils.DownloadFailedException;
 
 /**
  *
@@ -30,39 +31,33 @@ import org.owasp.dependencycheck.utils.Settings;
  */
 public class NvdCveUpdaterIntegrationTest extends BaseTest {
 
-    @Before
+    public NvdCveUpdater getUpdater() throws MalformedURLException, DownloadFailedException, UpdateException {
-    public void setUp() throws Exception {
+        NvdCveUpdater instance = new NvdCveUpdater();
-        int year = Calendar.getInstance().get(Calendar.YEAR);
+        return instance;
-        if (year <= 2014) {
-            //File f = new File(NvdCveUpdaterIntegrationTest.class.getClassLoader().getResource("nvdcve-2.0-2014.xml").getPath());
-            File f = BaseTest.getResourceAsFile(this, "nvdcve-2.0-2014.xml");
-            String baseURL = f.toURI().toURL().toString();
-            String modified12 = baseURL.replace("nvdcve-2.0-2014.xml", "nvdcve-modified.xml");
-            String modified20 = baseURL.replace("nvdcve-2.0-2014.xml", "nvdcve-2.0-modified.xml");
-            String full12 = baseURL.replace("nvdcve-2.0-2014.xml", "nvdcve-%d.xml");
-            String full20 = baseURL.replace("nvdcve-2.0-2014.xml", "nvdcve-2.0-%d.xml");
-//        cve.url-1.2.modified=http://nvd.nist.gov/download/nvdcve-modified.xml
-//        cve.url-2.0.modified=http://static.nvd.nist.gov/feeds/xml/cve/nvdcve-2.0-modified.xml
-//        cve.startyear=2014
-//        cve.url-2.0.base=http://static.nvd.nist.gov/feeds/xml/cve/nvdcve-2.0-%d.xml
-//        cve.url-1.2.base=http://nvd.nist.gov/download/nvdcve-%d.xml
-
-            Settings.setString(Settings.KEYS.CVE_MODIFIED_12_URL, modified12);
-            Settings.setString(Settings.KEYS.CVE_MODIFIED_20_URL, modified20);
-            Settings.setString(Settings.KEYS.CVE_SCHEMA_1_2, full12);
-            Settings.setString(Settings.KEYS.CVE_SCHEMA_2_0, full20);
-            Settings.setString(Settings.KEYS.CVE_START_YEAR, "2014");
-        } else {
-            System.err.println("Consider updating the local data files to make the NvdCveUpdaterIntegrationTest perform faster");
-        }
     }
 
+// test removed as it is duplicative of the EngineIntegrationTest and the NvdCveUpdaterIntergraionTest
+//    /**
+//     * Test of update method, of class StandardUpdate.
+//     */
+//    @Test
+//    public void testUpdate() throws Exception {
+//        StandardUpdate instance = getStandardUpdateTask();
+//        instance.update();
+//        //TODO make this an actual test
+//    }
     /**
-     * Test of update method, of class NvdCveUpdater.
+     * Test of updatesNeeded method, of class StandardUpdate.
      */
     @Test
-    public void testUpdate() throws Exception {
+    public void testUpdatesNeeded() throws Exception {
-        NvdCveUpdater instance = new NvdCveUpdater();
+        NvdCveUpdater instance = getUpdater();
-        instance.update();
+        try {
+            instance.openDataStores();
+            UpdateableNvdCve result = instance.getUpdatesNeeded();
+            assertNotNull(result);
+        } finally {
+            instance.closeDataStores();
+        }
     }
 }

dependency-check-core/src/test/java/org/owasp/dependencycheck/data/update/StandardUpdateIntegrationTest.java

@@ -1,67 +0,0 @@
-/*
- * This file is part of dependency-check-core.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- * Copyright (c) 2013 Jeremy Long. All Rights Reserved.
- */
-package org.owasp.dependencycheck.data.update;
-
-import java.net.MalformedURLException;
-import static org.junit.Assert.assertNotNull;
-import org.junit.Test;
-import org.owasp.dependencycheck.BaseTest;
-import org.owasp.dependencycheck.data.update.exception.UpdateException;
-import org.owasp.dependencycheck.utils.DownloadFailedException;
-
-/**
- *
- * @author Jeremy Long
- */
-public class StandardUpdateIntegrationTest extends BaseTest {
-
-    public StandardUpdate getStandardUpdateTask() throws MalformedURLException, DownloadFailedException, UpdateException {
-        StandardUpdate instance = new StandardUpdate();
-        return instance;
-    }
-
-    /**
-     * Test of openDataStores method, of class StandardUpdate.
-     */
-    @Test
-    public void testOpenDataStores() throws Exception {
-        StandardUpdate instance = getStandardUpdateTask();
-        instance.openDataStores();
-        instance.closeDataStores();
-    }
-
-// test removed as it is duplicative of the EngineIntegrationTest and the NvdCveUpdaterIntergraionTest
-//    /**
-//     * Test of update method, of class StandardUpdate.
-//     */
-//    @Test
-//    public void testUpdate() throws Exception {
-//        StandardUpdate instance = getStandardUpdateTask();
-//        instance.update();
-//        //TODO make this an actual test
-//    }
-    /**
-     * Test of updatesNeeded method, of class StandardUpdate.
-     */
-    @Test
-    public void testUpdatesNeeded() throws Exception {
-        StandardUpdate instance = getStandardUpdateTask();
-        UpdateableNvdCve result = instance.updatesNeeded();
-        assertNotNull(result);
-    }
-}

dependency-check-core/src/test/java/org/owasp/dependencycheck/data/update/nvd/DownloadTaskTest.java

@@ -15,8 +15,10 @@
  *
  * Copyright (c) 2014 Jeremy Long. All Rights Reserved.
  */
-package org.owasp.dependencycheck.data.update.task;
+package org.owasp.dependencycheck.data.update.nvd;
 
+import org.owasp.dependencycheck.data.update.nvd.ProcessTask;
+import org.owasp.dependencycheck.data.update.nvd.DownloadTask;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Future;
 import org.junit.After;
@@ -26,7 +28,7 @@ import org.junit.Before;
 import org.junit.BeforeClass;
 import org.junit.Test;
 import org.owasp.dependencycheck.data.nvdcve.CveDB;
-import org.owasp.dependencycheck.data.update.NvdCveInfo;
+import org.owasp.dependencycheck.data.update.nvd.NvdCveInfo;
 import org.owasp.dependencycheck.utils.Settings;
 
 /**

dependency-check-core/src/test/java/org/owasp/dependencycheck/data/update/nvd/NvdCveInfoTest.java

@@ -15,8 +15,9 @@
  *
  * Copyright (c) 2013 Jeremy Long. All Rights Reserved.
  */
-package org.owasp.dependencycheck.data.update;
+package org.owasp.dependencycheck.data.update.nvd;
 
+import org.owasp.dependencycheck.data.update.nvd.NvdCveInfo;
 import static org.junit.Assert.assertEquals;
 import org.junit.Test;
 import org.owasp.dependencycheck.BaseTest;

dependency-check-core/src/test/java/org/owasp/dependencycheck/data/update/nvd/NvdCveUpdaterIntegrationTest.java

@@ -0,0 +1,69 @@
+/*
+ * This file is part of dependency-check-core.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * Copyright (c) 2013 Jeremy Long. All Rights Reserved.
+ */
+package org.owasp.dependencycheck.data.update.nvd;
+
+import java.io.File;
+import java.util.Calendar;
+import org.junit.Before;
+import org.junit.Test;
+import org.owasp.dependencycheck.BaseTest;
+import org.owasp.dependencycheck.data.update.NvdCveUpdater;
+import org.owasp.dependencycheck.utils.Settings;
+
+/**
+ *
+ * @author Jeremy Long
+ */
+public class NvdCveUpdaterIntegrationTest extends BaseTest {
+
+    @Before
+    public void setUp() throws Exception {
+        int year = Calendar.getInstance().get(Calendar.YEAR);
+        if (year <= 2014) {
+            //File f = new File(NvdCveUpdaterIntegrationTest.class.getClassLoader().getResource("nvdcve-2.0-2014.xml").getPath());
+            File f = BaseTest.getResourceAsFile(this, "nvdcve-2.0-2014.xml");
+            String baseURL = f.toURI().toURL().toString();
+            String modified12 = baseURL.replace("nvdcve-2.0-2014.xml", "nvdcve-modified.xml");
+            String modified20 = baseURL.replace("nvdcve-2.0-2014.xml", "nvdcve-2.0-modified.xml");
+            String full12 = baseURL.replace("nvdcve-2.0-2014.xml", "nvdcve-%d.xml");
+            String full20 = baseURL.replace("nvdcve-2.0-2014.xml", "nvdcve-2.0-%d.xml");
+//        cve.url-1.2.modified=http://nvd.nist.gov/download/nvdcve-modified.xml
+//        cve.url-2.0.modified=http://static.nvd.nist.gov/feeds/xml/cve/nvdcve-2.0-modified.xml
+//        cve.startyear=2014
+//        cve.url-2.0.base=http://static.nvd.nist.gov/feeds/xml/cve/nvdcve-2.0-%d.xml
+//        cve.url-1.2.base=http://nvd.nist.gov/download/nvdcve-%d.xml
+
+            Settings.setString(Settings.KEYS.CVE_MODIFIED_12_URL, modified12);
+            Settings.setString(Settings.KEYS.CVE_MODIFIED_20_URL, modified20);
+            Settings.setString(Settings.KEYS.CVE_SCHEMA_1_2, full12);
+            Settings.setString(Settings.KEYS.CVE_SCHEMA_2_0, full20);
+            Settings.setString(Settings.KEYS.CVE_START_YEAR, "2014");
+        } else {
+            System.err.println("Consider updating the local data files to make the NvdCveUpdaterIntegrationTest perform faster");
+        }
+    }
+
+    /**
+     * Test of update method, of class NvdCveUpdater.
+     */
+    @Test
+    public void testUpdate() throws Exception {
+        NvdCveUpdater instance = new NvdCveUpdater();
+        instance.update();
+    }
+}

dependency-check-core/src/test/java/org/owasp/dependencycheck/data/update/nvd/NvdCve_1_2_HandlerTest.java

@@ -15,8 +15,9 @@
  *
  * Copyright (c) 2012 Jeremy Long. All Rights Reserved.
  */
-package org.owasp.dependencycheck.data.update.xml;
+package org.owasp.dependencycheck.data.update.nvd;
 
+import org.owasp.dependencycheck.data.update.nvd.NvdCve12Handler;
 import java.io.File;
 import java.util.List;
 import java.util.Map;

dependency-check-core/src/test/java/org/owasp/dependencycheck/data/update/nvd/NvdCve_2_0_HandlerTest.java

@@ -15,8 +15,9 @@
  *
  * Copyright (c) 2012 Jeremy Long. All Rights Reserved.
  */
-package org.owasp.dependencycheck.data.update.xml;
+package org.owasp.dependencycheck.data.update.nvd;
 
+import org.owasp.dependencycheck.data.update.nvd.NvdCve20Handler;
 import java.io.File;
 import javax.xml.parsers.SAXParser;
 import javax.xml.parsers.SAXParserFactory;

dependency-check-core/src/test/java/org/owasp/dependencycheck/data/update/nvd/UpdateableNvdCveTest.java

@@ -15,8 +15,10 @@
  *
  * Copyright (c) 2013 Jeremy Long. All Rights Reserved.
  */
-package org.owasp.dependencycheck.data.update;
+package org.owasp.dependencycheck.data.update.nvd;
 
+import org.owasp.dependencycheck.data.update.nvd.UpdateableNvdCve;
+import org.owasp.dependencycheck.data.update.nvd.NvdCveInfo;
 import java.io.File;
 import java.io.IOException;
 import java.net.MalformedURLException;

dependency-check-core/src/test/resources/dependencycheck.properties

@@ -19,8 +19,8 @@ data.directory=[JAR]/data
 # if the filename has a %s it will be replaced with the current expected version. For file
 # based databases the below filename will be added to the data directory above and then
 # if the connection string has a %s it will be replaced by the directory/filename path.
-data.file_name=cve.%s.h2.db
+data.file_name=dc.h2.db
-data.version=2.9
+data.version=3.0
 data.connection_string=jdbc:h2:file:%s;FILE_LOCK=SERIALIZED;AUTOCOMMIT=ON;
 #data.connection_string=jdbc:mysql://localhost:3306/dependencycheck
 

dependency-check-utils/src/main/java/org/owasp/dependencycheck/utils/Settings.java

@@ -736,16 +736,15 @@ public final class Settings {
      *
      * @param connectionStringKey the property file key for the connection string
      * @param dbFileNameKey the settings key for the db filename
-     * @param dbVersionKey the settings key for the dbVersion
      * @return the connection string
      * @throws IOException thrown the data directory cannot be created
      * @throws InvalidSettingException thrown if there is an invalid setting
      */
-    public static String getConnectionString(String connectionStringKey, String dbFileNameKey, String dbVersionKey)
+    public static String getConnectionString(String connectionStringKey, String dbFileNameKey)
             throws IOException, InvalidSettingException {
         final String connStr = Settings.getString(connectionStringKey);
         if (connStr == null) {
-            final String msg = String.format("Invalid properties file to get the connection string; '%s' must be defined.",
+            final String msg = String.format("Invalid properties file; data.connection_string is missing.",
                     connectionStringKey);
             throw new InvalidSettingException(msg);
         }
@@ -760,18 +759,6 @@ public final class Settings {
                         dbFileNameKey);
                 throw new InvalidSettingException(msg);
             }
-            if (fileName.contains("%s")) {
-                String version = null;
-                if (dbVersionKey != null) {
-                    version = Settings.getString(dbVersionKey);
-                }
-                if (version == null) {
-                    final String msg = String.format("Invalid properties file to get a file based connection string; '%s' must be defined.",
-                            dbFileNameKey);
-                    throw new InvalidSettingException(msg);
-                }
-                fileName = String.format(fileName, version);
-            }
             if (connStr.startsWith("jdbc:h2:file:") && fileName.endsWith(".h2.db")) {
                 fileName = fileName.substring(0, fileName.length() - 6);
             }

dependency-check-utils/src/test/java/org/owasp/dependencycheck/utils/SettingsTest.java

@@ -163,21 +163,15 @@ public class SettingsTest extends BaseTest {
      */
     @Test
     public void testGetConnectionString() throws Exception {
-        String value = Settings.getConnectionString(Settings.KEYS.DB_CONNECTION_STRING, Settings.KEYS.DB_FILE_NAME, Settings.KEYS.DB_VERSION);
+        String value = Settings.getConnectionString(Settings.KEYS.DB_CONNECTION_STRING, Settings.KEYS.DB_FILE_NAME);
         Assert.assertNotNull(value);
         String msg = null;
         try {
-            value = Settings.getConnectionString(Settings.KEYS.DB_CONNECTION_STRING, Settings.KEYS.DB_FILE_NAME, null);
+            value = Settings.getConnectionString("invalidKey", null);
         } catch (InvalidSettingException e) {
             msg = e.getMessage();
         }
-        Assert.assertNotNull(msg, msg);
+        Assert.assertNotNull(msg);
-        try {
-            value = Settings.getConnectionString("invalidKey", null, null);
-        } catch (InvalidSettingException e) {
-            msg = e.getMessage();
-        }
-        Assert.assertNotNull(msg, msg);
     }
 
     /**

dependency-check-utils/src/test/resources/dependencycheck.properties

@@ -16,8 +16,8 @@ engine.version.url=http://jeremylong.github.io/DependencyCheck/current.txt
 # will not be used. The data.directory will be resolved and if the connection string
 # below contains a %s then the data.directory will replace the %s.
 data.directory=[JAR]/data
-data.file_name=cve.%s.h2.db
+data.file_name=dc.h2.db
-data.version=2.9
+data.version=3.0
 data.connection_string=jdbc:h2:file:%s;FILE_LOCK=SERIALIZED;AUTOCOMMIT=ON;
 #data.connection_string=jdbc:h2:file:%s;AUTO_SERVER=TRUE;AUTOCOMMIT=ON;
 #data.connection_string=jdbc:mysql://localhost:3306/dependencycheck