version 1.0.7

Former-commit-id: 3a17193efed4254ec0d4b566b01afcbda3e6af23

README.md

@@ -5,8 +5,70 @@ Dependency-Check is a utility that attempts to detect publicly disclosed vulnera
 
 Documentation and links to production binary releases can be found on the [github pages](http://jeremylong.github.io/DependencyCheck/). Additionally, more information about the architecture and ways to extend dependency-check can be found on the [wiki].
 
-Initial Usage
+Current Releases
 -------------
+### Jenkins Plugin
+
+For instructions on the use of the Jenkins plugin please see the [Jenkins dependency-check page](http://wiki.jenkins-ci.org/x/CwDgAQ).
+
+### Command Line
+
+More detailed instructions can be found on the [dependency-check github pages](http://jeremylong.github.io/DependencyCheck/dependency-check-cli/installation.html).
+The latest CLI can be downloaded from bintray's [dependency-check page](https://bintray.com/jeremy-long/owasp/dependency-check). 
+
+On *nix
+```
+$ ./bin/dependency-check.sh -h
+$ ./bin/dependency-check.sh --app Testing --out . --scan [path to jar files to be scanned]
+```
+On Windows
+```
+> bin/dependency-check.bat -h
+> bin/dependency-check.bat --app Testing --out . --scan [path to jar files to be scanned]
+```
+
+### Maven Plugin
+
+More detailed instructions can be found on the [dependency-check-maven github pages](http://jeremylong.github.io/DependencyCheck/dependency-check-maven/installation.html).
+The plugin can be configured using the following:
+
+```xml
+<project>
+    <build>
+        <plugins>
+            ...
+            <plugin>
+              <groupId>org.owasp</groupId>
+              <artifactId>dependency-check-maven</artifactId>
+              <version>1.0.2</version>
+              <executions>
+                  <execution>
+                      <goals>
+                          <goal>check</goal>
+                      </goals>
+                  </execution>
+              </executions>
+            </plugin>
+            ...
+        </plugins>
+        ...
+    </build>
+    ...
+</project>
+```
+
+### Ant Task
+
+For instructions on the use of the Ant Task, please see the [dependency-check-ant github page](http://jeremylong.github.io/DependencyCheck/dependency-check-maven/installation.html).
+
+Development Usage
+-------------
+The following instructions outline how to compile and use the current snapshot. While every intention is to maintain a stable snapshot it is recommended
+that the release versions listed above be used.
+
+Note, currently the install goal may take a long time to execute the integration tests. However, if this takes more then 30 minutes it is likely that the
+download of data from the NVD is having an issue. This issue is still being researched and a solution should be published soon.
+
 On *nix
 ```
 $ mvn install
@@ -20,7 +82,7 @@ On Windows
 > dependency-check-cli/target/release/bin/dependency-check.bat --app Testing --out . --scan ./src/test/resources
 ```
 
-Then load the resulting 'DependencyCheck-Report.html' into your favorite browser.
+Then load the resulting 'DependencyCheck-Report.html' into your favourite browser.
 
 Mailing List
 ------------
@@ -29,6 +91,8 @@ Subscribe: [dependency-check+subscribe@googlegroups.com] [subscribe]
 
 Post: [dependency-check@googlegroups.com] [post]
 
+Archive: [google group](https://groups.google.com/forum/#!forum/dependency-check)
+
 Copyright & License
 -
 

dependency-check-ant/README.md

@@ -1,29 +1,25 @@
-Dependency-Check-Ant
-===================
+Dependency-Check Ant Task
+=========
 
-Dependency-Check is a utility that attempts to detect publicly disclosed vulnerabilities contained within project dependencies. It does this by determining if there is a Common Platform Enumeration (CPE) identifier for a given dependency. If found, it will generate a report linking to the associated CVE entries.
+Dependency-Check Ant Task can be used to check the project dependencies for published security vulnerabilities. The checks
+performed are a "best effort" and as such, there could be false positives as well as false negatives. However,
+vulnerabilities in 3rd party components is a well-known problem and is currently documented in the 2013 OWASP
+Top 10 as [A9 - Using Components with Known Vulnerabilities](https://www.owasp.org/index.php/Top_10_2013-A9-Using_Components_with_Known_Vulnerabilities).
 
 Documentation and links to production binary releases can be found on the [github pages](http://jeremylong.github.io/DependencyCheck/dependency-check-ant/installation.html).
 
 Mailing List
 ------------
 
-Subscribe: [dependency-check+subscribe@googlegroups.com] [subscribe]
+Subscribe: [dependency-check+subscribe@googlegroups.com](mailto:dependency-check+subscribe@googlegroups.com)
 
-Post: [dependency-check@googlegroups.com] [post]
+Post: [dependency-check@googlegroups.com](mailto:dependency-check@googlegroups.com)
 
 Copyright & License
--
+-------------------
 
 Dependency-Check is Copyright (c) 2012-2013 Jeremy Long. All Rights Reserved.
 
-Permission to modify and redistribute is granted under the terms of the GPLv3 license. See the [LICENSE.txt] [GPLv3] file for the full license.
+Permission to modify and redistribute is granted under the terms of the GPLv3 license. See the [LICENSE.txt](https://github.com/jeremylong/DependencyCheck/dependency-check-ant/blob/master/LICENSE.txt) file for the full license.
 
-Dependency-Check makes use of several other open source libraries. Please see the [NOTICE.txt] [notices] file for more information.
-
-
-  [wiki]: https://github.com/jeremylong/DependencyCheck/wiki
-  [subscribe]: mailto:dependency-check+subscribe@googlegroups.com
-  [post]: mailto:dependency-check@googlegroups.com
-  [GPLv3]: https://github.com/jeremylong/DependencyCheck/blob/master/LICENSE.txt
-  [notices]: https://github.com/jeremylong/DependencyCheck/blob/master/NOTICES.txt
+Dependency-Check-Ant makes use of other open source libraries. Please see the [NOTICE.txt](https://github.com/jeremylong/DependencyCheck/dependency-check-ant/blob/master/NOTICES.txt) file for more information.

dependency-check-ant/pom.xml

@@ -22,13 +22,13 @@ Copyright (c) 2013 - Jeremy Long. All Rights Reserved.
     <parent>
         <groupId>org.owasp</groupId>
         <artifactId>dependency-check-parent</artifactId>
-        <version>1.0.1</version>
+        <version>1.0.7</version>
     </parent>
 
     <artifactId>dependency-check-ant</artifactId>
     <packaging>jar</packaging>
 
-    <name>dependency-check-ant</name>
+    <name>Dependency-Check Ant Task</name>
     <description>Dependency-check is a utility that attempts to detect publicly disclosed vulnerabilities contained within project dependencies. It does this by determining if there is a Common Platform Enumeration (CPE) identifier for a given dependency. If found, it will generate a report linking to the associated CVE entries.</description>
     <!-- begin copy from http://minds.coremedia.com/2012/09/11/problem-solved-deploy-multi-module-maven-project-site-as-github-pages/ -->
     <distributionManagement>
@@ -76,6 +76,25 @@ Copyright (c) 2013 - Jeremy Long. All Rights Reserved.
                 </configuration>
                 <executions>
                     <!-- the following executions are solely to setup the test environment -->
+                    <execution>
+                        <id>copy-test-data.zip</id>
+                        <phase>validate</phase>
+                        <goals>
+                            <goal>copy-resources</goal>
+                        </goals>
+                        <configuration>
+                            <outputDirectory>${project.build.directory}/test-classes</outputDirectory>
+                            <resources>
+                                <resource>
+                                    <directory>${basedir}/../src/test/resources</directory>
+                                    <filtering>false</filtering>
+                                    <includes>
+                                        <include>data.zip</include>
+                                    </includes>
+                                </resource>
+                            </resources>
+                        </configuration>
+                    </execution>
                     <execution>
                         <id>copy-test-resources-1</id>
                         <phase>validate</phase>
@@ -89,7 +108,7 @@ Copyright (c) 2013 - Jeremy Long. All Rights Reserved.
                                     <directory>${basedir}/../src/test/resources</directory>
                                     <filtering>false</filtering>
                                     <includes>
-                                        <include>axis2*.jar</include>
+                                        <include>org.mortbay.*.jar</include>
                                     </includes>
                                 </resource>
                             </resources>
@@ -127,7 +146,7 @@ Copyright (c) 2013 - Jeremy Long. All Rights Reserved.
                                     <directory>${basedir}/../src/test/resources</directory>
                                     <filtering>false</filtering>
                                     <includes>
-                                        <include>*.war</include>
+                                        <include>struts.jar</include>
                                     </includes>
                                 </resource>
                             </resources>
@@ -146,13 +165,32 @@ Copyright (c) 2013 - Jeremy Long. All Rights Reserved.
                                     <directory>${basedir}/../src/test/resources</directory>
                                     <filtering>false</filtering>
                                     <includes>
-                                        <include>jetty-6.1.0.jar</include>
                                         <include>org.mortbay.jetty.jar</include>
                                     </includes>
                                 </resource>
                             </resources>
                         </configuration>
                     </execution>
+                    <execution>
+                        <id>copy-data</id>
+                        <phase>validate</phase>
+                        <goals>
+                            <goal>copy-resources</goal>
+                        </goals>
+                        <configuration>
+                            <outputDirectory>${project.build.directory}/test-classes</outputDirectory>
+                            <resources>
+                                <resource>
+                                    <directory>${basedir}/../src/test/resources</directory>
+                                    <filtering>false</filtering>
+                                    <includes>
+                                        <include>db.cve.zip</include>
+                                        <include>index.cpe.zip</include>
+                                    </includes>
+                                </resource>
+                            </resources>
+                        </configuration>
+                    </execution>
                 </executions>
             </plugin>
             <plugin>
@@ -238,6 +276,10 @@ Copyright (c) 2013 - Jeremy Long. All Rights Reserved.
                             <value>${project.build.directory}/cobertura/cobertura.ser</value>
                             <workingDirectory>target</workingDirectory>
                         </property>
+                        <property>
+                            <name>data.directory</name>
+                            <value>${project.build.directory}/dependency-check-data</value>
+                        </property>
                     </systemProperties>
                 </configuration>
             </plugin>
@@ -389,6 +431,13 @@ Copyright (c) 2013 - Jeremy Long. All Rights Reserved.
             <artifactId>dependency-check-core</artifactId>
             <version>${project.parent.version}</version>
         </dependency>
+        <dependency>
+            <groupId>org.owasp</groupId>
+            <artifactId>dependency-check-core</artifactId>
+            <version>${project.parent.version}</version>
+            <type>test-jar</type>
+            <scope>test</scope>
+        </dependency>
         <dependency>
             <groupId>org.apache.ant</groupId>
             <artifactId>ant</artifactId>

dependency-check-ant/src/main/java/org/owasp/dependencycheck/taskdefs/DependencyCheckTask.java

@@ -23,7 +23,6 @@ import java.io.IOException;
 import java.io.InputStream;
 import java.util.List;
 import java.util.logging.Level;
-import java.util.logging.LogManager;
 import java.util.logging.Logger;
 import org.apache.tools.ant.BuildException;
 import org.apache.tools.ant.Task;
@@ -38,6 +37,7 @@ import org.owasp.dependencycheck.dependency.Dependency;
 import org.owasp.dependencycheck.dependency.Vulnerability;
 import org.owasp.dependencycheck.reporting.ReportGenerator;
 import org.owasp.dependencycheck.reporting.ReportGenerator.Format;
+import org.owasp.dependencycheck.utils.LogUtils;
 import org.owasp.dependencycheck.utils.Settings;
 
 /**
@@ -323,6 +323,50 @@ public class DependencyCheckTask extends Task {
     public void setProxyPort(String proxyPort) {
         this.proxyPort = proxyPort;
     }
+    /**
+     * The Proxy username.
+     */
+    private String proxyUsername;
+
+    /**
+     * Get the value of proxyUsername.
+     *
+     * @return the value of proxyUsername
+     */
+    public String getProxyUsername() {
+        return proxyUsername;
+    }
+
+    /**
+     * Set the value of proxyUsername.
+     *
+     * @param proxyUsername new value of proxyUsername
+     */
+    public void setProxyUsername(String proxyUsername) {
+        this.proxyUsername = proxyUsername;
+    }
+    /**
+     * The Proxy password.
+     */
+    private String proxyPassword;
+
+    /**
+     * Get the value of proxyPassword.
+     *
+     * @return the value of proxyPassword
+     */
+    public String getProxyPassword() {
+        return proxyPassword;
+    }
+
+    /**
+     * Set the value of proxyPassword.
+     *
+     * @param proxyPassword new value of proxyPassword
+     */
+    public void setProxyPassword(String proxyPassword) {
+        this.proxyPassword = proxyPassword;
+    }
     /**
      * The Connection Timeout.
      */
@@ -345,41 +389,55 @@ public class DependencyCheckTask extends Task {
     public void setConnectionTimeout(String connectionTimeout) {
         this.connectionTimeout = connectionTimeout;
     }
+    /**
+     * The file path used for verbose logging.
+     */
+    private String logFile = null;
 
     /**
-     * Configures the logger for use by the application.
+     * Get the value of logFile.
+     *
+     * @return the value of logFile
      */
-    private static void prepareLogger() {
-        InputStream in = null;
-        try {
-            in = DependencyCheckTask.class.getClassLoader().getResourceAsStream(LOG_PROPERTIES_FILE);
-            LogManager.getLogManager().reset();
-            LogManager.getLogManager().readConfiguration(in);
-            //TODO add code to disable fine grained log file.
-//            Logger logger = LogManager.getLogManager().getLogger("");
-//            for (Handler h : logger.getHandlers()) {
-//                if (h.getFormatter(). h.toString());
-//            }
-        } catch (IOException ex) {
-            System.err.println(ex.toString());
-            Logger.getLogger(DependencyCheckTask.class.getName()).log(Level.SEVERE, null, ex);
-        } catch (SecurityException ex) {
-            Logger.getLogger(DependencyCheckTask.class.getName()).log(Level.SEVERE, null, ex);
-        } finally {
-            if (in != null) {
-                try {
-                    in.close();
-                } catch (Exception ex) {
-                    //noinspection UnusedAssignment
-                    in = null;
-                }
-            }
-        }
+    public String getLogFile() {
+        return logFile;
+    }
+
+    /**
+     * Set the value of logFile.
+     *
+     * @param logFile new value of logFile
+     */
+    public void setLogFile(String logFile) {
+        this.logFile = logFile;
+    }
+    /**
+     * The path to the suppression file.
+     */
+    private String suppressionFile;
+
+    /**
+     * Get the value of suppressionFile.
+     *
+     * @return the value of suppressionFile
+     */
+    public String getSuppressionFile() {
+        return suppressionFile;
+    }
+
+    /**
+     * Set the value of suppressionFile.
+     *
+     * @param suppressionFile new value of suppressionFile
+     */
+    public void setSuppressionFile(String suppressionFile) {
+        this.suppressionFile = suppressionFile;
     }
 
     @Override
     public void execute() throws BuildException {
-        prepareLogger();
+        final InputStream in = DependencyCheckTask.class.getClassLoader().getResourceAsStream(LOG_PROPERTIES_FILE);
+        LogUtils.prepareLogger(in, logFile);
 
         dealWithReferences();
         validateConfiguration();
@@ -467,9 +525,18 @@ public class DependencyCheckTask extends Task {
         if (proxyPort != null && !proxyPort.isEmpty()) {
             Settings.setString(Settings.KEYS.PROXY_PORT, proxyPort);
         }
+        if (proxyUsername != null && !proxyUsername.isEmpty()) {
+            Settings.setString(Settings.KEYS.PROXY_USERNAME, proxyUsername);
+        }
+        if (proxyPassword != null && !proxyPassword.isEmpty()) {
+            Settings.setString(Settings.KEYS.PROXY_PASSWORD, proxyPassword);
+        }
         if (connectionTimeout != null && !connectionTimeout.isEmpty()) {
             Settings.setString(Settings.KEYS.CONNECTION_TIMEOUT, connectionTimeout);
         }
+        if (suppressionFile != null && !suppressionFile.isEmpty()) {
+            Settings.setString(Settings.KEYS.SUPPRESSION_FILE, suppressionFile);
+        }
     }
 
     /**
@@ -512,6 +579,7 @@ public class DependencyCheckTask extends Task {
          *
          * @return the list of values for the report format
          */
+        @Override
         public String[] getValues() {
             int i = 0;
             final Format[] formats = Format.values();

dependency-check-ant/src/main/resources/log.properties

@@ -4,7 +4,7 @@ handlers=java.util.logging.ConsoleHandler, java.util.logging.FileHandler
 # FINEST, FINER, FINE, CONFIG, INFO, WARNING and SEVERE.
 
 # Configure the ConsoleHandler.
-java.util.logging.ConsoleHandler.level=WARNING
+java.util.logging.ConsoleHandler.level=INFO
 
 #org.owasp.dependencycheck.data.nvdcve.xml
 

dependency-check-ant/src/site/markdown/configuration.md

@@ -28,8 +28,12 @@ ReportOutputDirectory | The directory where dependency-check will store data use
 FailBuildOn           | If set and a CVE is found that is greater then the specified value the build will fail. The default value is 11 which means that the build will not fail. Valid values are 0-11. | Optional
 AutoUpdate            | If set to false the NVD CVE data is not automatically updated. Setting this to false could result in false negatives. However, this may be required in some environments. The default value is true. | Optional
 DataDirectory         | The directory where dependency-check will store data used for analysis. Defaults to a folder called, called 'dependency-check-data', that is in the same directory as the dependency-check-ant jar file was installed in. *It is not recommended to change this.* | Optional
+LogFile               | The file path to write verbose logging information. | Optional
+SuppressionFile       | An XML file conforming to the suppression schema that suppresses findings; this is used to hide false positives. | Optional
 ProxyUrl              | Defines the proxy used to connect to the Internet. | Optional
 ProxyPort             | Defines the port for the proxy. | Optional
+ProxyUsername         | Defines the proxy user name. | Optional
+ProxyPassword         | Defines the proxy password. | Optional
 ConnectionTimeout     | The connection timeout used when downloading data files from the Internet. | Optional
 
 

dependency-check-ant/src/site/markdown/installation.md.vm

@@ -1,6 +1,6 @@
 Installation
 ====================
-Download dependency-check-ant from [bintray here](http://dl.bintray.com/jeremy-long/owasp/dependency-check-ant-1.0.0.jar).
+Download dependency-check-ant from [bintray here](http://dl.bintray.com/jeremy-long/owasp/dependency-check-ant-${project.version}.jar).
 To install dependency-check-ant place the dependency-check-ant-${project.version}.jar into
 the lib directory of your Ant instalation directory. Once installed you can add
 the taskdef to you build.xml and add the task to a new or existing target.

dependency-check-ant/src/test/java/org/owasp/dependencycheck/taskdefs/DependencyCheckTaskTest.java

@@ -26,6 +26,7 @@ import org.junit.Before;
 import org.junit.BeforeClass;
 import org.junit.Test;
 import org.apache.tools.ant.BuildFileTest;
+import org.owasp.dependencycheck.data.nvdcve.BaseDBTestCase;
 
 /**
  *
@@ -46,7 +47,8 @@ public class DependencyCheckTaskTest extends BuildFileTest {
 
     @Before
     @Override
-    public void setUp() {
+    public void setUp() throws Exception {
+        BaseDBTestCase.ensureDBExists();
         final String buildFile = this.getClass().getClassLoader().getResource("build.xml").getPath();
         configureProject(buildFile);
     }

dependency-check-cli/README.md

@@ -1,29 +1,24 @@
-Dependency-Check
+Dependency-Check Command Line
 ================
-
-Dependency-Check is a utility that attempts to detect publicly disclosed vulnerabilities contained within project dependencies. It does this by determining if there is a Common Platform Enumeration (CPE) identifier for a given dependency. If found, it will generate a report linking to the associated CVE entries.
+Dependency-Check Command Line can be used to check project dependencies for published security vulnerabilities. The checks
+performed are a "best effort" and as such, there could be false positives as well as false negatives. However,
+vulnerabilities in 3rd party components is a well-known problem and is currently documented in the 2013 OWASP
+Top 10 as [A9 - Using Components with Known Vulnerabilities](https://www.owasp.org/index.php/Top_10_2013-A9-Using_Components_with_Known_Vulnerabilities).
 
 Documentation and links to production binary releases can be found on the [github pages](http://jeremylong.github.io/DependencyCheck/dependency-check-cli/installation.html).
 
 Mailing List
 ------------
 
-Subscribe: [dependency-check+subscribe@googlegroups.com] [subscribe]
+Subscribe: [dependency-check+subscribe@googlegroups.com](mailto:dependency-check+subscribe@googlegroups.com)
 
-Post: [dependency-check@googlegroups.com] [post]
+Post: [dependency-check@googlegroups.com](mailto:dependency-check@googlegroups.com)
 
 Copyright & License
 ------------
 
 Dependency-Check is Copyright (c) 2012-2013 Jeremy Long. All Rights Reserved.
 
-Permission to modify and redistribute is granted under the terms of the GPLv3 license. See the [LICENSE.txt] [GPLv3] file for the full license.
+Permission to modify and redistribute is granted under the terms of the GPLv3 license. See the [LICENSE.txt](https://github.com/jeremylong/DependencyCheck/dependency-check-cli/blob/master/LICENSE.txt) file for the full license.
 
-Dependency-Check makes use of several other open source libraries. Please see the [NOTICE.txt] [notices] file for more information.
-
-
-  [wiki]: https://github.com/jeremylong/DependencyCheck/wiki
-  [subscribe]: mailto:dependency-check+subscribe@googlegroups.com
-  [post]: mailto:dependency-check@googlegroups.com
-  [GPLv3]: https://github.com/jeremylong/DependencyCheck/blob/master/LICENSE.txt
-  [notices]: https://github.com/jeremylong/DependencyCheck/blob/master/NOTICES.txt
+Dependency-Check Command Line makes use of other open source libraries. Please see the [NOTICE.txt](https://github.com/jeremylong/DependencyCheck/dependency-check-cli/blob/master/NOTICES.txt) file for more information.

dependency-check-cli/pom.xml

@@ -22,13 +22,13 @@ Copyright (c) 2012 - Jeremy Long. All Rights Reserved.
     <parent>
         <groupId>org.owasp</groupId>
         <artifactId>dependency-check-parent</artifactId>
-        <version>1.0.1</version>
+        <version>1.0.7</version>
     </parent>
 
     <artifactId>dependency-check-cli</artifactId>
     <packaging>jar</packaging>
 
-    <name>dependency-check-cli</name>
+    <name>Dependency-Check Command Line</name>
     <description>Dependency-Check-Maven is a Maven Plugin that attempts to detect publicly disclosed vulnerabilities contained within project dependencies. It does this by determining if there is a Common Platform Enumeration (CPE) identifier for a given dependency. If found, it will generate a report linking to the associated CVE entries.</description>
     <!-- begin copy from http://minds.coremedia.com/2012/09/11/problem-solved-deploy-multi-module-maven-project-site-as-github-pages/ -->
     <distributionManagement>

dependency-check-cli/src/main/java/org/owasp/dependencycheck/App.java

@@ -24,12 +24,12 @@ import java.io.IOException;
 import java.io.InputStream;
 import java.util.List;
 import java.util.logging.Level;
-import java.util.logging.LogManager;
 import java.util.logging.Logger;
 import org.apache.commons.cli.ParseException;
 import org.owasp.dependencycheck.reporting.ReportGenerator;
 import org.owasp.dependencycheck.dependency.Dependency;
 import org.owasp.dependencycheck.cli.CliParser;
+import org.owasp.dependencycheck.utils.LogUtils;
 import org.owasp.dependencycheck.utils.Settings;
 
 /*
@@ -67,35 +67,10 @@ public class App {
      * @param args the command line arguments
      */
     public static void main(String[] args) {
-        prepareLogger();
         final App app = new App();
         app.run(args);
     }
 
-    /**
-     * Configures the logger for use by the application.
-     */
-    private static void prepareLogger() {
-        InputStream in = null;
-        try {
-            in = App.class.getClassLoader().getResourceAsStream(LOG_PROPERTIES_FILE);
-            LogManager.getLogManager().reset();
-            LogManager.getLogManager().readConfiguration(in);
-        } catch (IOException ex) {
-            Logger.getLogger(App.class.getName()).log(Level.FINE, "IO Error preparing the logger", ex);
-        } catch (SecurityException ex) {
-            Logger.getLogger(App.class.getName()).log(Level.FINE, "Error preparing the logger", ex);
-        } finally {
-            if (in != null) {
-                try {
-                    in.close();
-                } catch (Exception ex) {
-                    Logger.getLogger(App.class.getName()).log(Level.FINEST, "Error closing resource stream", ex);
-                }
-            }
-        }
-    }
-
     /**
      * Main CLI entry-point into the application.
      *
@@ -116,10 +91,15 @@ public class App {
             return;
         }
 
+        final InputStream in = App.class.getClassLoader().getResourceAsStream(LOG_PROPERTIES_FILE);
+        LogUtils.prepareLogger(in, cli.getVerboseLog());
+
         if (cli.isGetVersion()) {
             cli.printVersionInfo();
         } else if (cli.isRunScan()) {
-            updateSettings(cli.isAutoUpdate(), cli.getConnectionTimeout(), cli.getProxyUrl(), cli.getProxyPort(), cli.getDataDirectory());
+            updateSettings(cli.isAutoUpdate(), cli.getConnectionTimeout(), cli.getProxyUrl(),
+                    cli.getProxyPort(), cli.getProxyUsername(), cli.getProxyPassword(),
+                    cli.getDataDirectory(), cli.getPropertiesFile(), cli.getSuppressionFile());
             runScan(cli.getReportDirectory(), cli.getReportFormat(), cli.getApplicationName(), cli.getScanFiles());
         } else {
             cli.printHelp();
@@ -167,9 +147,29 @@ public class App {
      * @param proxyUrl the proxy url (null or blank means no proxy will be used)
      * @param proxyPort the proxy port (null or blank means no port will be
      * used)
+     * @param proxyUser the proxy user name
+     * @param proxyPass the password for the proxy
      * @param dataDirectory the directory to store/retrieve persistent data from
+     * @param propertiesFile the properties file to utilize
+     * @param suppressionFile the path to the suppression file
      */
-    private void updateSettings(boolean autoUpdate, String connectionTimeout, String proxyUrl, String proxyPort, String dataDirectory) {
+    private void updateSettings(boolean autoUpdate, String connectionTimeout, String proxyUrl, String proxyPort,
+            String proxyUser, String proxyPass, String dataDirectory, File propertiesFile,
+            String suppressionFile) {
+
+        if (propertiesFile != null) {
+            try {
+                Settings.mergeProperties(propertiesFile);
+            } catch (FileNotFoundException ex) {
+                final String msg = String.format("Unable to load properties file '%s'", propertiesFile.getPath());
+                Logger.getLogger(App.class.getName()).log(Level.SEVERE, msg);
+                Logger.getLogger(App.class.getName()).log(Level.FINE, null, ex);
+            } catch (IOException ex) {
+                final String msg = String.format("Unable to find properties file '%s'", propertiesFile.getPath());
+                Logger.getLogger(App.class.getName()).log(Level.SEVERE, msg);
+                Logger.getLogger(App.class.getName()).log(Level.FINE, null, ex);
+            }
+        }
         if (dataDirectory != null) {
             Settings.setString(Settings.KEYS.DATA_DIRECTORY, dataDirectory);
         } else if (System.getProperty("basedir") != null) {
@@ -182,8 +182,6 @@ public class App {
             final File dataDir = new File(base, sub);
             Settings.setString(Settings.KEYS.DATA_DIRECTORY, dataDir.getAbsolutePath());
         }
-
-
         Settings.setBoolean(Settings.KEYS.AUTO_UPDATE, autoUpdate);
         if (proxyUrl != null && !proxyUrl.isEmpty()) {
             Settings.setString(Settings.KEYS.PROXY_URL, proxyUrl);
@@ -191,8 +189,17 @@ public class App {
         if (proxyPort != null && !proxyPort.isEmpty()) {
             Settings.setString(Settings.KEYS.PROXY_PORT, proxyPort);
         }
+        if (proxyUser != null && !proxyUser.isEmpty()) {
+            Settings.setString(Settings.KEYS.PROXY_USERNAME, proxyUser);
+        }
+        if (proxyPass != null && !proxyPass.isEmpty()) {
+            Settings.setString(Settings.KEYS.PROXY_PASSWORD, proxyPass);
+        }
         if (connectionTimeout != null && !connectionTimeout.isEmpty()) {
             Settings.setString(Settings.KEYS.CONNECTION_TIMEOUT, connectionTimeout);
         }
+        if (suppressionFile != null && !suppressionFile.isEmpty()) {
+            Settings.setString(Settings.KEYS.SUPPRESSION_FILE, suppressionFile);
+        }
     }
 }

dependency-check-cli/src/main/java/org/owasp/dependencycheck/cli/CliParser.java

@@ -175,6 +175,14 @@ public final class CliParser {
                 .withDescription("The proxy port to use when downloading resources.")
                 .create(ArgumentName.PROXY_PORT_SHORT);
 
+        final Option proxyUsername = OptionBuilder.withArgName("user").hasArg().withLongOpt(ArgumentName.PROXY_USERNAME)
+                .withDescription("The proxy username to use when downloading resources.")
+                .create(ArgumentName.PROXY_USERNAME_SHORT);
+
+        final Option proxyPassword = OptionBuilder.withArgName("pass").hasArg().withLongOpt(ArgumentName.PROXY_PASSWORD)
+                .withDescription("The proxy password to use when downloading resources.")
+                .create(ArgumentName.PROXY_PASSWORD_SHORT);
+
         final Option path = OptionBuilder.withArgName("path").hasArg().withLongOpt(ArgumentName.SCAN)
                 .withDescription("The path to scan - this option can be specified multiple times.")
                 .create(ArgumentName.SCAN_SHORT);
@@ -195,6 +203,15 @@ public final class CliParser {
                 .withDescription("The output format to write to (XML, HTML, VULN, ALL). The default is HTML.")
                 .create(ArgumentName.OUTPUT_FORMAT_SHORT);
 
+        final Option verboseLog = OptionBuilder.withArgName("file").hasArg().withLongOpt(ArgumentName.VERBOSE_LOG)
+                .withDescription("The file path to write verbose logging information.")
+                .create(ArgumentName.VERBOSE_LOG_SHORT);
+
+        final Option suppressionFile = OptionBuilder.withArgName("file").hasArg().withLongOpt(ArgumentName.SUPPRESION_FILE)
+                .withDescription("The file path to the suppression XML file.")
+                .create(ArgumentName.SUPPRESION_FILE_SHORT);
+
+
         final OptionGroup og = new OptionGroup();
         og.addOption(path);
 
@@ -208,8 +225,12 @@ public final class CliParser {
         opts.addOption(noUpdate);
         opts.addOption(props);
         opts.addOption(data);
+        opts.addOption(verboseLog);
+        opts.addOption(suppressionFile);
         opts.addOption(proxyPort);
         opts.addOption(proxyUrl);
+        opts.addOption(proxyUsername);
+        opts.addOption(proxyPassword);
         opts.addOption(connectionTimeout);
 
         return opts;
@@ -325,6 +346,24 @@ public final class CliParser {
         return line.getOptionValue(ArgumentName.PROXY_PORT);
     }
 
+    /**
+     * Returns the proxy username.
+     *
+     * @return the proxy username
+     */
+    public String getProxyUsername() {
+        return line.getOptionValue(ArgumentName.PROXY_USERNAME);
+    }
+
+    /**
+     * Returns the proxy password.
+     *
+     * @return the proxy password
+     */
+    public String getProxyPassword() {
+        return line.getOptionValue(ArgumentName.PROXY_PASSWORD);
+    }
+
     /**
      * Get the value of dataDirectory.
      *
@@ -334,6 +373,37 @@ public final class CliParser {
         return line.getOptionValue(ArgumentName.DATA_DIRECTORY);
     }
 
+    /**
+     * Returns the properties file specified on the command line.
+     *
+     * @return the properties file specified on the command line
+     */
+    public File getPropertiesFile() {
+        final String path = line.getOptionValue(ArgumentName.PROP);
+        if (path != null) {
+            return new File(path);
+        }
+        return null;
+    }
+
+    /**
+     * Returns the path to the verbose log file.
+     *
+     * @return the path to the verbose log file
+     */
+    public String getVerboseLog() {
+        return line.getOptionValue(ArgumentName.VERBOSE_LOG);
+    }
+
+    /**
+     * Returns the path to the suppression file.
+     *
+     * @return the path to the suppression file
+     */
+    public String getSuppressionFile() {
+        return line.getOptionValue(ArgumentName.SUPPRESION_FILE);
+    }
+
     /**
      * <p>Prints the manifest information to standard output.</p>
      * <ul><li>Implementation-Title: ${pom.name}</li>
@@ -443,11 +513,27 @@ public final class CliParser {
          */
         public static final String PROXY_URL = "proxyurl";
         /**
-         * The short CLI argument name indicating the proxy url.
+         * The short CLI argument name indicating the proxy username.
+         */
+        public static final String PROXY_USERNAME_SHORT = "pu";
+        /**
+         * The CLI argument name indicating the proxy username.
+         */
+        public static final String PROXY_USERNAME = "proxyuser";
+        /**
+         * The short CLI argument name indicating the proxy password.
+         */
+        public static final String PROXY_PASSWORD_SHORT = "pp";
+        /**
+         * The CLI argument name indicating the proxy password.
+         */
+        public static final String PROXY_PASSWORD = "proxypass";
+        /**
+         * The short CLI argument name indicating the connection timeout.
          */
         public static final String CONNECTION_TIMEOUT_SHORT = "c";
         /**
-         * The CLI argument name indicating the proxy url.
+         * The CLI argument name indicating the connection timeout.
          */
         public static final String CONNECTION_TIMEOUT = "connectiontimeout";
         /**
@@ -469,5 +555,24 @@ public final class CliParser {
          * directory.
          */
         public static final String DATA_DIRECTORY_SHORT = "d";
+        /**
+         * The CLI argument name for setting the location of the data directory.
+         */
+        public static final String VERBOSE_LOG = "log";
+        /**
+         * The short CLI argument name for setting the location of the data
+         * directory.
+         */
+        public static final String VERBOSE_LOG_SHORT = "l";
+        /**
+         * The CLI argument name for setting the location of the suppression
+         * file.
+         */
+        public static final String SUPPRESION_FILE = "suppression";
+        /**
+         * The short CLI argument name for setting the location of the
+         * suppression file.
+         */
+        public static final String SUPPRESION_FILE_SHORT = "sf";
     }
 }

dependency-check-cli/src/main/resources/log.properties

@@ -7,8 +7,6 @@ handlers=java.util.logging.ConsoleHandler
 # Configure the ConsoleHandler.
 java.util.logging.ConsoleHandler.level=INFO
 
-org.owasp.dependencycheck.data.nvdcve.xml
-
 # Configure the FileHandler.
 java.util.logging.FileHandler.formatter=java.util.logging.SimpleFormatter
 java.util.logging.FileHandler.level=FINE
@@ -21,4 +19,4 @@ java.util.logging.FileHandler.level=FINE
 #   %g - generation number for rotating logs
 #   %u - unique number to avoid conflicts
 # FileHandler writes to %h/demo0.log by default.
-java.util.logging.FileHandler.pattern=./logs/DependencyCheck.log
+java.util.logging.FileHandler.pattern=./dependency-check.log

dependency-check-cli/src/site/markdown/arguments.md

@@ -0,0 +1,22 @@
+Command Line Arguments
+====================
+
+The following table lists the command line arguments:
+
+Short  | Argument Name         | Parameter   | Description | Requirement
+-------|-----------------------|-------------|-------------|------------
+ \-a   | \-\-app               | \<name\>    | The name of the application being scanned. This is a required argument. |
+ \-c   | \-\-connectiontimeout | \<timeout\> | The connection timeout (in milliseconds) to use when downloading resources. | Optional
+ \-d   | \-\-data              | \<path\>    | The location of the data directory used to store persistent data. This option should generally not be set. | Optional
+ \-f   | \-\-format            | \<format\>  | The output format to write to (XML, HTML, VULN, ALL). The default is HTML. |
+ \-h   | \-\-help              |             | Print the help message. | Optional
+ \-l   | \-\-log               | \<file\>    | The file path to write verbose logging information. | Optional
+ \-n   | \-\-noupdate          |             | Disables the automatic updating of the CPE data. | Optional
+ \-o   | \-\-out               | \<folder\>  | The folder to write reports to. This defaults to the current directory. | Optional
+ \-p   | \-\-proxyport         | \<port\>    | The proxy port to use when downloading resources. | Optional
+ \-pp  | \-\-proxypass         | \<pass\>    | The proxy password to use when downloading resources. | Optional
+ \-pu  | \-\-proxyuser         | \<user\>    | The proxy username to use when downloading resources. | Optional
+ \-s   | \-\-scan              | \<path\>    | The path to scan \- this option can be specified multiple times. |
+ \-sf  | \-\-suppression       | \<file\>    | The file path to the suppression XML file. | Optional
+ \-u   | \-\-proxyurl          | \<url\>     | The proxy url to use when downloading resources. | Optional
+ \-v   | \-\-version           |             | Print the version information. | Optional

dependency-check-cli/src/site/markdown/installation.md.vm

@@ -1,6 +1,6 @@
 Installation & Usage
 --------------------
-Downlod the dependency-check command line tool [here](http://dl.bintray.com/jeremy-long/owasp/dependency-check-1.0.0-release.zip).
+Downlod the dependency-check command line tool [here](http://dl.bintray.com/jeremy-long/owasp/dependency-check-${project.version}-release.zip).
 Extract the zip file to a location on your computer and put the 'bin' directory into the
 path environment variable. On \*nix systems you will likely need to make the shell
 script executable:
@@ -13,4 +13,11 @@ To scan a folder on the system you can run:
     dependency-check.bat --app "My App Name" --scan "c:\java\application\lib"
 
 ### \*nix
-    dependency-check.sh --app "My App Name" --scan "/java/application/lib"
+    dependency-check.sh --app "My App Name" --scan "/java/application/lib"
+
+To view the command line arguments, see the <a href="arguments.html">arguments page</a>, or you can run:
+### Windows
+    dependency-check.bat --help
+
+### \*nix
+    dependency-check.sh --help

dependency-check-cli/src/site/site.xml

@@ -27,6 +27,7 @@ Copyright (c) 2013 Jeremy Long. All Rights Reserved.
         </breadcrumbs>
         <menu name="Getting Started">
             <item name="Installation" href="installation.html"/>
+            <item name="Configuration" href="arguments.html"/>
         </menu>
         <menu ref="Project Documentation" />
         <menu ref="reports" />

dependency-check-core/pom.xml

@@ -22,13 +22,13 @@ along with Dependency-Check.  If not, see <http://www.gnu.org/licenses />.
     <parent>
         <groupId>org.owasp</groupId>
         <artifactId>dependency-check-parent</artifactId>
-        <version>1.0.1</version>
+        <version>1.0.7</version>
     </parent>
 
     <artifactId>dependency-check-core</artifactId>
     <packaging>jar</packaging>
 
-    <name>dependency-check-core</name>
+    <name>Dependency-Check Core</name>
     <!-- begin copy from http://minds.coremedia.com/2012/09/11/problem-solved-deploy-multi-module-maven-project-site-as-github-pages/ -->
     <distributionManagement>
         <site>
@@ -44,6 +44,7 @@ along with Dependency-Check.  If not, see <http://www.gnu.org/licenses />.
                 <directory>src/main/resources</directory>
                 <includes>
                     <include>**/*.properties</include>
+                    <include>**/schema/*.xsd</include>
                 </includes>
                 <filtering>true</filtering>
             </resource>
@@ -83,16 +84,16 @@ along with Dependency-Check.  If not, see <http://www.gnu.org/licenses />.
             <plugin>
                 <groupId>org.apache.maven.plugins</groupId>
                 <artifactId>maven-dependency-plugin</artifactId>
-                <version>2.7</version>
+                <version>2.8</version>
                 <executions>
                     <execution>
-                        <phase>package</phase>
+                        <phase>generate-resources</phase>
                         <goals>
                             <goal>copy-dependencies</goal>
                         </goals>
                         <configuration>
-                            <outputDirectory>${project.build.directory}/lib</outputDirectory>
-                            <excludeScope>provided</excludeScope>
+                            <outputDirectory>${project.build.directory}/test-classes</outputDirectory>
+                            <includeScope>provided</includeScope>
                         </configuration>
                     </execution>
                 </executions>
@@ -101,6 +102,22 @@ along with Dependency-Check.  If not, see <http://www.gnu.org/licenses />.
                 <groupId>org.apache.maven.plugins</groupId>
                 <artifactId>maven-jar-plugin</artifactId>
                 <version>2.4</version>
+                <executions>
+                    <execution>
+                        <id>jar</id>
+                        <phase>package</phase>
+                        <goals>
+                            <goal>jar</goal>
+                        </goals>
+                    </execution>
+                    <execution>
+                        <id>test-jar</id>
+                        <phase>package</phase>
+                        <goals>
+                            <goal>test-jar</goal>
+                        </goals>
+                    </execution>
+                </executions>
                 <configuration>
                     <archive>
                         <manifest>
@@ -177,6 +194,10 @@ along with Dependency-Check.  If not, see <http://www.gnu.org/licenses />.
                             <name>data.directory</name>
                             <value>${project.build.directory}/data</value>
                         </property>
+                        <property>
+                            <name>temp.directory</name>
+                            <value>${project.build.directory}/temp</value>
+                        </property>
                     </systemProperties>
                     <excludes>
                         <exclude>**/*IntegrationTest.java</exclude>
@@ -369,7 +390,7 @@ along with Dependency-Check.  If not, see <http://www.gnu.org/licenses />.
             <groupId>com.google.code.findbugs</groupId>
             <artifactId>annotations</artifactId>
             <version>2.0.1</version>
-            <scope>provided</scope><!-- don't include this in the libs-->
+            <optional>true</optional>
         </dependency>
         <dependency>
             <groupId>commons-cli</groupId>
@@ -389,17 +410,17 @@ along with Dependency-Check.  If not, see <http://www.gnu.org/licenses />.
         <dependency>
             <groupId>org.apache.lucene</groupId>
             <artifactId>lucene-core</artifactId>
-            <version>4.3.1</version>
+            <version>4.5.1</version>
         </dependency>
         <dependency>
             <groupId>org.apache.lucene</groupId>
             <artifactId>lucene-analyzers-common</artifactId>
-            <version>4.3.1</version>
+            <version>4.5.1</version>
         </dependency>
         <dependency>
             <groupId>org.apache.lucene</groupId>
             <artifactId>lucene-queryparser</artifactId>
-            <version>4.3.1</version>
+            <version>4.5.1</version>
         </dependency>
         <dependency>
             <groupId>org.apache.velocity</groupId>
@@ -461,12 +482,91 @@ along with Dependency-Check.  If not, see <http://www.gnu.org/licenses />.
             <version>1.7.2</version>
             <type>jar</type>
         </dependency>
-        <!-- The following dependencies are only scanned during integration testing -->
-        <!--<dependency>
+        <dependency>
+            <groupId>org.apache.commons</groupId>
+            <artifactId>commons-compress</artifactId>
+            <version>1.5</version>
+        </dependency>
+        <!-- The following dependencies are only used during testing -->
+        <dependency>
+            <groupId>org.apache.maven.scm</groupId>
+            <artifactId>maven-scm-provider-cvsexe</artifactId>
+            <version>1.8.1</version>
+            <scope>provided</scope>
+            <optional>true</optional>
+        </dependency>
+        <dependency>
             <groupId>org.springframework</groupId>
             <artifactId>spring-webmvc</artifactId>
             <version>2.5.5</version>
-            <scope>test</scope>
-        </dependency>-->
+            <scope>provided</scope>
+            <optional>true</optional>
+        </dependency>
+        <dependency>
+            <groupId>com.hazelcast</groupId>
+            <artifactId>hazelcast</artifactId>
+            <version>2.5</version>
+            <scope>provided</scope>
+            <optional>true</optional>
+        </dependency>
+        <dependency>
+            <groupId>net.sf.ehcache</groupId>
+            <artifactId>ehcache-core</artifactId>
+            <version>2.2.0</version>
+            <scope>provided</scope>
+            <optional>true</optional>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.struts</groupId>
+            <artifactId>struts2-core</artifactId>
+            <version>2.1.2</version>
+            <scope>provided</scope>
+            <optional>true</optional>
+        </dependency>
+        <dependency>
+            <groupId>org.mortbay.jetty</groupId>
+            <artifactId>jetty</artifactId>
+            <version>6.1.0</version>
+            <scope>provided</scope>
+            <optional>true</optional>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.axis2</groupId>
+            <artifactId>axis2-spring</artifactId>
+            <version>1.4.1</version>
+            <scope>provided</scope>
+            <optional>true</optional>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.axis2</groupId>
+            <artifactId>axis2-adb</artifactId>
+            <version>1.4.1</version>
+            <scope>provided</scope>
+            <optional>true</optional>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.geronimo.daytrader</groupId>
+            <artifactId>daytrader-ear</artifactId>
+            <version>2.1.7</version>
+            <type>ear</type>
+            <scope>provided</scope>
+            <optional>true</optional>
+        </dependency>
+        <dependency>
+            <groupId>org.glassfish.main.admingui</groupId>
+            <artifactId>war</artifactId>
+            <version>4.0</version>
+            <type>war</type>
+            <scope>provided</scope>
+            <optional>true</optional>
+        </dependency>
+        <dependency>
+            <groupId>org.dojotoolkit</groupId>
+            <artifactId>dojo-war</artifactId>
+            <version>1.3.0</version>
+            <type>war</type>
+            <scope>provided</scope>
+            <optional>true</optional>
+        </dependency>
     </dependencies>
 </project>

dependency-check-core/src/main/java/org/owasp/dependencycheck/Engine.java

@@ -20,6 +20,8 @@ package org.owasp.dependencycheck;
 
 import java.util.EnumMap;
 import java.io.File;
+import java.io.IOException;
+import java.sql.SQLException;
 import java.util.ArrayList;
 import java.util.HashSet;
 import java.util.Iterator;
@@ -32,8 +34,13 @@ import org.owasp.dependencycheck.analyzer.AnalysisPhase;
 import org.owasp.dependencycheck.analyzer.Analyzer;
 import org.owasp.dependencycheck.analyzer.AnalyzerService;
 import org.owasp.dependencycheck.data.CachedWebDataSource;
+import org.owasp.dependencycheck.data.NoDataException;
 import org.owasp.dependencycheck.data.UpdateException;
 import org.owasp.dependencycheck.data.UpdateService;
+import org.owasp.dependencycheck.data.cpe.CpeMemoryIndex;
+import org.owasp.dependencycheck.data.cpe.IndexException;
+import org.owasp.dependencycheck.data.nvdcve.CveDB;
+import org.owasp.dependencycheck.data.nvdcve.DatabaseException;
 import org.owasp.dependencycheck.dependency.Dependency;
 import org.owasp.dependencycheck.utils.FileUtils;
 import org.owasp.dependencycheck.utils.InvalidSettingException;
@@ -277,16 +284,28 @@ public class Engine {
      * Runs the analyzers against all of the dependencies.
      */
     public void analyzeDependencies() {
+        //need to ensure that data exists
+        try {
+            ensureDataExists();
+        } catch (NoDataException ex) {
+            final String msg = String.format("%n%n%s%n%nUnable to continue dependency-check analysis.", ex.getMessage());
+            Logger.getLogger(Engine.class.getName()).log(Level.SEVERE, msg);
+            Logger.getLogger(Engine.class.getName()).log(Level.FINE, null, ex);
+            return;
+        }
+
         //phase one initialize
         for (AnalysisPhase phase : AnalysisPhase.values()) {
             final List<Analyzer> analyzerList = analyzers.get(phase);
             for (Analyzer a : analyzerList) {
                 try {
+                    final String msg = String.format("Initializing %s", a.getName());
+                    Logger.getLogger(Engine.class.getName()).log(Level.FINE, msg);
                     a.initialize();
                 } catch (Exception ex) {
-                    final String msg = String.format("\"Exception occurred initializing \"%s\".\"", a.getName());
+                    final String msg = String.format("Exception occurred initializing %s.", a.getName());
                     Logger.getLogger(Engine.class.getName()).log(Level.SEVERE, msg);
-                    Logger.getLogger(Engine.class.getName()).log(Level.INFO, msg, ex);
+                    Logger.getLogger(Engine.class.getName()).log(Level.INFO, null, ex);
                     try {
                         a.close();
                     } catch (Exception ex1) {
@@ -305,9 +324,13 @@ public class Engine {
                  * analyzers may modify it. This prevents ConcurrentModificationExceptions.
                  * This is okay for adds/deletes because it happens per analyzer.
                  */
+                final String msg = String.format("Begin Analyzer '%s'", a.getName());
+                Logger.getLogger(Engine.class.getName()).log(Level.FINE, msg);
                 final Set<Dependency> dependencySet = new HashSet<Dependency>();
                 dependencySet.addAll(dependencies);
                 for (Dependency d : dependencySet) {
+                    final String msgFile = String.format("Begin Analysis of '%s'", d.getActualFilePath());
+                    Logger.getLogger(Engine.class.getName()).log(Level.FINE, msgFile);
                     if (a.supportsExtension(d.getFileExtension())) {
                         try {
                             a.analyze(d, this);
@@ -323,6 +346,8 @@ public class Engine {
         for (AnalysisPhase phase : AnalysisPhase.values()) {
             final List<Analyzer> analyzerList = analyzers.get(phase);
             for (Analyzer a : analyzerList) {
+                final String msg = String.format("Closing Analyzer '%s'", a.getName());
+                Logger.getLogger(Engine.class.getName()).log(Level.FINE, msg);
                 try {
                     a.close();
                 } catch (Exception ex) {
@@ -388,4 +413,36 @@ public class Engine {
         }
         return false;
     }
+
+    /**
+     * Checks the CPE Index to ensure documents exists. If none exist a
+     * NoDataException is thrown.
+     *
+     * @throws NoDataException thrown if no data exists in the CPE Index
+     */
+    private void ensureDataExists() throws NoDataException {
+        final CpeMemoryIndex cpe = CpeMemoryIndex.getInstance();
+        final CveDB cve = new CveDB();
+
+        try {
+            cve.open();
+            cpe.open(cve);
+        } catch (IndexException ex) {
+            throw new NoDataException(ex);
+        } catch (IOException ex) {
+            throw new NoDataException(ex);
+        } catch (SQLException ex) {
+            throw new NoDataException(ex);
+        } catch (DatabaseException ex) {
+            throw new NoDataException(ex);
+        } catch (ClassNotFoundException ex) {
+            throw new NoDataException(ex);
+        } finally {
+            cve.close();
+        }
+        if (cpe.numDocs() <= 0) {
+            cpe.close();
+            throw new NoDataException();
+        }
+    }
 }

dependency-check-core/src/main/java/org/owasp/dependencycheck/analyzer/AbstractSuppressionAnalyzer.java

@@ -0,0 +1,115 @@
+/*
+ * This file is part of dependency-check-core.
+ *
+ * Dependency-check-core is free software: you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation, either version 3 of the License, or (at your option) any
+ * later version.
+ *
+ * Dependency-check-core is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+ * details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * dependency-check-core. If not, see http://www.gnu.org/licenses/.
+ *
+ * Copyright (c) 2013 Jeremy Long. All Rights Reserved.
+ */
+package org.owasp.dependencycheck.analyzer;
+
+import java.io.File;
+import java.util.List;
+import java.util.Set;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+import org.owasp.dependencycheck.suppression.SuppressionParseException;
+import org.owasp.dependencycheck.suppression.SuppressionParser;
+import org.owasp.dependencycheck.suppression.SuppressionRule;
+import org.owasp.dependencycheck.utils.Settings;
+
+/**
+ * Abstract base suppression analyzer that contains methods for parsing the
+ * suppression xml file.
+ *
+ * @author Jeremy Long (jeremy.long@owasp.org)
+ */
+public abstract class AbstractSuppressionAnalyzer extends AbstractAnalyzer {
+
+    //<editor-fold defaultstate="collapsed" desc="All standard implmentation details of Analyzer">
+    /**
+     * Returns a list of file EXTENSIONS supported by this analyzer.
+     *
+     * @return a list of file EXTENSIONS supported by this analyzer.
+     */
+    public Set<String> getSupportedExtensions() {
+        return null;
+    }
+
+    /**
+     * Returns whether or not this analyzer can process the given extension.
+     *
+     * @param extension the file extension to test for support.
+     * @return whether or not the specified file extension is supported by this
+     * analyzer.
+     */
+    @Override
+    public boolean supportsExtension(String extension) {
+        return true;
+    }
+
+    //</editor-fold>
+    /**
+     * The initialize method loads the suppression XML file.
+     *
+     * @throws Exception thrown if there is an exception
+     */
+    @Override
+    public void initialize() throws Exception {
+        super.initialize();
+        loadSuppressionData();
+    }
+    /**
+     * The list of suppression rules
+     */
+    private List<SuppressionRule> rules;
+
+    /**
+     * Get the value of rules.
+     *
+     * @return the value of rules
+     */
+    public List<SuppressionRule> getRules() {
+        return rules;
+    }
+
+    /**
+     * Set the value of rules.
+     *
+     * @param rules new value of rules
+     */
+    public void setRules(List<SuppressionRule> rules) {
+        this.rules = rules;
+    }
+
+    /**
+     * Loads the suppression rules file.
+     *
+     * @throws SuppressionParseException thrown if the XML cannot be parsed.
+     */
+    private void loadSuppressionData() throws SuppressionParseException {
+        final File file = Settings.getFile(Settings.KEYS.SUPPRESSION_FILE);
+        if (file != null) {
+            final SuppressionParser parser = new SuppressionParser();
+            try {
+                rules = parser.parseSuppressionRules(file);
+            } catch (SuppressionParseException ex) {
+                final String msg = String.format("Unable to parse suppression xml file '%s'", file.getPath());
+                Logger.getLogger(AbstractSuppressionAnalyzer.class.getName()).log(Level.WARNING, msg);
+                Logger.getLogger(AbstractSuppressionAnalyzer.class.getName()).log(Level.WARNING, ex.getMessage());
+                Logger.getLogger(AbstractSuppressionAnalyzer.class.getName()).log(Level.FINE, null, ex);
+                throw ex;
+            }
+        }
+    }
+}

dependency-check-core/src/main/java/org/owasp/dependencycheck/analyzer/AnalysisException.java

@@ -56,7 +56,7 @@ public class AnalysisException extends Exception {
     }
 
     /**
-     * Creates a new DownloadFailedException.
+     * Creates a new AnalysisException.
      *
      * @param msg a message for the exception.
      * @param ex the cause of the failure.

dependency-check-core/src/main/java/org/owasp/dependencycheck/analyzer/ArchiveAnalyzer.java

@@ -32,26 +32,21 @@ import java.util.List;
 import java.util.Set;
 import java.util.logging.Level;
 import java.util.logging.Logger;
-import java.util.zip.ZipEntry;
-import java.util.zip.ZipInputStream;
+import org.apache.commons.compress.archivers.ArchiveEntry;
+import org.apache.commons.compress.archivers.ArchiveInputStream;
+import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;
+import org.apache.commons.compress.archivers.zip.ZipArchiveInputStream;
+import org.apache.commons.compress.compressors.CompressorInputStream;
+import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream;
+import org.apache.commons.compress.compressors.gzip.GzipUtils;
 import org.h2.store.fs.FileUtils;
 import org.owasp.dependencycheck.Engine;
 import org.owasp.dependencycheck.dependency.Dependency;
 import org.owasp.dependencycheck.utils.Settings;
 
 /**
- * <p>An analyzer that works on archive files:
- * <ul>
- * <li><b>ZIP</b> - if it is determined to be a JAR, WAR or EAR a copy is made
- * and the copy is given the correct extension so that it will be correctly
- * analyzed.</li>
- * <li><b>WAR</b> - the WAR contents are extracted and added as dependencies to
- * the scan. The displayed path is relative to the WAR.</li>
- * <li><b>EAR</b> - the WAR contents are extracted and added as dependencies to
- * the scan. Any WAR files are also processed so that the contained JAR files
- * are added to the list of dependencies. The displayed path is relative to the
- * EAR.</li>
- * </ul></p>
+ * <p>An analyzer that extracts files from archives and ensures any supported
+ * files contained within the archive are added to the dependency list.</p>
  *
  * @author Jeremy Long (jeremy.long@owasp.org)
  */
@@ -91,7 +86,7 @@ public class ArchiveAnalyzer extends AbstractAnalyzer implements Analyzer {
     /**
      * The set of file extensions supported by this analyzer.
      */
-    private static final Set<String> EXTENSIONS = newHashSet("zip", "ear", "war");
+    private static final Set<String> EXTENSIONS = newHashSet("zip", "ear", "war", "tar", "gz", "tgz");
 
     /**
      * Returns a list of file EXTENSIONS supported by this analyzer.
@@ -140,14 +135,21 @@ public class ArchiveAnalyzer extends AbstractAnalyzer implements Analyzer {
      */
     @Override
     public void initialize() throws Exception {
-        final String tmpDir = Settings.getString(Settings.KEYS.TEMP_DIRECTORY, System.getProperty("java.io.tmpdir"));
-        final File baseDir = new File(tmpDir);
+        final File baseDir = Settings.getTempDirectory();
+        if (!baseDir.exists()) {
+            if (!baseDir.mkdirs()) {
+                final String msg = String.format("Unable to make a temporary folder '%s'", baseDir.getPath());
+                throw new AnalysisException(msg);
+            }
+        }
         tempFileLocation = File.createTempFile("check", "tmp", baseDir);
         if (!tempFileLocation.delete()) {
-            throw new AnalysisException("Unable to delete temporary file '" + tempFileLocation.getAbsolutePath() + "'.");
+            final String msg = String.format("Unable to delete temporary file '%s'.", tempFileLocation.getAbsolutePath());
+            throw new AnalysisException(msg);
         }
         if (!tempFileLocation.mkdirs()) {
-            throw new AnalysisException("Unable to create directory '" + tempFileLocation.getAbsolutePath() + "'.");
+            final String msg = String.format("Unable to create directory '%s'.", tempFileLocation.getAbsolutePath());
+            throw new AnalysisException(msg);
         }
     }
 
@@ -223,8 +225,13 @@ public class ArchiveAnalyzer extends AbstractAnalyzer implements Analyzer {
     private File getNextTempDirectory() throws AnalysisException {
         dirCount += 1;
         final File directory = new File(tempFileLocation, String.valueOf(dirCount));
+        //getting an exception for some directories not being able to be created; might be because the directory already exists?
+        if (directory.exists()) {
+            return getNextTempDirectory();
+        }
         if (!directory.mkdirs()) {
-            throw new AnalysisException("Unable to create temp directory '" + directory.getAbsolutePath() + "'.");
+            final String msg = String.format("Unable to create temp directory '%s'.", directory.getAbsolutePath());
+            throw new AnalysisException(msg);
         }
         return directory;
     }
@@ -233,35 +240,75 @@ public class ArchiveAnalyzer extends AbstractAnalyzer implements Analyzer {
      * Extracts the contents of an archive into the specified directory.
      *
      * @param archive an archive file such as a WAR or EAR
-     * @param extractTo a directory to extract the contents to
+     * @param destination a directory to extract the contents to
      * @param engine the scanning engine
      * @throws AnalysisException thrown if the archive is not found
      */
-    private void extractFiles(File archive, File extractTo, Engine engine) throws AnalysisException {
-        if (archive == null || extractTo == null) {
+    private void extractFiles(File archive, File destination, Engine engine) throws AnalysisException {
+        if (archive == null || destination == null) {
             return;
         }
 
         FileInputStream fis = null;
-        ZipInputStream zis = null;
-
         try {
             fis = new FileInputStream(archive);
         } catch (FileNotFoundException ex) {
             Logger.getLogger(ArchiveAnalyzer.class.getName()).log(Level.INFO, null, ex);
             throw new AnalysisException("Archive file was not found.", ex);
         }
-        zis = new ZipInputStream(new BufferedInputStream(fis));
-        ZipEntry entry;
+        final String archiveExt = org.owasp.dependencycheck.utils.FileUtils.getFileExtension(archive.getName()).toLowerCase();
         try {
-            while ((entry = zis.getNextEntry()) != null) {
+            if ("zip".equals(archiveExt) || "war".equals(archiveExt) || "ear".equals(archiveExt)) {
+                extractArchive(new ZipArchiveInputStream(new BufferedInputStream(fis)), destination, engine);
+            } else if ("tar".equals(archiveExt)) {
+                extractArchive(new TarArchiveInputStream(new BufferedInputStream(fis)), destination, engine);
+            } else if ("gz".equals(archiveExt) || "tgz".equals(archiveExt)) {
+                final String uncompressedName = GzipUtils.getUncompressedFilename(archive.getName());
+                final String uncompressedExt = org.owasp.dependencycheck.utils.FileUtils.getFileExtension(uncompressedName).toLowerCase();
+                if (engine.supportsExtension(uncompressedExt)) {
+                    decompressFile(new GzipCompressorInputStream(new BufferedInputStream(fis)), new File(destination, uncompressedName));
+                }
+            }
+        } catch (ArchiveExtractionException ex) {
+            final String msg = String.format("Exception extracting archive '%s'.", archive.getName());
+            Logger.getLogger(ArchiveAnalyzer.class.getName()).log(Level.WARNING, msg);
+            Logger.getLogger(ArchiveAnalyzer.class.getName()).log(Level.FINE, null, ex);
+        } catch (IOException ex) {
+            final String msg = String.format("Exception reading archive '%s'.", archive.getName());
+            Logger.getLogger(ArchiveAnalyzer.class.getName()).log(Level.WARNING, msg);
+            Logger.getLogger(ArchiveAnalyzer.class.getName()).log(Level.FINE, null, ex);
+        } finally {
+            try {
+                fis.close();
+            } catch (IOException ex) {
+                Logger.getLogger(ArchiveAnalyzer.class.getName()).log(Level.FINEST, null, ex);
+            }
+        }
+    }
+
+    /**
+     * Extracts files from an archive.
+     *
+     * @param input the archive to extract files from
+     * @param destination the location to write the files too
+     * @param engine the dependency-check engine
+     * @throws ArchiveExtractionException thrown if there is an exception
+     * extracting files from the archive
+     */
+    private void extractArchive(ArchiveInputStream input, File destination, Engine engine) throws ArchiveExtractionException {
+        ArchiveEntry entry;
+        try {
+            while ((entry = input.getNextEntry()) != null) {
                 if (entry.isDirectory()) {
-                    final File d = new File(extractTo, entry.getName());
-                    if (!d.mkdirs()) {
-                        throw new AnalysisException("Unable to create '" + d.getAbsolutePath() + "'.");
+                    final File d = new File(destination, entry.getName());
+                    if (!d.exists()) {
+                        if (!d.mkdirs()) {
+                            final String msg = String.format("Unable to create '%s'.", d.getAbsolutePath());
+                            throw new AnalysisException(msg);
+                        }
                     }
                 } else {
-                    final File file = new File(extractTo, entry.getName());
+                    final File file = new File(destination, entry.getName());
                     final String ext = org.owasp.dependencycheck.utils.FileUtils.getFileExtension(file.getName());
                     if (engine.supportsExtension(ext)) {
                         BufferedOutputStream bos = null;
@@ -271,22 +318,27 @@ public class ArchiveAnalyzer extends AbstractAnalyzer implements Analyzer {
                             bos = new BufferedOutputStream(fos, BUFFER_SIZE);
                             int count;
                             final byte data[] = new byte[BUFFER_SIZE];
-                            while ((count = zis.read(data, 0, BUFFER_SIZE)) != -1) {
+                            while ((count = input.read(data, 0, BUFFER_SIZE)) != -1) {
                                 bos.write(data, 0, count);
                             }
                             bos.flush();
                         } catch (FileNotFoundException ex) {
-                            Logger.getLogger(ArchiveAnalyzer.class.getName()).log(Level.FINE, null, ex);
-                            throw new AnalysisException("Unable to find file '" + file.getName() + "'.", ex);
+                            Logger.getLogger(ArchiveAnalyzer.class
+                                    .getName()).log(Level.FINE, null, ex);
+                            final String msg = String.format("Unable to find file '%s'.", file.getName());
+                            throw new AnalysisException(msg, ex);
                         } catch (IOException ex) {
-                            Logger.getLogger(ArchiveAnalyzer.class.getName()).log(Level.FINE, null, ex);
-                            throw new AnalysisException("IO Exception while parsing file '" + file.getName() + "'.", ex);
+                            Logger.getLogger(ArchiveAnalyzer.class
+                                    .getName()).log(Level.FINE, null, ex);
+                            final String msg = String.format("IO Exception while parsing file '%s'.", file.getName());
+                            throw new AnalysisException(msg, ex);
                         } finally {
                             if (bos != null) {
                                 try {
                                     bos.close();
                                 } catch (IOException ex) {
-                                    Logger.getLogger(ArchiveAnalyzer.class.getName()).log(Level.FINEST, null, ex);
+                                    Logger.getLogger(ArchiveAnalyzer.class
+                                            .getName()).log(Level.FINEST, null, ex);
                                 }
                             }
                         }
@@ -294,14 +346,50 @@ public class ArchiveAnalyzer extends AbstractAnalyzer implements Analyzer {
                 }
             }
         } catch (IOException ex) {
-            final String msg = String.format("Exception reading archive '%s'.", archive.getName());
-            Logger.getLogger(ArchiveAnalyzer.class.getName()).log(Level.FINE, msg, ex);
-            throw new AnalysisException(msg, ex);
+            throw new ArchiveExtractionException(ex);
+        } catch (Throwable ex) {
+            throw new ArchiveExtractionException(ex);
         } finally {
-            try {
-                zis.close();
-            } catch (IOException ex) {
-                Logger.getLogger(ArchiveAnalyzer.class.getName()).log(Level.FINEST, null, ex);
+            if (input != null) {
+                try {
+                    input.close();
+                } catch (IOException ex) {
+                    Logger.getLogger(ArchiveAnalyzer.class.getName()).log(Level.FINEST, null, ex);
+                }
+            }
+        }
+    }
+
+    /**
+     * Decompresses a file.
+     *
+     * @param inputStream the compressed file
+     * @param outputFile the location to write the decompressed file
+     * @throws ArchiveExtractionException thrown if there is an exception
+     * decompressing the file
+     */
+    private void decompressFile(CompressorInputStream inputStream, File outputFile) throws ArchiveExtractionException {
+        FileOutputStream out = null;
+        try {
+            out = new FileOutputStream(outputFile);
+            final byte[] buffer = new byte[BUFFER_SIZE];
+            int n = 0;
+            while (-1 != (n = inputStream.read(buffer))) {
+                out.write(buffer, 0, n);
+            }
+        } catch (FileNotFoundException ex) {
+            Logger.getLogger(ArchiveAnalyzer.class.getName()).log(Level.FINE, null, ex);
+            throw new ArchiveExtractionException(ex);
+        } catch (IOException ex) {
+            Logger.getLogger(ArchiveAnalyzer.class.getName()).log(Level.FINE, null, ex);
+            throw new ArchiveExtractionException(ex);
+        } finally {
+            if (out != null) {
+                try {
+                    out.close();
+                } catch (IOException ex) {
+                    Logger.getLogger(ArchiveAnalyzer.class.getName()).log(Level.FINEST, null, ex);
+                }
             }
         }
     }

dependency-check-core/src/main/java/org/owasp/dependencycheck/analyzer/ArchiveExtractionException.java

@@ -0,0 +1,67 @@
+/*
+ * This file is part of dependency-check-core.
+ *
+ * Dependency-check-core is free software: you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation, either version 3 of the License, or (at your option) any
+ * later version.
+ *
+ * Dependency-check-core is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+ * details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * dependency-check-core. If not, see http://www.gnu.org/licenses/.
+ *
+ * Copyright (c) 2013 Jeremy Long. All Rights Reserved.
+ */
+package org.owasp.dependencycheck.analyzer;
+
+/**
+ * An exception thrown when files in an archive cannot be extracted.
+ *
+ * @author Jeremy Long (jeremy.long@owasp.org)
+ */
+public class ArchiveExtractionException extends Exception {
+
+    /**
+     * The serial version UID for serialization.
+     */
+    private static final long serialVersionUID = 1L;
+
+    /**
+     * Creates a new ArchiveExtractionException.
+     */
+    public ArchiveExtractionException() {
+        super();
+    }
+
+    /**
+     * Creates a new ArchiveExtractionException.
+     *
+     * @param msg a message for the exception.
+     */
+    public ArchiveExtractionException(String msg) {
+        super(msg);
+    }
+
+    /**
+     * Creates a new ArchiveExtractionException.
+     *
+     * @param ex the cause of the failure.
+     */
+    public ArchiveExtractionException(Throwable ex) {
+        super(ex);
+    }
+
+    /**
+     * Creates a new ArchiveExtractionException.
+     *
+     * @param msg a message for the exception.
+     * @param ex the cause of the failure.
+     */
+    public ArchiveExtractionException(String msg, Throwable ex) {
+        super(msg, ex);
+    }
+}

dependency-check-core/src/main/java/org/owasp/dependencycheck/analyzer/CPEAnalyzer.java

@@ -16,7 +16,7 @@
  *
  * Copyright (c) 2012 Jeremy Long. All Rights Reserved.
  */
-package org.owasp.dependencycheck.data.cpe;
+package org.owasp.dependencycheck.analyzer;
 
 import java.io.IOException;
 import java.io.UnsupportedEncodingException;
@@ -35,14 +35,15 @@ import org.apache.lucene.queryparser.classic.ParseException;
 import org.apache.lucene.search.ScoreDoc;
 import org.apache.lucene.search.TopDocs;
 import org.owasp.dependencycheck.Engine;
-import org.owasp.dependencycheck.analyzer.AnalysisException;
-import org.owasp.dependencycheck.analyzer.AnalysisPhase;
 import org.owasp.dependencycheck.data.lucene.LuceneUtils;
 import org.owasp.dependencycheck.dependency.Dependency;
 import org.owasp.dependencycheck.dependency.Evidence;
 import org.owasp.dependencycheck.dependency.Evidence.Confidence;
 import org.owasp.dependencycheck.dependency.EvidenceCollection;
-import org.owasp.dependencycheck.analyzer.Analyzer;
+import org.owasp.dependencycheck.data.cpe.CpeMemoryIndex;
+import org.owasp.dependencycheck.data.cpe.Fields;
+import org.owasp.dependencycheck.data.cpe.IndexEntry;
+import org.owasp.dependencycheck.data.cpe.IndexException;
 import org.owasp.dependencycheck.data.nvdcve.CveDB;
 import org.owasp.dependencycheck.data.nvdcve.DatabaseException;
 import org.owasp.dependencycheck.dependency.Identifier;
@@ -83,9 +84,9 @@ public class CPEAnalyzer implements Analyzer {
      */
     static final int STRING_BUILDER_BUFFER = 20;
     /**
-     * The CPE Index.
+     * The CPE in memory index.
      */
-    private Index cpe;
+    private CpeMemoryIndex cpe;
     /**
      * The CVE Database.
      */
@@ -100,8 +101,6 @@ public class CPEAnalyzer implements Analyzer {
      * usually occurs when the database is in use by another process.
      */
     public void open() throws IOException, DatabaseException {
-        cpe = new Index();
-        cpe.open();
         cve = new CveDB();
         try {
             cve.open();
@@ -112,36 +111,25 @@ public class CPEAnalyzer implements Analyzer {
             Logger.getLogger(CPEAnalyzer.class.getName()).log(Level.FINE, null, ex);
             throw new DatabaseException("Unable to open the cve db", ex);
         }
+        cpe = CpeMemoryIndex.getInstance();
+        try {
+            cpe.open(cve);
+        } catch (IndexException ex) {
+            Logger.getLogger(CPEAnalyzer.class.getName()).log(Level.SEVERE, null, ex);
+            throw new DatabaseException(ex);
+        }
     }
 
     /**
-     * Closes the data source.
+     * Closes the data sources.
      */
     @Override
     public void close() {
-        cpe.close();
-        cve.close();
-    }
-
-    /**
-     * Returns the status of the data source - is the index open.
-     *
-     * @return true or false.
-     */
-    public boolean isOpen() {
-        return (cpe != null) && cpe.isOpen();
-    }
-
-    /**
-     * Ensures that the Lucene index is closed.
-     *
-     * @throws Throwable when a throwable is thrown.
-     */
-    @Override
-    protected void finalize() throws Throwable {
-        super.finalize();
-        if (isOpen()) {
-            close();
+        if (cpe != null) {
+            cpe.close();
+        }
+        if (cve != null) {
+            cve.close();
         }
     }
 
@@ -162,7 +150,6 @@ public class CPEAnalyzer implements Analyzer {
         String vendors = addEvidenceWithoutDuplicateTerms("", dependency.getVendorEvidence(), vendorConf);
         String products = addEvidenceWithoutDuplicateTerms("", dependency.getProductEvidence(), productConf);
 
-        //boolean found = false;
         int ctr = 0;
         do {
             if (!vendors.isEmpty() && !products.isEmpty()) {
@@ -171,27 +158,20 @@ public class CPEAnalyzer implements Analyzer {
 
                 for (IndexEntry e : entries) {
                     if (verifyEntry(e, dependency)) {
-                        //found = true; // we found a vendor/product pair. Now find version from the cve db.
                         final String vendor = e.getVendor();
                         final String product = e.getProduct();
-                        // cve.getVersions(vendor, product);
                         determineIdentifiers(dependency, vendor, product);
                     }
                 }
             }
-            //if (!found) {
             vendorConf = reduceConfidence(vendorConf);
             if (dependency.getVendorEvidence().contains(vendorConf)) {
-                //vendors += " " + dependency.getVendorEvidence().toString(vendorConf);
                 vendors = addEvidenceWithoutDuplicateTerms(vendors, dependency.getVendorEvidence(), vendorConf);
             }
             productConf = reduceConfidence(productConf);
             if (dependency.getProductEvidence().contains(productConf)) {
-                //products += " " + dependency.getProductEvidence().toString(productConf);
                 products = addEvidenceWithoutDuplicateTerms(products, dependency.getProductEvidence(), productConf);
             }
-            //}
-            //} while (!found && (++ctr) < 4);
         } while ((++ctr) < 4);
     }
 
@@ -275,13 +255,22 @@ public class CPEAnalyzer implements Analyzer {
 
         final TopDocs docs = cpe.search(searchString, MAX_QUERY_RESULTS);
         for (ScoreDoc d : docs.scoreDocs) {
-            final Document doc = cpe.getDocument(d.doc);
-            final IndexEntry entry = new IndexEntry();
-            entry.setVendor(doc.get(Fields.VENDOR));
-            entry.setProduct(doc.get(Fields.PRODUCT));
-            entry.setSearchScore(d.score);
-            if (!ret.contains(entry)) {
-                ret.add(entry);
+            if (d.score >= 0.08) {
+                final Document doc = cpe.getDocument(d.doc);
+                final IndexEntry entry = new IndexEntry();
+                entry.setVendor(doc.get(Fields.VENDOR));
+                entry.setProduct(doc.get(Fields.PRODUCT));
+//                if (d.score < 0.08) {
+//                    System.out.print(entry.getVendor());
+//                    System.out.print(":");
+//                    System.out.print(entry.getProduct());
+//                    System.out.print(":");
+//                    System.out.println(d.score);
+//                }
+                entry.setSearchScore(d.score);
+                if (!ret.contains(entry)) {
+                    ret.add(entry);
+                }
             }
         }
         return ret;
@@ -439,6 +428,9 @@ public class CPEAnalyzer implements Analyzer {
         //</editor-fold>
 
         //TODO - likely need to change the split... not sure if this will work for CPE with special chars
+        if (text == null) {
+            return false;
+        }
         final String[] words = text.split("[\\s_-]");
         final List<String> list = new ArrayList<String>();
         String tempWord = null;
@@ -454,9 +446,10 @@ public class CPEAnalyzer implements Analyzer {
                 list.add(word);
             }
         }
-//        if (tempWord != null) {
-//            //for now ignore any last single letter words...
-//        }
+        if (tempWord != null && !list.isEmpty()) {
+            final String tmp = list.get(list.size() - 1) + tempWord;
+            list.add(tmp);
+        }
         boolean contains = true;
         for (String word : list) {
             contains &= ec.containsUsedString(word);

dependency-check-core/src/main/java/org/owasp/dependencycheck/analyzer/CpeSuppressionAnalyzer.java

@@ -0,0 +1,76 @@
+/*
+ * This file is part of dependency-check-core.
+ *
+ * Dependency-check-core is free software: you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation, either version 3 of the License, or (at your option) any
+ * later version.
+ *
+ * Dependency-check-core is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+ * details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * dependency-check-core. If not, see http://www.gnu.org/licenses/.
+ *
+ * Copyright (c) 2013 Jeremy Long. All Rights Reserved.
+ */
+package org.owasp.dependencycheck.analyzer;
+
+import org.owasp.dependencycheck.Engine;
+import org.owasp.dependencycheck.dependency.Dependency;
+import org.owasp.dependencycheck.suppression.SuppressionRule;
+
+/**
+ * The suppression analyzer processes an externally defined XML document that
+ * complies with the suppressions.xsd schema. Any identified CPE entries within
+ * the dependencies that match will be removed.
+ *
+ * @author Jeremy Long (jeremy.long@owasp.org)
+ */
+public class CpeSuppressionAnalyzer extends AbstractSuppressionAnalyzer {
+
+    //<editor-fold defaultstate="collapsed" desc="All standard implmentation details of Analyzer">
+    /**
+     * The name of the analyzer.
+     */
+    private static final String ANALYZER_NAME = "Cpe Suppression Analyzer";
+    /**
+     * The phase that this analyzer is intended to run in.
+     */
+    private static final AnalysisPhase ANALYSIS_PHASE = AnalysisPhase.POST_IDENTIFIER_ANALYSIS;
+
+    /**
+     * Returns the name of the analyzer.
+     *
+     * @return the name of the analyzer.
+     */
+    @Override
+    public String getName() {
+        return ANALYZER_NAME;
+    }
+
+    /**
+     * Returns the phase that the analyzer is intended to run in.
+     *
+     * @return the phase that the analyzer is intended to run in.
+     */
+    @Override
+    public AnalysisPhase getAnalysisPhase() {
+        return ANALYSIS_PHASE;
+    }
+    //</editor-fold>
+
+    @Override
+    public void analyze(final Dependency dependency, final Engine engine) throws AnalysisException {
+
+        if (getRules() == null || getRules().size() <= 0) {
+            return;
+        }
+
+        for (final SuppressionRule rule : getRules()) {
+            rule.process(dependency);
+        }
+    }
+}

dependency-check-core/src/main/java/org/owasp/dependencycheck/analyzer/DependencyBundlingAnalyzer.java

@@ -23,12 +23,15 @@ import java.util.HashSet;
 import java.util.Iterator;
 import java.util.ListIterator;
 import java.util.Set;
+import java.util.logging.Level;
+import java.util.logging.Logger;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 import org.owasp.dependencycheck.Engine;
 import org.owasp.dependencycheck.dependency.Dependency;
 import org.owasp.dependencycheck.utils.DependencyVersion;
 import org.owasp.dependencycheck.utils.DependencyVersionUtil;
+import org.owasp.dependencycheck.utils.LogUtils;
 
 /**
  * <p>This analyzer ensures dependencies that should be grouped together, to
@@ -144,16 +147,14 @@ public class DependencyBundlingAnalyzer extends AbstractAnalyzer implements Anal
                                 }
                                 dependenciesToRemove.add(nextDependency);
                             } else {
-                                if (isCore(nextDependency, dependency)) {
-                                    nextDependency.addRelatedDependency(dependency);
-                                    //move any "related dependencies" to the new "parent" dependency
-                                    final Iterator<Dependency> i = dependency.getRelatedDependencies().iterator();
-                                    while (i.hasNext()) {
-                                        nextDependency.addRelatedDependency(i.next());
-                                        i.remove();
-                                    }
-                                    dependenciesToRemove.add(dependency);
+                                nextDependency.addRelatedDependency(dependency);
+                                //move any "related dependencies" to the new "parent" dependency
+                                final Iterator<Dependency> i = dependency.getRelatedDependencies().iterator();
+                                while (i.hasNext()) {
+                                    nextDependency.addRelatedDependency(i.next());
+                                    i.remove();
                                 }
+                                dependenciesToRemove.add(dependency);
                             }
                         }
                     }
@@ -260,8 +261,13 @@ public class DependencyBundlingAnalyzer extends AbstractAnalyzer implements Anal
                 || dependency2 == null || dependency2.getIdentifiers() == null) {
             return false;
         }
-        return dependency1.getIdentifiers().size() > 0
+        final boolean matches = dependency1.getIdentifiers().size() > 0
                 && dependency2.getIdentifiers().equals(dependency1.getIdentifiers());
+        if (LogUtils.isVerboseLoggingEnabled()) {
+            final String msg = String.format("IdentifiersMatch=%s (%s, %s)", matches, dependency1.getFileName(), dependency2.getFileName());
+            Logger.getLogger(DependencyBundlingAnalyzer.class.getName()).log(Level.FINE, msg);
+        }
+        return matches;
     }
 
     /**
@@ -299,10 +305,6 @@ public class DependencyBundlingAnalyzer extends AbstractAnalyzer implements Anal
      * This is likely a very broken attempt at determining if the 'left'
      * dependency is the 'core' library in comparison to the 'right' library.
      *
-     * TODO - consider splitting on /\._-\s/ and checking if all of one side is
-     * fully contained in the other With the exception of the word "core". This
-     * might work even on groups when we don't have a CVE.
-     *
      * @param left the dependency to test
      * @param right the dependency to test against
      * @return a boolean indicating whether or not the left dependency should be
@@ -311,18 +313,31 @@ public class DependencyBundlingAnalyzer extends AbstractAnalyzer implements Anal
     private boolean isCore(Dependency left, Dependency right) {
         final String leftName = left.getFileName().toLowerCase();
         final String rightName = right.getFileName().toLowerCase();
-
+        final boolean returnVal;
         if (rightName.contains("core") && !leftName.contains("core")) {
-            return false;
+            returnVal = false;
         } else if (!rightName.contains("core") && leftName.contains("core")) {
-            return true;
+            returnVal = true;
         } else {
-            //TODO should we be splitting the name on [-_(.\d)+] and seeing if the
-            //  parts are contained in the other side?
+            /*
+             * considered splitting the names up and comparing the components,
+             * but decided that the file name length should be sufficient as the
+             * "core" component, if this follows a normal namming protocol should
+             * be shorter:
+             * axis2-saaj-1.4.1.jar
+             * axis2-1.4.1.jar       <-----
+             * axis2-kernal-1.4.1.jar
+             */
             if (leftName.length() > rightName.length()) {
-                return false;
+                returnVal = false;
+            } else {
+                returnVal = true;
             }
-            return true;
         }
+        if (LogUtils.isVerboseLoggingEnabled()) {
+            final String msg = String.format("IsCore=%s (%s, %s)", returnVal, left.getFileName(), right.getFileName());
+            Logger.getLogger(DependencyBundlingAnalyzer.class.getName()).log(Level.FINE, msg);
+        }
+        return returnVal;
     }
 }

dependency-check-core/src/main/java/org/owasp/dependencycheck/analyzer/FalsePositiveAnalyzer.java

@@ -109,6 +109,7 @@ public class FalsePositiveAnalyzer extends AbstractAnalyzer {
     public void analyze(Dependency dependency, Engine engine) throws AnalysisException {
         removeJreEntries(dependency);
         removeBadMatches(dependency);
+        removeWrongVersionMatches(dependency);
         removeSpuriousCPE(dependency);
         addFalseNegativeCPEs(dependency);
     }
@@ -129,6 +130,7 @@ public class FalsePositiveAnalyzer extends AbstractAnalyzer {
      *
      * @param dependency the dependency being analyzed
      */
+    @SuppressWarnings("null")
     private void removeSpuriousCPE(Dependency dependency) {
         final List<Identifier> ids = new ArrayList<Identifier>();
         ids.addAll(dependency.getIdentifiers());
@@ -155,6 +157,8 @@ public class FalsePositiveAnalyzer extends AbstractAnalyzer {
                         final String nextVersion = nextCpe.getVersion();
                         if (currentVersion == null && nextVersion == null) {
                             //how did we get here?
+                            Logger.getLogger(FalsePositiveAnalyzer.class
+                                    .getName()).log(Level.FINE, "currentVersion and nextVersion are both null?");
                         } else if (currentVersion == null && nextVersion != null) {
                             dependency.getIdentifiers().remove(currentId);
                         } else if (nextVersion == null && currentVersion != null) {
@@ -273,18 +277,56 @@ public class FalsePositiveAnalyzer extends AbstractAnalyzer {
                 if ((i.getValue().matches(".*c\\+\\+.*")
                         || i.getValue().startsWith("cpe:/a:jquery:jquery")
                         || i.getValue().startsWith("cpe:/a:prototypejs:prototype")
-                        || i.getValue().startsWith("cpe:/a:yahoo:yui"))
+                        || i.getValue().startsWith("cpe:/a:yahoo:yui")
+                        || i.getValue().startsWith("cpe:/a:file:file")
+                        || i.getValue().startsWith("cpe:/a:mozilla:mozilla")
+                        || i.getValue().startsWith("cpe:/a:cvs:cvs")
+                        || i.getValue().startsWith("cpe:/a:ftp:ftp")
+                        || i.getValue().startsWith("cpe:/a:ssh:ssh"))
                         && dependency.getFileName().toLowerCase().endsWith(".jar")) {
                     itr.remove();
-                } else if (i.getValue().startsWith("cpe:/a:file:file")
-                        || i.getValue().startsWith("cpe:/a:mozilla:mozilla")
-                        || i.getValue().startsWith("cpe:/a:ssh:ssh")) {
+                } else if (i.getValue().startsWith("cpe:/a:apache:maven")
+                        && !dependency.getFileName().toLowerCase().matches("maven-core-[\\d\\.]+\\.jar")) {
                     itr.remove();
                 }
             }
         }
     }
 
+    /**
+     * Removes CPE matches for the wrong version of a dependency. Currently,
+     * this only covers Axis 1 & 2.
+     *
+     * @param dependency the dependency to analyze
+     */
+    private void removeWrongVersionMatches(Dependency dependency) {
+        final Set<Identifier> identifiers = dependency.getIdentifiers();
+        final Iterator<Identifier> itr = identifiers.iterator();
+
+        final String fileName = dependency.getFileName();
+        if (fileName != null && fileName.contains("axis2")) {
+            while (itr.hasNext()) {
+                final Identifier i = itr.next();
+                if ("cpe".equals(i.getType())) {
+                    final String cpe = i.getValue();
+                    if (cpe != null && (cpe.startsWith("cpe:/a:apache:axis:") || "cpe:/a:apache:axis".equals(cpe))) {
+                        itr.remove();
+                    }
+                }
+            }
+        } else if (fileName != null && fileName.contains("axis")) {
+            while (itr.hasNext()) {
+                final Identifier i = itr.next();
+                if ("cpe".equals(i.getType())) {
+                    final String cpe = i.getValue();
+                    if (cpe != null && (cpe.startsWith("cpe:/a:apache:axis2:") || "cpe:/a:apache:axis2".equals(cpe))) {
+                        itr.remove();
+                    }
+                }
+            }
+        }
+    }
+
     /**
      * There are some known CPE entries, specifically regarding sun and oracle
      * products due to the acquisition and changes in product names, that based

dependency-check-core/src/main/java/org/owasp/dependencycheck/analyzer/JarAnalyzer.java

@@ -159,7 +159,7 @@ public class JarAnalyzer extends AbstractAnalyzer implements Analyzer {
     /**
      * The set of file extensions supported by this analyzer.
      */
-    private static final Set<String> EXTENSIONS = newHashSet("jar");
+    private static final Set<String> EXTENSIONS = newHashSet("jar", "war");
 
     /**
      * Returns a list of file EXTENSIONS supported by this analyzer.
@@ -515,7 +515,7 @@ public class JarAnalyzer extends AbstractAnalyzer implements Analyzer {
             if (ratio > 0.5) {
                 //TODO remove weighting
                 vendor.addWeighting(entry.getKey());
-                if (addPackagesAsEvidence) {
+                if (addPackagesAsEvidence && entry.getKey().length() > 1) {
                     vendor.addEvidence("jar", "package", entry.getKey(), Evidence.Confidence.LOW);
                 }
             }
@@ -524,7 +524,7 @@ public class JarAnalyzer extends AbstractAnalyzer implements Analyzer {
             final float ratio = entry.getValue() / (float) classCount;
             if (ratio > 0.5) {
                 product.addWeighting(entry.getKey());
-                if (addPackagesAsEvidence) {
+                if (addPackagesAsEvidence && entry.getKey().length() > 1) {
                     product.addEvidence("jar", "package", entry.getKey(), Evidence.Confidence.LOW);
                 }
             }
@@ -580,38 +580,38 @@ public class JarAnalyzer extends AbstractAnalyzer implements Analyzer {
                 }
                 if (IGNORE_VALUES.contains(value)) {
                     continue;
-                } else if (key.equals(Attributes.Name.IMPLEMENTATION_TITLE.toString())) {
+                } else if (key.equalsIgnoreCase(Attributes.Name.IMPLEMENTATION_TITLE.toString())) {
                     foundSomething = true;
                     productEvidence.addEvidence(source, key, value, Evidence.Confidence.HIGH);
                     addMatchingValues(classInformation, value, productEvidence);
-                } else if (key.equals(Attributes.Name.IMPLEMENTATION_VERSION.toString())) {
+                } else if (key.equalsIgnoreCase(Attributes.Name.IMPLEMENTATION_VERSION.toString())) {
                     foundSomething = true;
                     versionEvidence.addEvidence(source, key, value, Evidence.Confidence.HIGH);
-                } else if (key.equals(Attributes.Name.IMPLEMENTATION_VENDOR.toString())) {
+                } else if (key.equalsIgnoreCase(Attributes.Name.IMPLEMENTATION_VENDOR.toString())) {
                     foundSomething = true;
                     vendorEvidence.addEvidence(source, key, value, Evidence.Confidence.HIGH);
                     addMatchingValues(classInformation, value, vendorEvidence);
-                } else if (key.equals(Attributes.Name.IMPLEMENTATION_VENDOR_ID.toString())) {
+                } else if (key.equalsIgnoreCase(Attributes.Name.IMPLEMENTATION_VENDOR_ID.toString())) {
                     foundSomething = true;
                     vendorEvidence.addEvidence(source, key, value, Evidence.Confidence.MEDIUM);
                     addMatchingValues(classInformation, value, vendorEvidence);
-                } else if (key.equals(BUNDLE_DESCRIPTION)) {
+                } else if (key.equalsIgnoreCase(BUNDLE_DESCRIPTION)) {
                     foundSomething = true;
                     addDescription(dependency, value, "manifest", key);
                     //productEvidence.addEvidence(source, key, value, Evidence.Confidence.MEDIUM);
                     addMatchingValues(classInformation, value, productEvidence);
-                } else if (key.equals(BUNDLE_NAME)) {
+                } else if (key.equalsIgnoreCase(BUNDLE_NAME)) {
                     foundSomething = true;
                     productEvidence.addEvidence(source, key, value, Evidence.Confidence.MEDIUM);
                     addMatchingValues(classInformation, value, productEvidence);
-                } else if (key.equals(BUNDLE_VENDOR)) {
+                } else if (key.equalsIgnoreCase(BUNDLE_VENDOR)) {
                     foundSomething = true;
                     vendorEvidence.addEvidence(source, key, value, Evidence.Confidence.HIGH);
                     addMatchingValues(classInformation, value, vendorEvidence);
-                } else if (key.equals(BUNDLE_VERSION)) {
+                } else if (key.equalsIgnoreCase(BUNDLE_VERSION)) {
                     foundSomething = true;
                     versionEvidence.addEvidence(source, key, value, Evidence.Confidence.HIGH);
-                } else if (key.equals(Attributes.Name.MAIN_CLASS.toString())) {
+                } else if (key.equalsIgnoreCase(Attributes.Name.MAIN_CLASS.toString())) {
                     continue;
                     //skipping main class as if this has important information to add
                     // it will be added during class name analysis...  if other fields
@@ -637,13 +637,22 @@ public class JarAnalyzer extends AbstractAnalyzer implements Analyzer {
 
                         foundSomething = true;
                         if (key.contains("version")) {
-                            versionEvidence.addEvidence(source, key, value, Evidence.Confidence.MEDIUM);
+                            if (key.contains("specification")) {
+                                versionEvidence.addEvidence(source, key, value, Evidence.Confidence.LOW);
+                            } else {
+                                versionEvidence.addEvidence(source, key, value, Evidence.Confidence.MEDIUM);
+                            }
+
                         } else if (key.contains("title")) {
                             productEvidence.addEvidence(source, key, value, Evidence.Confidence.MEDIUM);
                             addMatchingValues(classInformation, value, productEvidence);
                         } else if (key.contains("vendor")) {
-                            vendorEvidence.addEvidence(source, key, value, Evidence.Confidence.MEDIUM);
-                            addMatchingValues(classInformation, value, vendorEvidence);
+                            if (key.contains("specification")) {
+                                vendorEvidence.addEvidence(source, key, value, Evidence.Confidence.LOW);
+                            } else {
+                                vendorEvidence.addEvidence(source, key, value, Evidence.Confidence.MEDIUM);
+                                addMatchingValues(classInformation, value, vendorEvidence);
+                            }
                         } else if (key.contains("name")) {
                             productEvidence.addEvidence(source, key, value, Evidence.Confidence.MEDIUM);
                             vendorEvidence.addEvidence(source, key, value, Evidence.Confidence.MEDIUM);

dependency-check-core/src/main/java/org/owasp/dependencycheck/analyzer/NvdCveAnalyzer.java

@@ -16,19 +16,18 @@
  *
  * Copyright (c) 2012 Jeremy Long. All Rights Reserved.
  */
-package org.owasp.dependencycheck.data.nvdcve;
+package org.owasp.dependencycheck.analyzer;
 
 import java.io.IOException;
 import java.sql.SQLException;
 import java.util.List;
 import java.util.Set;
 import org.owasp.dependencycheck.Engine;
-import org.owasp.dependencycheck.analyzer.AnalysisException;
-import org.owasp.dependencycheck.analyzer.AnalysisPhase;
 import org.owasp.dependencycheck.dependency.Dependency;
 import org.owasp.dependencycheck.dependency.Vulnerability;
 import org.owasp.dependencycheck.dependency.Identifier;
-import org.owasp.dependencycheck.analyzer.Analyzer;
+import org.owasp.dependencycheck.data.nvdcve.CveDB;
+import org.owasp.dependencycheck.data.nvdcve.DatabaseException;
 
 /**
  * NvdCveAnalyzer is a utility class that takes a project dependency and

dependency-check-core/src/main/java/org/owasp/dependencycheck/analyzer/VulnerabilitySuppressionAnalyzer.java

@@ -0,0 +1,76 @@
+/*
+ * This file is part of dependency-check-core.
+ *
+ * Dependency-check-core is free software: you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation, either version 3 of the License, or (at your option) any
+ * later version.
+ *
+ * Dependency-check-core is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+ * details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * dependency-check-core. If not, see http://www.gnu.org/licenses/.
+ *
+ * Copyright (c) 2013 Jeremy Long. All Rights Reserved.
+ */
+package org.owasp.dependencycheck.analyzer;
+
+import org.owasp.dependencycheck.Engine;
+import org.owasp.dependencycheck.dependency.Dependency;
+import org.owasp.dependencycheck.suppression.SuppressionRule;
+
+/**
+ * The suppression analyzer processes an externally defined XML document that
+ * complies with the suppressions.xsd schema. Any identified Vulnerability
+ * entries within the dependencies that match will be removed.
+ *
+ * @author Jeremy Long (jeremy.long@owasp.org)
+ */
+public class VulnerabilitySuppressionAnalyzer extends AbstractSuppressionAnalyzer {
+
+    //<editor-fold defaultstate="collapsed" desc="All standard implmentation details of Analyzer">
+    /**
+     * The name of the analyzer.
+     */
+    private static final String ANALYZER_NAME = "Vulnerability Suppression Analyzer";
+    /**
+     * The phase that this analyzer is intended to run in.
+     */
+    private static final AnalysisPhase ANALYSIS_PHASE = AnalysisPhase.POST_FINDING_ANALYSIS;
+
+    /**
+     * Returns the name of the analyzer.
+     *
+     * @return the name of the analyzer.
+     */
+    @Override
+    public String getName() {
+        return ANALYZER_NAME;
+    }
+
+    /**
+     * Returns the phase that the analyzer is intended to run in.
+     *
+     * @return the phase that the analyzer is intended to run in.
+     */
+    @Override
+    public AnalysisPhase getAnalysisPhase() {
+        return ANALYSIS_PHASE;
+    }
+    //</editor-fold>
+
+    @Override
+    public void analyze(final Dependency dependency, final Engine engine) throws AnalysisException {
+
+        if (getRules() == null || getRules().size() <= 0) {
+            return;
+        }
+
+        for (final SuppressionRule rule : getRules()) {
+            rule.process(dependency);
+        }
+    }
+}

dependency-check-core/src/main/java/org/owasp/dependencycheck/concurrency/DirectoryLockException.java

@@ -0,0 +1,67 @@
+/*
+ * This file is part of dependency-check-core.
+ *
+ * Dependency-check-core is free software: you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation, either version 3 of the License, or (at your option) any
+ * later version.
+ *
+ * Dependency-check-core is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+ * details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * dependency-check-core. If not, see http://www.gnu.org/licenses/.
+ *
+ * Copyright (c) 2013 Jeremy Long. All Rights Reserved.
+ */
+package org.owasp.dependencycheck.concurrency;
+
+/**
+ * If thrown, indicates that a problem occurred when locking a directory.
+ *
+ * @author Jeremy Long (jeremy.long@owasp.org)
+ */
+public class DirectoryLockException extends Exception {
+
+    /**
+     * Default serial version UID.
+     */
+    private static final long serialVersionUID = 1L;
+
+    /**
+     * Constructs a new Directory Lock Exception.
+     */
+    public DirectoryLockException() {
+        super();
+    }
+
+    /**
+     * Constructs a new Directory Lock Exception.
+     *
+     * @param msg the message describing the exception
+     */
+    public DirectoryLockException(String msg) {
+        super(msg);
+    }
+
+    /**
+     * Constructs a new Directory Lock Exception.
+     *
+     * @param ex the cause of the exception
+     */
+    public DirectoryLockException(Throwable ex) {
+        super(ex);
+    }
+
+    /**
+     * Constructs a new Directory Lock Exception.
+     *
+     * @param msg the message describing the exception
+     * @param ex the cause of the exception
+     */
+    public DirectoryLockException(String msg, Throwable ex) {
+        super(msg, ex);
+    }
+}

dependency-check-core/src/main/java/org/owasp/dependencycheck/concurrency/DirectorySpinLock.java

@@ -0,0 +1,267 @@
+/*
+ * This file is part of dependency-check-core.
+ *
+ * Dependency-check-core is free software: you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation, either version 3 of the License, or (at your option) any
+ * later version.
+ *
+ * Dependency-check-core is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+ * details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * dependency-check-core. If not, see http://www.gnu.org/licenses/.
+ *
+ * Copyright (c) 2013 Jeremy Long. All Rights Reserved.
+ */
+package org.owasp.dependencycheck.concurrency;
+
+import java.io.Closeable;
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.RandomAccessFile;
+import java.nio.channels.AsynchronousCloseException;
+import java.nio.channels.ClosedChannelException;
+import java.nio.channels.FileChannel;
+import java.nio.channels.FileLock;
+import java.nio.channels.FileLockInterruptionException;
+import java.nio.channels.NonWritableChannelException;
+import java.nio.channels.OverlappingFileLockException;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+/**
+ * Implements a spin lock on a given directory. If the lock cannot be obtained,
+ * the process will "spin" waiting for an opportunity to obtain the lock
+ * requested.
+ *
+ * @author Jeremy Long (jeremy.long@owasp.org)
+ */
+public class DirectorySpinLock implements Closeable /*, AutoCloseable*/ {
+
+    /**
+     * The name of the lock file.
+     */
+    public static final String LOCK_NAME = "data.lock";
+    /**
+     * The maximum wait period used when attempting to obtain a lock.
+     */
+    public static final int MAX_SPIN = 100;
+    /**
+     * The file channel used to perform the lock.
+     */
+    private FileChannel channel = null;
+    /**
+     * The file used to perform the lock.
+     */
+    private File lockFile = null;
+    /**
+     * The lock object.
+     */
+    private FileLock lock = null;
+    /**
+     * The maximum number of seconds that the spin lock will wait while trying
+     * to obtain a lock.
+     */
+    private long maxWait = MAX_SPIN;
+
+    /**
+     * Get the maximum wait time, in seconds, that the spin lock will wait while
+     * trying to obtain a lock.
+     *
+     * @return the number of seconds the spin lock will wait
+     */
+    public long getMaxWait() {
+        return maxWait / 2; //sleep is for 500, so / 2
+    }
+
+    /**
+     * Set the maximum wait time, in seconds, that the spin lock will wait while
+     * trying to obtain a lock.
+     *
+     * @param maxWait the number of seconds the spin lock will wait
+     */
+    public void setMaxWait(long maxWait) {
+        this.maxWait = maxWait * 2; //sleep is for 500, so * 2
+    }
+
+    /**
+     * Constructs a new spin lock on the given directory.
+     *
+     * @param directory the directory to monitor/lock
+     * @throws InvalidDirectoryException thrown if there is an issue with the
+     * directory provided
+     * @throws DirectoryLockException thrown there is an issue obtaining a
+     * handle to the lock file
+     */
+    public DirectorySpinLock(File directory) throws InvalidDirectoryException, DirectoryLockException {
+        checkDirectory(directory);
+        lockFile = new File(directory, LOCK_NAME);
+        RandomAccessFile file = null;
+        try {
+            file = new RandomAccessFile(lockFile, "rw");
+        } catch (FileNotFoundException ex) {
+            throw new DirectoryLockException("Lock file not found", ex);
+        }
+        channel = file.getChannel();
+    }
+
+    /**
+     * Attempts to obtain an exclusive lock; an exception is thrown if the lock
+     * could not be obtained. This method may block for a few seconds if a lock
+     * cannot be obtained.
+     *
+     * @throws DirectoryLockException thrown if there is an exception obtaining
+     * the lock
+     */
+    public void obtainSharedLock() throws DirectoryLockException {
+        obtainLock(true);
+    }
+
+    /**
+     * Attempts to obtain an exclusive lock; an exception is thrown if the lock
+     * could not be obtained. This method may block for a few seconds if a lock
+     * cannot be obtained.
+     *
+     * @throws DirectoryLockException thrown if there is an exception obtaining
+     * the lock
+     */
+    public void obtainExclusiveLock() throws DirectoryLockException {
+        obtainLock(false);
+    }
+
+    /**
+     * Attempts to obtain a lock; an exception is thrown if the lock could not
+     * be obtained. This method may block for a few seconds if a lock cannot be
+     * obtained.
+     *
+     * @param shared true if the lock is shared, otherwise false
+     * @param maxWait the maximum time to wait, in seconds, while trying to
+     * obtain the lock
+     * @throws DirectoryLockException thrown if there is an exception obtaining
+     * the lock
+     */
+    protected void obtainLock(boolean shared, long maxWait) throws DirectoryLockException {
+        setMaxWait(maxWait);
+        obtainLock(shared);
+    }
+
+    /**
+     * Attempts to obtain a lock; an exception is thrown if the lock could not
+     * be obtained. This method may block for a few seconds if a lock cannot be
+     * obtained.
+     *
+     * @param shared true if the lock is shared, otherwise false
+     * @throws DirectoryLockException thrown if there is an exception obtaining
+     * the lock
+     */
+    protected void obtainLock(boolean shared) throws DirectoryLockException {
+        if (lock != null) {
+            release();
+        }
+        if (channel == null) {
+            throw new DirectoryLockException("Unable to create lock, no file channel exists");
+        }
+        int count = 0;
+        Exception lastException = null;
+        while (lock == null && count++ < maxWait) {
+            try {
+                lock = channel.lock(0, Long.MAX_VALUE, shared);
+            } catch (AsynchronousCloseException ex) {
+                lastException = ex;
+            } catch (ClosedChannelException ex) {
+                lastException = ex;
+            } catch (FileLockInterruptionException ex) {
+                lastException = ex;
+            } catch (OverlappingFileLockException ex) {
+                lastException = ex;
+            } catch (NonWritableChannelException ex) {
+                lastException = ex;
+            } catch (IOException ex) {
+                lastException = ex;
+            }
+            try {
+                Thread.sleep(500);
+            } catch (InterruptedException ex) {
+                Thread.currentThread().interrupt();
+            }
+        }
+        if (lock == null) {
+            if (lastException == null) {
+                throw new DirectoryLockException("Unable to obtain lock");
+            } else {
+                throw new DirectoryLockException("Unable to obtain lock", lastException);
+            }
+        }
+    }
+
+    /**
+     * Performs a few simple rudimentary checks on the specified directory.
+     * Specifically, does the file exist and is it a directory.
+     *
+     * @param directory the File object to inspect
+     * @throws InvalidDirectoryException thrown if the directory is null or is
+     * not a directory
+     */
+    private void checkDirectory(File directory) throws InvalidDirectoryException {
+        if (directory == null) {
+            throw new InvalidDirectoryException("Unable to obtain lock on a null File");
+        }
+        if (!directory.isDirectory()) {
+            final String msg = String.format("File, '%s', does not exist or is not a directory", directory.getAbsolutePath());
+            throw new InvalidDirectoryException(msg);
+        }
+    }
+
+    /**
+     * Releases any locks and closes the underlying channel.
+     *
+     * @throws IOException if an IO Exception occurs
+     */
+    @Override
+    public void close() throws IOException {
+        release();
+// TODO uncomment this once support for 1.6 is dropped.
+//        if (lock != null) {
+//            try {
+//                lock.close();
+//            } catch (IOException ex) {
+//                Logger.getLogger(DirectorySpinLock.class.getName()).log(Level.FINEST, "Unable to close file lock due to IO Exception", ex);
+//            }
+//        }
+        if (channel != null) {
+            try {
+                channel.close();
+            } catch (IOException ex) {
+                Logger.getLogger(DirectorySpinLock.class.getName()).log(Level.FINEST, "Unable to close the channel for the file lock", ex);
+            }
+        }
+        if (lockFile != null) {
+            if (lockFile.exists()) {
+                /* yes, this delete could fail which is totally fine. The other
+                 * thread holding the lock while delete it.
+                 */
+                lockFile.delete();
+            }
+        }
+    }
+
+    /**
+     * Releases the lock. Any exceptions that are thrown by the underlying lock
+     * during the release are ignored.
+     */
+    public void release() {
+        if (lock != null) {
+            try {
+                lock.release();
+            } catch (ClosedChannelException ex) {
+                Logger.getLogger(DirectorySpinLock.class.getName()).log(Level.FINEST, "Uable to release file lock", ex);
+            } catch (IOException ex) {
+                Logger.getLogger(DirectorySpinLock.class.getName()).log(Level.FINEST, "Unable to release file lock due to IO Exception", ex);
+            }
+        }
+    }
+}

dependency-check-core/src/main/java/org/owasp/dependencycheck/concurrency/InvalidDirectoryException.java

@@ -0,0 +1,67 @@
+/*
+ * This file is part of dependency-check-core.
+ *
+ * Dependency-check-core is free software: you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation, either version 3 of the License, or (at your option) any
+ * later version.
+ *
+ * Dependency-check-core is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+ * details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * dependency-check-core. If not, see http://www.gnu.org/licenses/.
+ *
+ * Copyright (c) 2013 Jeremy Long. All Rights Reserved.
+ */
+package org.owasp.dependencycheck.concurrency;
+
+/**
+ * If thrown, indicates that there is a problem with a directory.
+ *
+ * @author Jeremy Long (jeremy.long@owasp.org)
+ */
+public class InvalidDirectoryException extends Exception {
+
+    /**
+     * Default serial version UID.
+     */
+    private static final long serialVersionUID = 1L;
+
+    /**
+     * Constructs a new Invalid Directory Exception.
+     */
+    public InvalidDirectoryException() {
+        super();
+    }
+
+    /**
+     * Constructs a new Invalid Directory Exception.
+     *
+     * @param msg the message describing the exception
+     */
+    public InvalidDirectoryException(String msg) {
+        super(msg);
+    }
+
+    /**
+     * Constructs a new Invalid Directory Exception.
+     *
+     * @param ex the cause of the exception
+     */
+    public InvalidDirectoryException(Throwable ex) {
+        super(ex);
+    }
+
+    /**
+     * Constructs a new Invalid Directory Exception.
+     *
+     * @param msg the message describing the exception
+     * @param ex the cause of the exception
+     */
+    public InvalidDirectoryException(String msg, Throwable ex) {
+        super(msg, ex);
+    }
+}

dependency-check-core/src/main/java/org/owasp/dependencycheck/concurrency/package-info.java

@@ -0,0 +1,11 @@
+/**
+ * <html>
+ * <head>
+ * <title>org.owasp.dependencycheck.concurrency</title>
+ * </head>
+ * <body>
+ * Contains classes used to create shared and exclusive locks on directories.
+ * </body>
+ * </html>
+ */
+package org.owasp.dependencycheck.concurrency;

dependency-check-core/src/main/java/org/owasp/dependencycheck/data/BaseDB.java

@@ -0,0 +1,245 @@
+/*
+ * This file is part of dependency-check-core.
+ *
+ * Dependency-check-core is free software: you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation, either version 3 of the License, or (at your option) any
+ * later version.
+ *
+ * Dependency-check-core is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+ * details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * dependency-check-core. If not, see http://www.gnu.org/licenses/.
+ *
+ * Copyright (c) 2013 Jeremy Long. All Rights Reserved.
+ */
+package org.owasp.dependencycheck.data;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+import org.owasp.dependencycheck.data.nvdcve.CveDB;
+import static org.owasp.dependencycheck.data.nvdcve.CveDB.DB_SCHEMA_VERSION;
+import org.owasp.dependencycheck.data.nvdcve.DatabaseException;
+import org.owasp.dependencycheck.utils.Settings;
+
+/**
+ *
+ * @author Jeremy Long (jeremy.long@owasp.org)
+ */
+public class BaseDB {
+
+    /**
+     * Resource location for SQL file used to create the database schema.
+     */
+    public static final String DB_STRUCTURE_RESOURCE = "data/initialize.sql";
+    /**
+     * The version of the current DB Schema.
+     */
+    public static final String DB_SCHEMA_VERSION = "2.7";
+    /**
+     * Database connection
+     */
+    private Connection conn;
+
+    /**
+     * Returns the database connection.
+     *
+     * @return the database connection
+     */
+    protected Connection getConnection() {
+        return conn;
+    }
+
+    /**
+     * Opens the database connection. If the database does not exist, it will
+     * create a new one.
+     *
+     * @throws IOException thrown if there is an IO Exception
+     * @throws SQLException thrown if there is a SQL Exception
+     * @throws DatabaseException thrown if there is an error initializing a new
+     * database
+     * @throws ClassNotFoundException thrown if the h2 database driver cannot be
+     * loaded
+     */
+    @edu.umd.cs.findbugs.annotations.SuppressWarnings(
+            value = "DMI_EMPTY_DB_PASSWORD",
+            justification = "Yes, I know... Blank password.")
+    public void open() throws IOException, SQLException, DatabaseException, ClassNotFoundException {
+        final String fileName = CveDB.getDataDirectory().getCanonicalPath();
+        final File f = new File(fileName, "cve." + DB_SCHEMA_VERSION);
+        final File check = new File(f.getAbsolutePath() + ".h2.db");
+        final boolean createTables = !check.exists();
+        final String connStr = String.format("jdbc:h2:file:%s;AUTO_SERVER=TRUE", f.getAbsolutePath());
+        Class.forName("org.h2.Driver");
+        conn = DriverManager.getConnection(connStr, "sa", "");
+        if (createTables) {
+            createTables();
+        }
+    }
+
+    /**
+     * Closes the DB4O database. Close should be called on this object when it
+     * is done being used.
+     */
+    public void close() {
+        if (conn != null) {
+            try {
+                conn.close();
+            } catch (SQLException ex) {
+                final String msg = "There was an error attempting to close the CveDB, see the log for more details.";
+                Logger.getLogger(BaseDB.class.getName()).log(Level.SEVERE, msg, ex);
+                Logger.getLogger(BaseDB.class.getName()).log(Level.FINE, null, ex);
+            }
+            conn = null;
+        }
+    }
+
+    /**
+     * Commits all completed transactions.
+     *
+     * @throws SQLException thrown if a SQL Exception occurs
+     */
+    public void commit() throws SQLException {
+        if (conn != null) {
+            conn.commit();
+        }
+    }
+
+    /**
+     * Cleans up the object and ensures that "close" has been called.
+     *
+     * @throws Throwable thrown if there is a problem
+     */
+    @Override
+    protected void finalize() throws Throwable {
+        close();
+        super.finalize(); //not necessary if extending Object.
+    }
+
+    /**
+     * Creates the database structure (tables and indexes) to store the CVE data
+     *
+     * @throws SQLException thrown if there is a sql exception
+     * @throws DatabaseException thrown if there is a database exception
+     */
+    private void createTables() throws SQLException, DatabaseException {
+        InputStream is;
+        InputStreamReader reader;
+        BufferedReader in = null;
+        try {
+            is = this.getClass().getClassLoader().getResourceAsStream(DB_STRUCTURE_RESOURCE);
+            reader = new InputStreamReader(is, "UTF-8");
+            in = new BufferedReader(reader);
+            final StringBuilder sb = new StringBuilder(2110);
+            String tmp;
+            while ((tmp = in.readLine()) != null) {
+                sb.append(tmp);
+            }
+            Statement statement = null;
+            try {
+                statement = conn.createStatement();
+                statement.execute(sb.toString());
+            } finally {
+                closeStatement(statement);
+            }
+        } catch (IOException ex) {
+            throw new DatabaseException("Unable to create database schema", ex);
+        } finally {
+            if (in != null) {
+                try {
+                    in.close();
+                } catch (IOException ex) {
+                    Logger.getLogger(CveDB.class
+                            .getName()).log(Level.FINEST, null, ex);
+                }
+            }
+        }
+    }
+
+    /**
+     * Retrieves the directory that the JAR file exists in so that we can ensure
+     * we always use a common data directory.
+     *
+     * @return the data directory for this index.
+     * @throws IOException is thrown if an IOException occurs of course...
+     */
+    public static File getDataDirectory() throws IOException {
+        final File path = Settings.getDataFile(Settings.KEYS.CVE_DATA_DIRECTORY);
+        if (!path.exists()) {
+            if (!path.mkdirs()) {
+                throw new IOException("Unable to create NVD CVE Data directory");
+            }
+        }
+        return path;
+    }
+
+    /**
+     * Returns the generated integer primary key for a newly inserted row.
+     *
+     * @param statement a prepared statement that just executed an insert
+     * @return a primary key
+     * @throws DatabaseException thrown if there is an exception obtaining the
+     * key
+     */
+    protected int getGeneratedKey(PreparedStatement statement) throws DatabaseException {
+        ResultSet rs = null;
+        int id = 0;
+        try {
+            rs = statement.getGeneratedKeys();
+            rs.next();
+            id = rs.getInt(1);
+        } catch (SQLException ex) {
+            throw new DatabaseException("Unable to get primary key for inserted row");
+        } finally {
+            closeResultSet(rs);
+        }
+        return id;
+    }
+
+    /**
+     * Closes the given statement object ignoring any exceptions that occur.
+     *
+     * @param statement a Statement object
+     */
+    public void closeStatement(Statement statement) {
+        if (statement != null) {
+            try {
+                statement.close();
+            } catch (SQLException ex) {
+                Logger.getLogger(CveDB.class
+                        .getName()).log(Level.FINEST, statement.toString(), ex);
+            }
+        }
+    }
+
+    /**
+     * Closes the result set capturing and ignoring any SQLExceptions that
+     * occur.
+     *
+     * @param rs a ResultSet to close
+     */
+    public void closeResultSet(ResultSet rs) {
+        if (rs != null) {
+            try {
+                rs.close();
+            } catch (SQLException ex) {
+                Logger.getLogger(CveDB.class
+                        .getName()).log(Level.FINEST, rs.toString(), ex);
+            }
+        }
+    }
+}

dependency-check-core/src/main/java/org/owasp/dependencycheck/data/NoDataException.java

@@ -0,0 +1,69 @@
+/*
+ * This file is part of dependency-check-core.
+ *
+ * Dependency-check-core is free software: you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation, either version 3 of the License, or (at your option) any
+ * later version.
+ *
+ * Dependency-check-core is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+ * details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * dependency-check-core. If not, see http://www.gnu.org/licenses/.
+ *
+ * Copyright (c) 2013 Jeremy Long. All Rights Reserved.
+ */
+package org.owasp.dependencycheck.data;
+
+import java.io.IOException;
+
+/**
+ * An exception used when the data needed does not exist to perform analysis.
+ *
+ * @author Jeremy Long (jeremy.long@owasp.org)
+ */
+public class NoDataException extends IOException {
+
+    /**
+     * The serial version uid.
+     */
+    private static final long serialVersionUID = 1L;
+
+    /**
+     * Creates a new NoDataException.
+     */
+    public NoDataException() {
+        super();
+    }
+
+    /**
+     * Creates a new NoDataException.
+     *
+     * @param msg a message for the exception.
+     */
+    public NoDataException(String msg) {
+        super(msg);
+    }
+
+    /**
+     * Creates a new NoDataException.
+     *
+     * @param ex the cause of the exception.
+     */
+    public NoDataException(Throwable ex) {
+        super(ex);
+    }
+
+    /**
+     * Creates a new NoDataException.
+     *
+     * @param msg a message for the exception.
+     * @param ex the cause of the exception.
+     */
+    public NoDataException(String msg, Throwable ex) {
+        super(msg, ex);
+    }
+}

dependency-check-core/src/main/java/org/owasp/dependencycheck/data/cpe/CpeMemoryIndex.java

@@ -0,0 +1,328 @@
+/*
+ * This file is part of dependency-check-core.
+ *
+ * Dependency-check-core is free software: you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation, either version 3 of the License, or (at your option) any
+ * later version.
+ *
+ * Dependency-check-core is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+ * details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * dependency-check-core. If not, see http://www.gnu.org/licenses/.
+ *
+ * Copyright (c) 2013 Jeremy Long. All Rights Reserved.
+ */
+package org.owasp.dependencycheck.data.cpe;
+
+import java.io.IOException;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.core.KeywordAnalyzer;
+import org.apache.lucene.analysis.miscellaneous.PerFieldAnalyzerWrapper;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.document.TextField;
+import org.apache.lucene.index.CorruptIndexException;
+import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.queryparser.classic.ParseException;
+import org.apache.lucene.queryparser.classic.QueryParser;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.TopDocs;
+import org.owasp.dependencycheck.data.lucene.FieldAnalyzer;
+import org.owasp.dependencycheck.data.nvdcve.CveDB;
+import org.apache.lucene.store.RAMDirectory;
+import org.owasp.dependencycheck.data.lucene.LuceneUtils;
+import org.owasp.dependencycheck.data.lucene.SearchFieldAnalyzer;
+
+/**
+ * An in memory lucene index that contains the vendor/product combinations from
+ * the CPE (application) identifiers within the NVD CVE data.
+ *
+ * @author Jeremy Long (jeremy.long@owasp.org)
+ */
+public final class CpeMemoryIndex {
+
+    /**
+     * singleton instance.
+     */
+    private static CpeMemoryIndex instance = new CpeMemoryIndex();
+
+    /**
+     * private constructor for singleton.
+     */
+    private CpeMemoryIndex() {
+    }
+
+    /**
+     * Gets the singleton instance of the CpeMemoryIndex.
+     *
+     * @return the instance of the CpeMemoryIndex
+     */
+    public static CpeMemoryIndex getInstance() {
+        return instance;
+    }
+    /**
+     * The in memory Lucene index.
+     */
+    private RAMDirectory index;
+    /**
+     * The Lucene IndexReader.
+     */
+    private IndexReader indexReader;
+    /**
+     * The Lucene IndexSearcher.
+     */
+    private IndexSearcher indexSearcher;
+    /**
+     * The Lucene Analyzer used for Searching.
+     */
+    private Analyzer searchingAnalyzer;
+    /**
+     * The Lucene QueryParser used for Searching.
+     */
+    private QueryParser queryParser;
+    /**
+     * The search field analyzer for the product field.
+     */
+    private SearchFieldAnalyzer productSearchFieldAnalyzer;
+    /**
+     * The search field analyzer for the vendor field.
+     */
+    private SearchFieldAnalyzer vendorSearchFieldAnalyzer;
+
+    /**
+     * Creates and loads data into an in memory index.
+     *
+     * @param cve the data source to retrieve the cpe data
+     * @throws IndexException thrown if there is an error creating the index
+     */
+    public void open(CveDB cve) throws IndexException {
+        if (!openState) {
+            index = new RAMDirectory();
+            buildIndex(cve);
+            try {
+                indexReader = DirectoryReader.open(index);
+            } catch (IOException ex) {
+                throw new IndexException(ex);
+            }
+            indexSearcher = new IndexSearcher(indexReader);
+            searchingAnalyzer = createSearchingAnalyzer();
+            queryParser = new QueryParser(LuceneUtils.CURRENT_VERSION, Fields.DOCUMENT_KEY, searchingAnalyzer);
+            openState = true;
+        }
+    }
+    /**
+     * A flag indicating whether or not the index is open.
+     */
+    private boolean openState = false;
+
+    /**
+     * returns whether or not the index is open.
+     *
+     * @return whether or not the index is open
+     */
+    public boolean isOpen() {
+        return openState;
+    }
+
+    /**
+     * Creates the indexing analyzer for the CPE Index.
+     *
+     * @return the CPE Analyzer.
+     */
+    @SuppressWarnings("unchecked")
+    private Analyzer createIndexingAnalyzer() {
+        final Map fieldAnalyzers = new HashMap();
+        fieldAnalyzers.put(Fields.DOCUMENT_KEY, new KeywordAnalyzer());
+        return new PerFieldAnalyzerWrapper(new FieldAnalyzer(LuceneUtils.CURRENT_VERSION), fieldAnalyzers);
+    }
+
+    /**
+     * Creates an Analyzer for searching the CPE Index.
+     *
+     * @return the CPE Analyzer.
+     */
+    @SuppressWarnings("unchecked")
+    private Analyzer createSearchingAnalyzer() {
+        final Map fieldAnalyzers = new HashMap();
+        fieldAnalyzers.put(Fields.DOCUMENT_KEY, new KeywordAnalyzer());
+        productSearchFieldAnalyzer = new SearchFieldAnalyzer(LuceneUtils.CURRENT_VERSION);
+        vendorSearchFieldAnalyzer = new SearchFieldAnalyzer(LuceneUtils.CURRENT_VERSION);
+        fieldAnalyzers.put(Fields.PRODUCT, productSearchFieldAnalyzer);
+        fieldAnalyzers.put(Fields.VENDOR, vendorSearchFieldAnalyzer);
+
+        return new PerFieldAnalyzerWrapper(new FieldAnalyzer(LuceneUtils.CURRENT_VERSION), fieldAnalyzers);
+    }
+
+    /**
+     * Saves a CPE IndexEntry into the Lucene index.
+     *
+     * @param vendor the vendor to index
+     * @param product the product to index
+     * @param indexWriter the index writer to write the entry into
+     * @throws CorruptIndexException is thrown if the index is corrupt
+     * @throws IOException is thrown if an IOException occurs
+     */
+    public void saveEntry(String vendor, String product, IndexWriter indexWriter) throws CorruptIndexException, IOException {
+        final Document doc = new Document();
+        final Field v = new TextField(Fields.VENDOR, vendor, Field.Store.YES);
+        final Field p = new TextField(Fields.PRODUCT, product, Field.Store.YES);
+        doc.add(v);
+        doc.add(p);
+        indexWriter.addDocument(doc);
+    }
+
+    /**
+     * Closes the CPE Index.
+     */
+    public void close() {
+        if (searchingAnalyzer != null) {
+            searchingAnalyzer.close();
+            searchingAnalyzer = null;
+        }
+        if (indexReader != null) {
+            try {
+                indexReader.close();
+            } catch (IOException ex) {
+                Logger.getLogger(CpeMemoryIndex.class.getName()).log(Level.FINEST, null, ex);
+            }
+            indexReader = null;
+        }
+        queryParser = null;
+        indexSearcher = null;
+        if (index != null) {
+            index.close();
+            index = null;
+        }
+        openState = false;
+    }
+
+    /**
+     * Builds the lucene index based off of the data within the CveDB.
+     *
+     * @param cve the data base containing the CPE data
+     * @throws IndexException thrown if there is an issue creating the index
+     */
+    private void buildIndex(CveDB cve) throws IndexException {
+        Analyzer analyzer = null;
+        IndexWriter indexWriter = null;
+        try {
+            analyzer = createIndexingAnalyzer();
+            final IndexWriterConfig conf = new IndexWriterConfig(LuceneUtils.CURRENT_VERSION, analyzer);
+            indexWriter = new IndexWriter(index, conf);
+            final ResultSet rs = cve.getVendorProductList();
+            if (rs == null) {
+                throw new IndexException("No data exists");
+            }
+            try {
+                while (rs.next()) {
+                    saveEntry(rs.getString(1), rs.getString(2), indexWriter);
+                }
+            } catch (SQLException ex) {
+                Logger.getLogger(CpeMemoryIndex.class.getName()).log(Level.FINE, null, ex);
+                throw new IndexException("Error reading CPE data", ex);
+            }
+        } catch (CorruptIndexException ex) {
+            throw new IndexException("Unable to close an in-memory index", ex);
+        } catch (IOException ex) {
+            throw new IndexException("Unable to close an in-memory index", ex);
+        } finally {
+            if (indexWriter != null) {
+                try {
+                    try {
+                        indexWriter.commit();
+                    } finally {
+                        indexWriter.close(true);
+                    }
+                } catch (CorruptIndexException ex) {
+                    throw new IndexException("Unable to close an in-memory index", ex);
+                } catch (IOException ex) {
+                    throw new IndexException("Unable to close an in-memory index", ex);
+                }
+                if (analyzer != null) {
+                    analyzer.close();
+                }
+            }
+        }
+    }
+
+    /**
+     * Resets the searching analyzers
+     */
+    private void resetSearchingAnalyzer() {
+        if (productSearchFieldAnalyzer != null) {
+            productSearchFieldAnalyzer.clear();
+        }
+        if (vendorSearchFieldAnalyzer != null) {
+            vendorSearchFieldAnalyzer.clear();
+        }
+    }
+
+    /**
+     * Searches the index using the given search string.
+     *
+     * @param searchString the query text
+     * @param maxQueryResults the maximum number of documents to return
+     * @return the TopDocs found by the search
+     * @throws ParseException thrown when the searchString is invalid
+     * @throws IOException is thrown if there is an issue with the underlying
+     * Index
+     */
+    public TopDocs search(String searchString, int maxQueryResults) throws ParseException, IOException {
+        if (searchString == null || searchString.trim().isEmpty()) {
+            throw new ParseException("Query is null or empty");
+        }
+        final Query query = queryParser.parse(searchString);
+        return indexSearcher.search(query, maxQueryResults);
+    }
+
+    /**
+     * Searches the index using the given query.
+     *
+     * @param query the query used to search the index
+     * @param maxQueryResults the max number of results to return
+     * @return the TopDocs found be the query
+     * @throws CorruptIndexException thrown if the Index is corrupt
+     * @throws IOException thrown if there is an IOException
+     */
+    public TopDocs search(Query query, int maxQueryResults) throws CorruptIndexException, IOException {
+        resetSearchingAnalyzer();
+        return indexSearcher.search(query, maxQueryResults);
+    }
+
+    /**
+     * Retrieves a document from the Index.
+     *
+     * @param documentId the id of the document to retrieve
+     * @return the Document
+     * @throws IOException thrown if there is an IOException
+     */
+    public Document getDocument(int documentId) throws IOException {
+        return indexSearcher.doc(documentId);
+    }
+
+    /**
+     * Returns the number of CPE entries stored in the index.
+     *
+     * @return the number of CPE entries stored in the index
+     */
+    public int numDocs() {
+        if (indexReader == null) {
+            return -1;
+        }
+        return indexReader.numDocs();
+    }
+}

dependency-check-core/src/main/java/org/owasp/dependencycheck/data/cpe/Index.java

@@ -1,170 +0,0 @@
-/*
- * This file is part of dependency-check-core.
- *
- * Dependency-check-core is free software: you can redistribute it and/or modify it
- * under the terms of the GNU General Public License as published by the Free
- * Software Foundation, either version 3 of the License, or (at your option) any
- * later version.
- *
- * Dependency-check-core is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
- * details.
- *
- * You should have received a copy of the GNU General Public License along with
- * dependency-check-core. If not, see http://www.gnu.org/licenses/.
- *
- * Copyright (c) 2012 Jeremy Long. All Rights Reserved.
- */
-package org.owasp.dependencycheck.data.cpe;
-
-import java.io.File;
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.Map;
-import org.apache.lucene.analysis.Analyzer;
-import org.apache.lucene.analysis.core.KeywordAnalyzer;
-import org.apache.lucene.analysis.miscellaneous.PerFieldAnalyzerWrapper;
-import org.apache.lucene.document.Document;
-import org.apache.lucene.document.Field;
-import org.apache.lucene.document.TextField;
-import org.apache.lucene.index.CorruptIndexException;
-import org.apache.lucene.index.Term;
-import org.apache.lucene.queryparser.classic.QueryParser;
-import org.apache.lucene.store.Directory;
-import org.apache.lucene.store.FSDirectory;
-import org.apache.lucene.util.Version;
-import org.owasp.dependencycheck.data.lucene.AbstractIndex;
-import org.owasp.dependencycheck.utils.Settings;
-import org.owasp.dependencycheck.data.lucene.FieldAnalyzer;
-import org.owasp.dependencycheck.data.lucene.SearchFieldAnalyzer;
-
-/**
- * The Index class is used to utilize and maintain the CPE Index.
- *
- * @author Jeremy Long (jeremy.long@owasp.org)
- */
-public class Index extends AbstractIndex {
-
-    /**
-     * Returns the directory that holds the CPE Index.
-     *
-     * @return the Directory containing the CPE Index.
-     * @throws IOException is thrown if an IOException occurs.
-     */
-    @Override
-    public Directory getDirectory() throws IOException {
-        final File path = getDataDirectory();
-        return FSDirectory.open(path);
-    }
-
-    /**
-     * Retrieves the directory that the JAR file exists in so that we can ensure
-     * we always use a common data directory.
-     *
-     * @return the data directory for this index.
-     * @throws IOException is thrown if an IOException occurs of course...
-     */
-    public File getDataDirectory() throws IOException {
-        final File path = Settings.getFile(Settings.KEYS.CPE_DATA_DIRECTORY);
-        if (!path.exists()) {
-            if (!path.mkdirs()) {
-                throw new IOException("Unable to create CPE Data directory");
-            }
-        }
-        return path;
-    }
-
-    /**
-     * Creates an Analyzer for the CPE Index.
-     *
-     * @return the CPE Analyzer.
-     */
-    @SuppressWarnings("unchecked")
-    @Override
-    public Analyzer createIndexingAnalyzer() {
-        final Map fieldAnalyzers = new HashMap();
-        fieldAnalyzers.put(Fields.DOCUMENT_KEY, new KeywordAnalyzer());
-        return new PerFieldAnalyzerWrapper(new FieldAnalyzer(Version.LUCENE_43), fieldAnalyzers);
-    }
-    /**
-     * The search field analyzer for the product field.
-     */
-    private SearchFieldAnalyzer productSearchFieldAnalyzer;
-    /**
-     * The search field analyzer for the vendor field.
-     */
-    private SearchFieldAnalyzer vendorSearchFieldAnalyzer;
-
-    /**
-     * Creates an Analyzer for searching the CPE Index.
-     *
-     * @return the CPE Analyzer.
-     */
-    @SuppressWarnings("unchecked")
-    @Override
-    public Analyzer createSearchingAnalyzer() {
-        final Map fieldAnalyzers = new HashMap();
-
-        fieldAnalyzers.put(Fields.DOCUMENT_KEY, new KeywordAnalyzer());
-        productSearchFieldAnalyzer = new SearchFieldAnalyzer(Version.LUCENE_43);
-        vendorSearchFieldAnalyzer = new SearchFieldAnalyzer(Version.LUCENE_43);
-        fieldAnalyzers.put(Fields.PRODUCT, productSearchFieldAnalyzer);
-        fieldAnalyzers.put(Fields.VENDOR, vendorSearchFieldAnalyzer);
-
-        return new PerFieldAnalyzerWrapper(new FieldAnalyzer(Version.LUCENE_43), fieldAnalyzers);
-    }
-
-    /**
-     * Creates the Lucene QueryParser used when querying the index.
-     *
-     * @return a QueryParser.
-     */
-    @Override
-    public QueryParser createQueryParser() {
-        return new QueryParser(Version.LUCENE_43, Fields.DOCUMENT_KEY, getSearchingAnalyzer());
-    }
-
-    /**
-     * Resets the searching analyzers
-     */
-    @Override
-    protected void resetSearchingAnalyzer() {
-        if (productSearchFieldAnalyzer != null) {
-            productSearchFieldAnalyzer.clear();
-        }
-        if (vendorSearchFieldAnalyzer != null) {
-            vendorSearchFieldAnalyzer.clear();
-        }
-    }
-
-    /**
-     * Saves a CPE IndexEntry into the Lucene index.
-     *
-     * @param entry a CPE entry.
-     * @throws CorruptIndexException is thrown if the index is corrupt.
-     * @throws IOException is thrown if an IOException occurs.
-     */
-    public void saveEntry(IndexEntry entry) throws CorruptIndexException, IOException {
-        final Document doc = convertEntryToDoc(entry);
-        final Term term = new Term(Fields.DOCUMENT_KEY, entry.getDocumentId());
-        getIndexWriter().updateDocument(term, doc);
-    }
-
-    /**
-     * Converts a CPE entry into a Lucene Document.
-     *
-     * @param entry a CPE IndexEntry.
-     * @return a Lucene Document containing a CPE IndexEntry.
-     */
-    protected Document convertEntryToDoc(IndexEntry entry) {
-        final Document doc = new Document();
-
-        final Field vendor = new TextField(Fields.VENDOR, entry.getVendor(), Field.Store.YES);
-        doc.add(vendor);
-
-        final Field product = new TextField(Fields.PRODUCT, entry.getProduct(), Field.Store.YES);
-        doc.add(product);
-        return doc;
-    }
-}

dependency-check-core/src/main/java/org/owasp/dependencycheck/data/cpe/IndexException.java

@@ -0,0 +1,67 @@
+/*
+ * This file is part of dependency-check-core.
+ *
+ * Dependency-check-core is free software: you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation, either version 3 of the License, or (at your option) any
+ * later version.
+ *
+ * Dependency-check-core is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+ * details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * dependency-check-core. If not, see http://www.gnu.org/licenses/.
+ *
+ * Copyright (c) 2012 Jeremy Long. All Rights Reserved.
+ */
+package org.owasp.dependencycheck.data.cpe;
+
+/**
+ * An exception thrown when the there is an issue using the in-memory CPE Index.
+ *
+ * @author Jeremy Long (jeremy.long@owasp.org)
+ */
+public class IndexException extends Exception {
+
+    /**
+     * The serial version UID for serialization.
+     */
+    private static final long serialVersionUID = 1L;
+
+    /**
+     * Creates a new IndexException.
+     */
+    public IndexException() {
+        super();
+    }
+
+    /**
+     * Creates a new IndexException.
+     *
+     * @param msg a message for the exception.
+     */
+    public IndexException(String msg) {
+        super(msg);
+    }
+
+    /**
+     * Creates a new IndexException.
+     *
+     * @param ex the cause of the failure.
+     */
+    public IndexException(Throwable ex) {
+        super(ex);
+    }
+
+    /**
+     * Creates a new IndexException.
+     *
+     * @param msg a message for the exception.
+     * @param ex the cause of the failure.
+     */
+    public IndexException(String msg, Throwable ex) {
+        super(msg, ex);
+    }
+}

dependency-check-core/src/main/java/org/owasp/dependencycheck/data/lucene/AbstractIndex.java

@@ -1,341 +0,0 @@
-/*
- * This file is part of dependency-check-core.
- *
- * Dependency-check-core is free software: you can redistribute it and/or modify it
- * under the terms of the GNU General Public License as published by the Free
- * Software Foundation, either version 3 of the License, or (at your option) any
- * later version.
- *
- * Dependency-check-core is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
- * details.
- *
- * You should have received a copy of the GNU General Public License along with
- * dependency-check-core. If not, see http://www.gnu.org/licenses/.
- *
- * Copyright (c) 2012 Jeremy Long. All Rights Reserved.
- */
-package org.owasp.dependencycheck.data.lucene;
-
-import java.io.IOException;
-import java.util.logging.Level;
-import java.util.logging.Logger;
-import org.apache.lucene.analysis.Analyzer;
-import org.apache.lucene.document.Document;
-import org.apache.lucene.index.CorruptIndexException;
-import org.apache.lucene.index.DirectoryReader;
-import org.apache.lucene.index.IndexReader;
-import org.apache.lucene.index.IndexWriter;
-import org.apache.lucene.index.IndexWriterConfig;
-import org.apache.lucene.queryparser.classic.ParseException;
-import org.apache.lucene.queryparser.classic.QueryParser;
-import org.apache.lucene.search.IndexSearcher;
-import org.apache.lucene.search.Query;
-import org.apache.lucene.search.TopDocs;
-import org.apache.lucene.store.Directory;
-import org.apache.lucene.store.LockObtainFailedException;
-import org.apache.lucene.util.Version;
-
-/**
- * The base Index for other index objects. Implements the open and close
- * methods.
- *
- * @author Jeremy Long (jeremy.long@owasp.org)
- */
-public abstract class AbstractIndex {
-
-    /**
-     * The Lucene directory containing the index.
-     */
-    private Directory directory;
-    /**
-     * The IndexWriter for the Lucene index.
-     */
-    private IndexWriter indexWriter;
-    /**
-     * The Lucene IndexReader.
-     */
-    private IndexReader indexReader;
-    /**
-     * The Lucene IndexSearcher.
-     */
-    private IndexSearcher indexSearcher;
-    /**
-     * The Lucene Analyzer used for Indexing.
-     */
-    private Analyzer indexingAnalyzer;
-    /**
-     * The Lucene Analyzer used for Searching.
-     */
-    private Analyzer searchingAnalyzer;
-    /**
-     * The Lucene QueryParser used for Searching.
-     */
-    private QueryParser queryParser;
-    /**
-     * Indicates whether or not the Lucene Index is open.
-     */
-    private boolean indexOpen = false;
-
-    /**
-     * Opens the CPE Index.
-     *
-     * @throws IOException is thrown if an IOException occurs opening the index.
-     */
-    public void open() throws IOException {
-        directory = this.getDirectory();
-        indexingAnalyzer = this.getIndexingAnalyzer();
-        searchingAnalyzer = this.getSearchingAnalyzer();
-        indexOpen = true;
-    }
-
-    /**
-     * Commits any pending changes.
-     */
-    public void commit() {
-        if (indexWriter != null) {
-            try {
-                indexWriter.commit();
-            } catch (CorruptIndexException ex) {
-                final String msg = "Unable to update database, there is a corrupt index.";
-                Logger.getLogger(AbstractIndex.class.getName()).log(Level.SEVERE, msg);
-                Logger.getLogger(AbstractIndex.class.getName()).log(Level.FINE, null, ex);
-            } catch (IOException ex) {
-                final String msg = "Unable to update database due to an IO error.";
-                Logger.getLogger(AbstractIndex.class.getName()).log(Level.SEVERE, msg);
-                Logger.getLogger(AbstractIndex.class.getName()).log(Level.FINE, null, ex);
-            }
-        }
-    }
-
-    /**
-     * Closes the CPE Index.
-     */
-    public void close() {
-        if (indexWriter != null) {
-            commit();
-            try {
-                indexWriter.close(true);
-            } catch (CorruptIndexException ex) {
-                final String msg = "Unable to update database, there is a corrupt index.";
-                Logger.getLogger(AbstractIndex.class.getName()).log(Level.SEVERE, msg);
-                Logger.getLogger(AbstractIndex.class.getName()).log(Level.FINE, null, ex);
-            } catch (IOException ex) {
-                final String msg = "Unable to update database due to an IO error.";
-                Logger.getLogger(AbstractIndex.class.getName()).log(Level.SEVERE, msg);
-                Logger.getLogger(AbstractIndex.class.getName()).log(Level.FINE, null, ex);
-            } finally {
-                indexWriter = null;
-            }
-        }
-        if (indexSearcher != null) {
-            indexSearcher = null;
-        }
-
-        if (indexingAnalyzer != null) {
-            indexingAnalyzer.close();
-            indexingAnalyzer = null;
-        }
-
-        if (searchingAnalyzer != null) {
-            searchingAnalyzer.close();
-            searchingAnalyzer = null;
-        }
-
-        try {
-            directory.close();
-        } catch (IOException ex) {
-            final String msg = "Unable to update database due to an IO error.";
-            Logger.getLogger(AbstractIndex.class.getName()).log(Level.SEVERE, msg);
-            Logger.getLogger(AbstractIndex.class.getName()).log(Level.FINE, null, ex);
-        } finally {
-            directory = null;
-        }
-        indexOpen = false;
-    }
-
-    /**
-     * Returns the status of the data source - is the index open.
-     *
-     * @return true or false.
-     */
-    public boolean isOpen() {
-        return indexOpen;
-    }
-
-    /**
-     * Opens the Lucene Index Writer.
-     *
-     * @throws CorruptIndexException is thrown if the Lucene index is corrupt.
-     * @throws IOException is thrown if an IOException occurs opening the index.
-     */
-    public void openIndexWriter() throws CorruptIndexException, IOException {
-        if (!isOpen()) {
-            open();
-        }
-        final IndexWriterConfig conf = new IndexWriterConfig(Version.LUCENE_43, indexingAnalyzer);
-        indexWriter = new IndexWriter(directory, conf);
-    }
-
-    /**
-     * Retrieves the IndexWriter for the Lucene Index.
-     *
-     * @return an IndexWriter.
-     * @throws CorruptIndexException is thrown if the Lucene Index is corrupt.
-     * @throws LockObtainFailedException is thrown if there is an exception
-     * obtaining a lock on the Lucene index.
-     * @throws IOException is thrown if an IOException occurs opening the index.
-     */
-    public IndexWriter getIndexWriter() throws CorruptIndexException, LockObtainFailedException, IOException {
-        if (indexWriter == null) {
-            openIndexWriter();
-        }
-        return indexWriter;
-    }
-
-    /**
-     * Opens the Lucene Index for reading.
-     *
-     * @throws CorruptIndexException is thrown if the index is corrupt.
-     * @throws IOException is thrown if there is an exception reading the index.
-     */
-    public void openIndexReader() throws CorruptIndexException, IOException {
-        if (!isOpen()) {
-            open();
-        }
-        //indexReader = IndexReader.open(directory, true);
-        indexReader = DirectoryReader.open(directory);
-    }
-
-    /**
-     * Returns an IndexSearcher for the Lucene Index.
-     *
-     * @return an IndexSearcher.
-     * @throws CorruptIndexException is thrown if the index is corrupt.
-     * @throws IOException is thrown if there is an exception reading the index.
-     */
-    protected IndexSearcher getIndexSearcher() throws CorruptIndexException, IOException {
-        if (indexReader == null) {
-            openIndexReader();
-        }
-        if (indexSearcher == null) {
-            indexSearcher = new IndexSearcher(indexReader);
-        }
-        return indexSearcher;
-    }
-
-    /**
-     * Returns an Analyzer to be used when indexing.
-     *
-     * @return an Analyzer.
-     */
-    public Analyzer getIndexingAnalyzer() {
-        if (indexingAnalyzer == null) {
-            indexingAnalyzer = createIndexingAnalyzer();
-        }
-        return indexingAnalyzer;
-    }
-
-    /**
-     * Returns an analyzer used for searching the index
-     *
-     * @return a lucene analyzer
-     */
-    protected Analyzer getSearchingAnalyzer() {
-        if (searchingAnalyzer == null) {
-            searchingAnalyzer = createSearchingAnalyzer();
-        }
-        return searchingAnalyzer;
-    }
-
-    /**
-     * Gets a query parser
-     *
-     * @return a query parser
-     */
-    protected QueryParser getQueryParser() {
-        if (queryParser == null) {
-            queryParser = createQueryParser();
-        }
-        return queryParser;
-    }
-
-    /**
-     * Searches the index using the given search string.
-     *
-     * @param searchString the query text
-     * @param maxQueryResults the maximum number of documents to return
-     * @return the TopDocs found by the search
-     * @throws ParseException thrown when the searchString is invalid
-     * @throws IOException is thrown if there is an issue with the underlying
-     * Index
-     */
-    public TopDocs search(String searchString, int maxQueryResults) throws ParseException, IOException {
-        final QueryParser parser = getQueryParser();
-        final Query query = parser.parse(searchString);
-        resetSearchingAnalyzer();
-        final IndexSearcher is = getIndexSearcher();
-        return is.search(query, maxQueryResults);
-    }
-
-    /**
-     * Searches the index using the given query.
-     *
-     * @param query the query used to search the index
-     * @param maxQueryResults the max number of results to return
-     * @return the TopDocs found be the query
-     * @throws CorruptIndexException thrown if the Index is corrupt
-     * @throws IOException thrown if there is an IOException
-     */
-    public TopDocs search(Query query, int maxQueryResults) throws CorruptIndexException, IOException {
-        final IndexSearcher is = getIndexSearcher();
-        return is.search(query, maxQueryResults);
-    }
-
-    /**
-     * Retrieves a document from the Index.
-     *
-     * @param documentId the id of the document to retrieve
-     * @return the Document
-     * @throws IOException thrown if there is an IOException
-     */
-    public Document getDocument(int documentId) throws IOException {
-        final IndexSearcher is = getIndexSearcher();
-        return is.doc(documentId);
-    }
-
-    /**
-     * Gets the directory that contains the Lucene Index.
-     *
-     * @return a Lucene Directory
-     * @throws IOException is thrown when an IOException occurs
-     */
-    public abstract Directory getDirectory() throws IOException;
-
-    /**
-     * Creates the Lucene Analyzer used when indexing.
-     *
-     * @return a Lucene Analyzer
-     */
-    public abstract Analyzer createIndexingAnalyzer();
-
-    /**
-     * Creates the Lucene Analyzer used when querying the index.
-     *
-     * @return a Lucene Analyzer
-     */
-    public abstract Analyzer createSearchingAnalyzer();
-
-    /**
-     * Creates the Lucene QueryParser used when querying the index.
-     *
-     * @return a QueryParser
-     */
-    public abstract QueryParser createQueryParser();
-
-    /**
-     * Resets the searching analyzers
-     */
-    protected abstract void resetSearchingAnalyzer();
-}

dependency-check-core/src/main/java/org/owasp/dependencycheck/data/lucene/LuceneUtils.java

@@ -18,6 +18,8 @@
  */
 package org.owasp.dependencycheck.data.lucene;
 
+import org.apache.lucene.util.Version;
+
 /**
  * <p>Lucene utils is a set of utilize written to make constructing Lucene
  * queries simpler.</p>
@@ -26,6 +28,12 @@ package org.owasp.dependencycheck.data.lucene;
  */
 public final class LuceneUtils {
 
+    /**
+     * The current version of Lucene being used. Declaring this one place so an
+     * upgrade doesn't require hunting through the code base.
+     */
+    public static final Version CURRENT_VERSION = Version.LUCENE_45;
+
     /**
      * Private constructor as this is a utility class.
      */

dependency-check-core/src/main/java/org/owasp/dependencycheck/data/lucene/TokenPairConcatenatingFilter.java

@@ -47,6 +47,24 @@ public final class TokenPairConcatenatingFilter extends TokenFilter {
      */
     private final LinkedList<String> words;
 
+    /**
+     * Returns the previous word. This is needed in the test cases.
+     *
+     * @return te previous word
+     */
+    protected String getPreviousWord() {
+        return previousWord;
+    }
+
+    /**
+     * Returns the words list. This is needed in the test cases.
+     *
+     * @return the words list
+     */
+    protected LinkedList<String> getWords() {
+        return words;
+    }
+
     /**
      * Constructs a new TokenPairConcatenatingFilter.
      *

dependency-check-core/src/main/java/org/owasp/dependencycheck/data/nvdcve/CveDB.java

@@ -18,14 +18,7 @@
  */
 package org.owasp.dependencycheck.data.nvdcve;
 
-import java.io.BufferedReader;
-import java.io.File;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.InputStreamReader;
 import java.io.UnsupportedEncodingException;
-import java.sql.Connection;
-import java.sql.DriverManager;
 import java.sql.PreparedStatement;
 import java.sql.ResultSet;
 import java.sql.SQLException;
@@ -36,84 +29,72 @@ import java.util.List;
 import java.util.Set;
 import java.util.logging.Level;
 import java.util.logging.Logger;
+import org.owasp.dependencycheck.data.BaseDB;
 import org.owasp.dependencycheck.data.cwe.CweDB;
 import org.owasp.dependencycheck.dependency.Reference;
 import org.owasp.dependencycheck.dependency.Vulnerability;
 import org.owasp.dependencycheck.dependency.VulnerableSoftware;
 import org.owasp.dependencycheck.utils.DependencyVersion;
 import org.owasp.dependencycheck.utils.DependencyVersionUtil;
-import org.owasp.dependencycheck.utils.Settings;
 
 /**
  * The database holding information about the NVD CVE data.
  *
  * @author Jeremy Long (jeremy.long@owasp.org)
  */
-public class CveDB {
+public class CveDB extends BaseDB {
 
-    /**
-     * Resource location for SQL file used to create the database schema.
-     */
-    public static final String DB_STRUCTURE_RESOURCE = "data/initialize.sql";
-    /**
-     * The version of the current DB Schema.
-     */
-    public static final String DB_SCHEMA_VERSION = "2.5";
-    /**
-     * Database connection
-     */
-    private Connection conn;
     //<editor-fold defaultstate="collapsed" desc="Constants to create, maintain, and retrieve data from the CVE Database">
     /**
      * SQL Statement to delete references by vulnerability ID.
      */
-    public static final String DELETE_REFERENCE = "DELETE FROM reference WHERE cveid = ?";
+    private static final String DELETE_REFERENCE = "DELETE FROM reference WHERE cveid = ?";
     /**
      * SQL Statement to delete software by vulnerability ID.
      */
-    public static final String DELETE_SOFTWARE = "DELETE FROM software WHERE cveid = ?";
+    private static final String DELETE_SOFTWARE = "DELETE FROM software WHERE cveid = ?";
     /**
      * SQL Statement to delete a vulnerability by CVE.
      */
-    public static final String DELETE_VULNERABILITY = "DELETE FROM vulnerability WHERE cve = ?";
+    private static final String DELETE_VULNERABILITY = "DELETE FROM vulnerability WHERE id = ?";
     /**
      * SQL Statement to cleanup orphan entries. Yes, the db schema could be a
      * little tighter, but what we have works well to keep the data file size
      * down a bit.
      */
-    public static final String CLEANUP_ORPHANS = "DELETE FROM CpeEntry WHERE id not in (SELECT CPEEntryId FROM Software); ";
+    private static final String CLEANUP_ORPHANS = "DELETE FROM CpeEntry WHERE id not in (SELECT CPEEntryId FROM Software); ";
     /**
      * SQL Statement to insert a new reference.
      */
-    public static final String INSERT_REFERENCE = "INSERT INTO reference (cveid, name, url, source) VALUES (?, ?, ?, ?)";
+    private static final String INSERT_REFERENCE = "INSERT INTO reference (cveid, name, url, source) VALUES (?, ?, ?, ?)";
     /**
      * SQL Statement to insert a new software.
      */
-    public static final String INSERT_SOFTWARE = "INSERT INTO software (cveid, cpeEntryId, previousVersion) VALUES (?, ?, ?)";
+    private static final String INSERT_SOFTWARE = "INSERT INTO software (cveid, cpeEntryId, previousVersion) VALUES (?, ?, ?)";
     /**
      * SQL Statement to insert a new cpe.
      */
-    public static final String INSERT_CPE = "INSERT INTO cpeEntry (cpe, vendor, product) VALUES (?, ?, ?)";
+    private static final String INSERT_CPE = "INSERT INTO cpeEntry (cpe, vendor, product) VALUES (?, ?, ?)";
     /**
      * SQL Statement to get a CPEProductID.
      */
-    public static final String SELECT_CPE_ID = "SELECT id FROM cpeEntry WHERE cpe = ?";
+    private static final String SELECT_CPE_ID = "SELECT id FROM cpeEntry WHERE cpe = ?";
     /**
      * SQL Statement to insert a new vulnerability.
      */
-    public static final String INSERT_VULNERABILITY = "INSERT INTO vulnerability (cve, description, cwe, cvssScore, cvssAccessVector, "
+    private static final String INSERT_VULNERABILITY = "INSERT INTO vulnerability (cve, description, cwe, cvssScore, cvssAccessVector, "
             + "cvssAccessComplexity, cvssAuthentication, cvssConfidentialityImpact, cvssIntegrityImpact, cvssAvailabilityImpact) "
             + "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)";
     /**
      * SQL Statement to update a vulnerability.
      */
-    public static final String UPDATE_VULNERABILITY = "UPDATE vulnerability SET description=?, cwe=?, cvssScore=?, cvssAccessVector=?, "
+    private static final String UPDATE_VULNERABILITY = "UPDATE vulnerability SET description=?, cwe=?, cvssScore=?, cvssAccessVector=?, "
             + "cvssAccessComplexity=?, cvssAuthentication=?, cvssConfidentialityImpact=?, cvssIntegrityImpact=?, cvssAvailabilityImpact=? "
             + "WHERE id=?";
     /**
      * SQL Statement to find CVE entries based on CPE data.
      */
-    public static final String SELECT_CVE_FROM_SOFTWARE = "SELECT cve, cpe, previousVersion "
+    private static final String SELECT_CVE_FROM_SOFTWARE = "SELECT cve, cpe, previousVersion "
             + "FROM software INNER JOIN vulnerability ON vulnerability.id = software.cveId "
             + "INNER JOIN cpeEntry ON cpeEntry.id = software.cpeEntryId "
             + "WHERE vendor = ? AND product = ?";
@@ -123,15 +104,19 @@ public class CveDB {
     /**
      * SQL Statement to find the CPE entry based on the vendor and product.
      */
-    public static final String SELECT_CPE_ENTRIES = "SELECT cpe FROM cpeEntry WHERE vendor = ? AND product = ?";
+    private static final String SELECT_CPE_ENTRIES = "SELECT cpe FROM cpeEntry WHERE vendor = ? AND product = ?";
     /**
      * SQL Statement to select references by CVEID.
      */
-    public static final String SELECT_REFERENCE = "SELECT source, name, url FROM reference WHERE cveid = ?";
+    private static final String SELECT_REFERENCE = "SELECT source, name, url FROM reference WHERE cveid = ?";
+    /**
+     * SQL Statement to select vendor and product for lucene index.
+     */
+    private static final String SELECT_VENDOR_PRODUCT_LIST = "SELECT vendor, product FROM cpeEntry GROUP BY vendor, product";
     /**
      * SQL Statement to select software by CVEID.
      */
-    public static final String SELECT_SOFTWARE = "SELECT cpe, previousVersion "
+    private static final String SELECT_SOFTWARE = "SELECT cpe, previousVersion "
             + "FROM software INNER JOIN cpeEntry ON software.cpeEntryId = cpeEntry.id WHERE cveid = ?";
 //    public static final String SELECT_SOFTWARE = "SELECT part, vendor, product, version, revision, previousVersion "
 //            + "FROM software INNER JOIN cpeProduct ON cpeProduct.id = software.cpeProductId LEFT JOIN cpeVersion ON "
@@ -139,91 +124,14 @@ public class CveDB {
     /**
      * SQL Statement to select a vulnerability by CVEID.
      */
-    public static final String SELECT_VULNERABILITY = "SELECT id, description, cwe, cvssScore, cvssAccessVector, cvssAccessComplexity, "
+    private static final String SELECT_VULNERABILITY = "SELECT id, description, cwe, cvssScore, cvssAccessVector, cvssAccessComplexity, "
             + "cvssAuthentication, cvssConfidentialityImpact, cvssIntegrityImpact, cvssAvailabilityImpact FROM vulnerability WHERE cve = ?";
     /**
      * SQL Statement to select a vulnerability's primary key.
      */
-    public static final String SELECT_VULNERABILITY_ID = "SELECT id FROM vulnerability WHERE cve = ?";
+    private static final String SELECT_VULNERABILITY_ID = "SELECT id FROM vulnerability WHERE cve = ?";
     //</editor-fold>
 
-    /**
-     * Opens the database connection. If the database does not exist, it will
-     * create a new one.
-     *
-     * @throws IOException thrown if there is an IO Exception
-     * @throws SQLException thrown if there is a SQL Exception
-     * @throws DatabaseException thrown if there is an error initializing a new
-     * database
-     * @throws ClassNotFoundException thrown if the h2 database driver cannot be
-     * loaded
-     */
-    @edu.umd.cs.findbugs.annotations.SuppressWarnings(
-            value = "DMI_EMPTY_DB_PASSWORD",
-            justification = "Yes, I know... Blank password.")
-    public void open() throws IOException, SQLException, DatabaseException, ClassNotFoundException {
-        /*
-         * TODO - make it so we can exteralize the database (lucene index is a problem), could I store it as a blob
-         *        and just download it when needed?
-         */
-//        String dbDriver = Settings.getString(Settings.KEYS.DB_DRIVER);
-//        String dbConnStr = Settings.getString(Settings.KEYS.DB_CONNECTION_STRING);
-//        if (dbDriver != null && dbConnStr != null) {
-//            Class.forName(dbDriver);
-//            conn = DriverManager.getConnection(dbConnStr);
-//        } else { //use the embeded version
-        final String fileName = CveDB.getDataDirectory().getCanonicalPath();
-        final File f = new File(fileName, "cve." + DB_SCHEMA_VERSION);
-        final File check = new File(f.getAbsolutePath() + ".h2.db");
-        final boolean createTables = !check.exists();
-        final String connStr = "jdbc:h2:file:" + f.getAbsolutePath();
-        Class.forName("org.h2.Driver");
-        conn = DriverManager.getConnection(connStr, "sa", "");
-        if (createTables) {
-            createTables();
-        }
-//        }
-    }
-
-    /**
-     * Commits all completed transactions.
-     *
-     * @throws SQLException thrown if a SQL Exception occurs
-     */
-    public void commit() throws SQLException {
-        if (conn != null) {
-            conn.commit();
-        }
-    }
-
-    /**
-     * Cleans up the object and ensures that "close" has been called.
-     *
-     * @throws Throwable thrown if there is a problem
-     */
-    @Override
-    protected void finalize() throws Throwable {
-        close();
-        super.finalize(); //not necessary if extending Object.
-    }
-
-    /**
-     * Closes the DB4O database. Close should be called on this object when it
-     * is done being used.
-     */
-    public void close() {
-        if (conn != null) {
-            try {
-                conn.close();
-            } catch (SQLException ex) {
-                final String msg = "There was an error attempting to close the CveDB, see the log for more details.";
-                Logger.getLogger(CveDB.class.getName()).log(Level.SEVERE, msg, ex);
-                Logger.getLogger(CveDB.class.getName()).log(Level.FINE, null, ex);
-            }
-            conn = null;
-        }
-    }
-
     /**
      * Searches the CPE entries in the database and retrieves all entries for a
      * given vendor and product combination. The returned list will include all
@@ -239,7 +147,7 @@ public class CveDB {
         ResultSet rs = null;
         PreparedStatement ps = null;
         try {
-            ps = conn.prepareStatement(SELECT_CPE_ENTRIES);
+            ps = getConnection().prepareStatement(SELECT_CPE_ENTRIES);
             ps.setString(1, vendor);
             ps.setString(2, product);
             rs = ps.executeQuery();
@@ -258,6 +166,22 @@ public class CveDB {
         return cpe;
     }
 
+    /**
+     * Returns the entire list of vendor/product combinations.
+     *
+     * @return the entire list of vendor/product combinations.
+     */
+    public ResultSet getVendorProductList() {
+        ResultSet rs = null;
+        try {
+            final PreparedStatement ps = getConnection().prepareStatement(SELECT_VENDOR_PRODUCT_LIST);
+            rs = ps.executeQuery();
+        } catch (SQLException ex) {
+            Logger.getLogger(CveDB.class.getName()).log(Level.SEVERE, null, ex);
+        } // can't close the statement in the PS as the resultset is returned, closing PS would close the resultset
+        return rs;
+    }
+
     /**
      * Retrieves the vulnerabilities associated with the specified CPE.
      *
@@ -279,7 +203,7 @@ public class CveDB {
         PreparedStatement ps;
         final HashSet<String> cveEntries = new HashSet<String>();
         try {
-            ps = conn.prepareStatement(SELECT_CVE_FROM_SOFTWARE);
+            ps = getConnection().prepareStatement(SELECT_CVE_FROM_SOFTWARE);
             ps.setString(1, cpe.getVendor());
             ps.setString(2, cpe.getProduct());
             rs = ps.executeQuery();
@@ -322,7 +246,7 @@ public class CveDB {
         ResultSet rsS = null;
         Vulnerability vuln = null;
         try {
-            psV = conn.prepareStatement(SELECT_VULNERABILITY);
+            psV = getConnection().prepareStatement(SELECT_VULNERABILITY);
             psV.setString(1, cve);
             rsV = psV.executeQuery();
             if (rsV.next()) {
@@ -346,13 +270,13 @@ public class CveDB {
                 vuln.setCvssIntegrityImpact(rsV.getString(9));
                 vuln.setCvssAvailabilityImpact(rsV.getString(10));
 
-                psR = conn.prepareStatement(SELECT_REFERENCE);
+                psR = getConnection().prepareStatement(SELECT_REFERENCE);
                 psR.setInt(1, cveId);
                 rsR = psR.executeQuery();
                 while (rsR.next()) {
                     vuln.addReference(rsR.getString(1), rsR.getString(2), rsR.getString(3));
                 }
-                psS = conn.prepareStatement(SELECT_SOFTWARE);
+                psS = getConnection().prepareStatement(SELECT_SOFTWARE);
                 psS.setInt(1, cveId);
                 rsS = psS.executeQuery();
                 while (rsS.next()) {
@@ -387,6 +311,7 @@ public class CveDB {
      */
     public void updateVulnerability(Vulnerability vuln) throws DatabaseException {
         PreparedStatement selectVulnerabilityId = null;
+        PreparedStatement deleteVulnerability = null;
         PreparedStatement deleteReferences = null;
         PreparedStatement deleteSoftware = null;
         PreparedStatement updateVulnerability = null;
@@ -397,15 +322,16 @@ public class CveDB {
         PreparedStatement insertSoftware = null;
 
         try {
-            selectVulnerabilityId = conn.prepareStatement(SELECT_VULNERABILITY_ID);
-            deleteReferences = conn.prepareStatement(DELETE_REFERENCE);
-            deleteSoftware = conn.prepareStatement(DELETE_SOFTWARE);
-            updateVulnerability = conn.prepareStatement(UPDATE_VULNERABILITY);
-            insertVulnerability = conn.prepareStatement(INSERT_VULNERABILITY, Statement.RETURN_GENERATED_KEYS);
-            insertReference = conn.prepareStatement(INSERT_REFERENCE);
-            selectCpeId = conn.prepareStatement(SELECT_CPE_ID);
-            insertCpe = conn.prepareStatement(INSERT_CPE, Statement.RETURN_GENERATED_KEYS);
-            insertSoftware = conn.prepareStatement(INSERT_SOFTWARE);
+            selectVulnerabilityId = getConnection().prepareStatement(SELECT_VULNERABILITY_ID);
+            deleteVulnerability = getConnection().prepareStatement(DELETE_VULNERABILITY);
+            deleteReferences = getConnection().prepareStatement(DELETE_REFERENCE);
+            deleteSoftware = getConnection().prepareStatement(DELETE_SOFTWARE);
+            updateVulnerability = getConnection().prepareStatement(UPDATE_VULNERABILITY);
+            insertVulnerability = getConnection().prepareStatement(INSERT_VULNERABILITY, Statement.RETURN_GENERATED_KEYS);
+            insertReference = getConnection().prepareStatement(INSERT_REFERENCE);
+            selectCpeId = getConnection().prepareStatement(SELECT_CPE_ID);
+            insertCpe = getConnection().prepareStatement(INSERT_CPE, Statement.RETURN_GENERATED_KEYS);
+            insertSoftware = getConnection().prepareStatement(INSERT_SOFTWARE);
             int vulnerabilityId = 0;
             selectVulnerabilityId.setString(1, vuln.getName());
             ResultSet rs = selectVulnerabilityId.executeQuery();
@@ -420,17 +346,22 @@ public class CveDB {
             closeResultSet(rs);
             rs = null;
             if (vulnerabilityId != 0) {
-                updateVulnerability.setString(1, vuln.getDescription());
-                updateVulnerability.setString(2, vuln.getCwe());
-                updateVulnerability.setFloat(3, vuln.getCvssScore());
-                updateVulnerability.setString(4, vuln.getCvssAccessVector());
-                updateVulnerability.setString(5, vuln.getCvssAccessComplexity());
-                updateVulnerability.setString(6, vuln.getCvssAuthentication());
-                updateVulnerability.setString(7, vuln.getCvssConfidentialityImpact());
-                updateVulnerability.setString(8, vuln.getCvssIntegrityImpact());
-                updateVulnerability.setString(9, vuln.getCvssAvailabilityImpact());
-                updateVulnerability.setInt(10, vulnerabilityId);
-                updateVulnerability.executeUpdate();
+                if (vuln.getDescription().contains("** REJECT **")) {
+                    deleteVulnerability.setInt(1, vulnerabilityId);
+                    deleteVulnerability.executeUpdate();
+                } else {
+                    updateVulnerability.setString(1, vuln.getDescription());
+                    updateVulnerability.setString(2, vuln.getCwe());
+                    updateVulnerability.setFloat(3, vuln.getCvssScore());
+                    updateVulnerability.setString(4, vuln.getCvssAccessVector());
+                    updateVulnerability.setString(5, vuln.getCvssAccessComplexity());
+                    updateVulnerability.setString(6, vuln.getCvssAuthentication());
+                    updateVulnerability.setString(7, vuln.getCvssConfidentialityImpact());
+                    updateVulnerability.setString(8, vuln.getCvssIntegrityImpact());
+                    updateVulnerability.setString(9, vuln.getCvssAvailabilityImpact());
+                    updateVulnerability.setInt(10, vulnerabilityId);
+                    updateVulnerability.executeUpdate();
+                }
             } else {
                 insertVulnerability.setString(1, vuln.getName());
                 insertVulnerability.setString(2, vuln.getDescription());
@@ -507,6 +438,7 @@ public class CveDB {
             closeStatement(deleteReferences);
             closeStatement(deleteSoftware);
             closeStatement(updateVulnerability);
+            closeStatement(deleteVulnerability);
             closeStatement(insertVulnerability);
             closeStatement(insertReference);
             closeStatement(selectCpeId);
@@ -515,23 +447,6 @@ public class CveDB {
         }
     }
 
-    /**
-     * Retrieves the directory that the JAR file exists in so that we can ensure
-     * we always use a common data directory.
-     *
-     * @return the data directory for this index.
-     * @throws IOException is thrown if an IOException occurs of course...
-     */
-    public static File getDataDirectory() throws IOException {
-        final File path = Settings.getFile(Settings.KEYS.CVE_DATA_DIRECTORY);
-        if (!path.exists()) {
-            if (!path.mkdirs()) {
-                throw new IOException("Unable to create NVD CVE Data directory");
-            }
-        }
-        return path;
-    }
-
     /**
      * It is possible that orphaned rows may be generated during database
      * updates. This should be called after all updates have been completed to
@@ -540,7 +455,7 @@ public class CveDB {
     public void cleanupDatabase() {
         PreparedStatement ps = null;
         try {
-            ps = conn.prepareStatement(CLEANUP_ORPHANS);
+            ps = getConnection().prepareStatement(CLEANUP_ORPHANS);
             if (ps != null) {
                 ps.executeUpdate();
             }
@@ -551,102 +466,6 @@ public class CveDB {
         }
     }
 
-    /**
-     * Creates the database structure (tables and indexes) to store the CVE data
-     *
-     * @throws SQLException thrown if there is a sql exception
-     * @throws DatabaseException thrown if there is a database exception
-     */
-    protected void createTables() throws SQLException, DatabaseException {
-        InputStream is;
-        InputStreamReader reader;
-        BufferedReader in = null;
-        try {
-            is = this.getClass().getClassLoader().getResourceAsStream(DB_STRUCTURE_RESOURCE);
-            reader = new InputStreamReader(is, "UTF-8");
-            in = new BufferedReader(reader);
-            final StringBuilder sb = new StringBuilder(2110);
-            String tmp;
-            while ((tmp = in.readLine()) != null) {
-                sb.append(tmp);
-            }
-            Statement statement = null;
-            try {
-                statement = conn.createStatement();
-                statement.execute(sb.toString());
-            } finally {
-                closeStatement(statement);
-            }
-        } catch (IOException ex) {
-            throw new DatabaseException("Unable to create database schema", ex);
-        } finally {
-            if (in != null) {
-                try {
-                    in.close();
-                } catch (IOException ex) {
-                    Logger.getLogger(CveDB.class
-                            .getName()).log(Level.FINEST, null, ex);
-                }
-            }
-        }
-    }
-
-    /**
-     * Closes the given statement object ignoring any exceptions that occur.
-     *
-     * @param statement a Statement object
-     */
-    private void closeStatement(Statement statement) {
-        if (statement != null) {
-            try {
-                statement.close();
-            } catch (SQLException ex) {
-                Logger.getLogger(CveDB.class
-                        .getName()).log(Level.FINEST, statement.toString(), ex);
-            }
-        }
-    }
-
-    /**
-     * Closes the result set capturing and ignoring any SQLExceptions that
-     * occur.
-     *
-     * @param rs a ResultSet to close
-     */
-    private void closeResultSet(ResultSet rs) {
-        if (rs != null) {
-            try {
-                rs.close();
-            } catch (SQLException ex) {
-                Logger.getLogger(CveDB.class
-                        .getName()).log(Level.FINEST, rs.toString(), ex);
-            }
-        }
-    }
-
-    /**
-     * Returns the generated integer primary key for a newly inserted row.
-     *
-     * @param statement a prepared statement that just executed an insert
-     * @return a primary key
-     * @throws DatabaseException thrown if there is an exception obtaining the
-     * key
-     */
-    private int getGeneratedKey(PreparedStatement statement) throws DatabaseException {
-        ResultSet rs = null;
-        int id = 0;
-        try {
-            rs = statement.getGeneratedKeys();
-            rs.next();
-            id = rs.getInt(1);
-        } catch (SQLException ex) {
-            throw new DatabaseException("Unable to get primary key for inserted row");
-        } finally {
-            closeResultSet(rs);
-        }
-        return id;
-    }
-
     /**
      * Determines if the given identifiedVersion is affected by the given cpeId
      * and previous version flag. A non-null, non-empty string passed to the

dependency-check-core/src/main/java/org/owasp/dependencycheck/data/nvdcve/DatabaseException.java

@@ -39,13 +39,22 @@ public class DatabaseException extends Exception {
         super(msg);
     }
 
+    /**
+     * Creates an DatabaseException.
+     *
+     * @param ex the cause of the exception
+     */
+    public DatabaseException(Throwable ex) {
+        super(ex);
+    }
+
     /**
      * Creates an DatabaseException.
      *
      * @param msg the exception message
      * @param ex the cause of the exception
      */
-    public DatabaseException(String msg, Exception ex) {
+    public DatabaseException(String msg, Throwable ex) {
         super(msg, ex);
     }
 }

dependency-check-core/src/main/java/org/owasp/dependencycheck/data/nvdcve/InvalidDataException.java

@@ -16,7 +16,7 @@
  *
  * Copyright (c) 2012 Jeremy Long. All Rights Reserved.
  */
-package org.owasp.dependencycheck.data.nvdcve.xml;
+package org.owasp.dependencycheck.data.nvdcve;
 
 /**
  * An InvalidDataDataException is a generic exception used when trying to load

dependency-check-core/src/main/java/org/owasp/dependencycheck/data/nvdcve/NvdCve12Handler.java

@@ -16,7 +16,7 @@
  *
  * Copyright (c) 2012 Jeremy Long. All Rights Reserved.
  */
-package org.owasp.dependencycheck.data.nvdcve.xml;
+package org.owasp.dependencycheck.data.nvdcve;
 
 import java.util.ArrayList;
 import java.util.HashMap;

dependency-check-core/src/main/java/org/owasp/dependencycheck/data/nvdcve/NvdCve20Handler.java

@@ -16,7 +16,7 @@
  *
  * Copyright (c) 2012 Jeremy Long. All Rights Reserved.
  */
-package org.owasp.dependencycheck.data.nvdcve.xml;
+package org.owasp.dependencycheck.data.nvdcve;
 
 import java.io.IOException;
 import java.util.List;
@@ -24,9 +24,6 @@ import java.util.Map;
 import java.util.logging.Level;
 import java.util.logging.Logger;
 import org.apache.lucene.index.CorruptIndexException;
-import org.owasp.dependencycheck.data.cpe.Index;
-import org.owasp.dependencycheck.data.nvdcve.CveDB;
-import org.owasp.dependencycheck.data.nvdcve.DatabaseException;
 import org.owasp.dependencycheck.dependency.Reference;
 import org.owasp.dependencycheck.dependency.Vulnerability;
 import org.owasp.dependencycheck.dependency.VulnerableSoftware;
@@ -210,6 +207,9 @@ public class NvdCve20Handler extends DefaultHandler {
             nodeText = null;
         } else if (current.isVulnSummaryNode()) {
             vulnerability.setDescription(nodeText.toString());
+            if (nodeText.indexOf("** REJECT **") >= 0) {
+                hasApplicationCpe = true; //ensure we process this to delete the vuln
+            }
             nodeText = null;
         }
     }
@@ -243,9 +243,7 @@ public class NvdCve20Handler extends DefaultHandler {
     }
 
     /**
-     * Saves a vulnerability to the CVE Database. This is a callback method
-     * called by the Sax Parser Handler
-     * {@link org.owasp.dependencycheck.data.nvdcve.xml.NvdCve20Handler}.
+     * Saves a vulnerability to the CVE Database.
      *
      * @param vuln the vulnerability to store in the database
      * @throws DatabaseException thrown if there is an error writing to the
@@ -253,7 +251,7 @@ public class NvdCve20Handler extends DefaultHandler {
      * @throws CorruptIndexException is thrown if the CPE Index is corrupt
      * @throws IOException thrown if there is an IOException with the CPE Index
      */
-    public void saveEntry(Vulnerability vuln) throws DatabaseException, CorruptIndexException, IOException {
+    private void saveEntry(Vulnerability vuln) throws DatabaseException, CorruptIndexException, IOException {
         if (cveDB == null) {
             return;
         }
@@ -264,26 +262,8 @@ public class NvdCve20Handler extends DefaultHandler {
                 vuln.updateVulnerableSoftware(vs);
             }
         }
-        for (VulnerableSoftware vs : vuln.getVulnerableSoftware()) {
-            if (cpeIndex != null) {
-                cpeIndex.saveEntry(vs);
-            }
-        }
         cveDB.updateVulnerability(vuln);
     }
-    /**
-     * the cpe index.
-     */
-    private Index cpeIndex;
-
-    /**
-     * Sets the cpe index.
-     *
-     * @param index the CPE Lucene Index
-     */
-    void setCpeIndex(Index index) {
-        cpeIndex = index;
-    }
 
     // <editor-fold defaultstate="collapsed" desc="The Element Class that maintains state information about the current node">
     /**

dependency-check-core/src/main/java/org/owasp/dependencycheck/data/nvdcve/xml/DatabaseUpdater.java

@@ -1,671 +0,0 @@
-/*
- * This file is part of dependency-check-core.
- *
- * Dependency-check-core is free software: you can redistribute it and/or modify it
- * under the terms of the GNU General Public License as published by the Free
- * Software Foundation, either version 3 of the License, or (at your option) any
- * later version.
- *
- * Dependency-check-core is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
- * details.
- *
- * You should have received a copy of the GNU General Public License along with
- * dependency-check-core. If not, see http://www.gnu.org/licenses/.
- *
- * Copyright (c) 2012 Jeremy Long. All Rights Reserved.
- */
-package org.owasp.dependencycheck.data.nvdcve.xml;
-
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileNotFoundException;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.io.OutputStreamWriter;
-import javax.xml.parsers.ParserConfigurationException;
-import org.owasp.dependencycheck.data.CachedWebDataSource;
-import java.net.MalformedURLException;
-import java.net.URL;
-import java.sql.SQLException;
-import java.util.Calendar;
-import java.util.Date;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Properties;
-import java.util.logging.Level;
-import java.util.logging.Logger;
-import javax.xml.parsers.SAXParser;
-import javax.xml.parsers.SAXParserFactory;
-import org.owasp.dependencycheck.data.UpdateException;
-import org.owasp.dependencycheck.data.cpe.Index;
-import org.owasp.dependencycheck.data.nvdcve.CveDB;
-import org.owasp.dependencycheck.dependency.VulnerableSoftware;
-import org.owasp.dependencycheck.utils.DownloadFailedException;
-import org.owasp.dependencycheck.utils.Downloader;
-import org.owasp.dependencycheck.utils.FileUtils;
-import org.owasp.dependencycheck.utils.InvalidSettingException;
-import org.owasp.dependencycheck.utils.Settings;
-import org.xml.sax.SAXException;
-import org.owasp.dependencycheck.data.nvdcve.DatabaseException;
-
-/**
- *
- * @author Jeremy Long (jeremy.long@owasp.org)
- */
-public class DatabaseUpdater implements CachedWebDataSource {
-
-    /**
-     * The name of the properties file containing the timestamp of the last
-     * update.
-     */
-    private static final String UPDATE_PROPERTIES_FILE = "lastupdated.prop";
-    /**
-     * The properties file key for the last updated field - used to store the
-     * last updated time of the Modified NVD CVE xml file.
-     */
-    private static final String LAST_UPDATED_MODIFIED = "lastupdated.modified";
-    /**
-     * Stores the last updated time for each of the NVD CVE files. These
-     * timestamps should be updated if we process the modified file within 7
-     * days of the last update.
-     */
-    private static final String LAST_UPDATED_BASE = "lastupdated.";
-    /**
-     * Modified key word.
-     */
-    public static final String MODIFIED = "modified";
-    /**
-     * Reference to the Cve Database.
-     */
-    private CveDB cveDB = null;
-    /**
-     * Reference to the Cpe Index.
-     */
-    private Index cpeIndex = null;
-
-    /**
-     * <p>Downloads the latest NVD CVE XML file from the web and imports it into
-     * the current CVE Database.</p>
-     *
-     * @throws UpdateException is thrown if there is an error updating the
-     * database
-     */
-    public void update() throws UpdateException {
-        try {
-            final Map<String, NvdCveUrl> update = updateNeeded();
-            int maxUpdates = 0;
-            for (NvdCveUrl cve : update.values()) {
-                if (cve.getNeedsUpdate()) {
-                    maxUpdates += 1;
-                }
-            }
-            if (maxUpdates > 3) {
-                Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.INFO,
-                        "NVD CVE requires several updates; this could take a couple of minutes.");
-            }
-            if (maxUpdates > 0) {
-                openDataStores();
-            }
-            int count = 0;
-
-            for (NvdCveUrl cve : update.values()) {
-                if (cve.getNeedsUpdate()) {
-                    count += 1;
-                    Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.INFO,
-                            "Updating NVD CVE ({0} of {1})", new Object[]{count, maxUpdates});
-                    URL url = new URL(cve.getUrl());
-                    File outputPath = null;
-                    File outputPath12 = null;
-                    try {
-                        Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.INFO,
-                                "Downloading {0}", cve.getUrl());
-
-                        outputPath = File.createTempFile("cve" + cve.getId() + "_", ".xml");
-                        Downloader.fetchFile(url, outputPath, false);
-
-                        url = new URL(cve.getOldSchemaVersionUrl());
-                        outputPath12 = File.createTempFile("cve_1_2_" + cve.getId() + "_", ".xml");
-                        Downloader.fetchFile(url, outputPath12, false);
-
-                        Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.INFO,
-                                "Processing {0}", cve.getUrl());
-
-                        importXML(outputPath, outputPath12);
-
-                        cveDB.commit();
-                        cpeIndex.commit();
-
-                        writeLastUpdatedPropertyFile(cve);
-
-                        Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.INFO,
-                                "Completed update {0} of {1}", new Object[]{count, maxUpdates});
-                    } catch (FileNotFoundException ex) {
-                        throw new UpdateException(ex);
-                    } catch (ParserConfigurationException ex) {
-                        throw new UpdateException(ex);
-                    } catch (SAXException ex) {
-                        throw new UpdateException(ex);
-                    } catch (IOException ex) {
-                        throw new UpdateException(ex);
-                    } catch (SQLException ex) {
-                        throw new UpdateException(ex);
-                    } catch (DatabaseException ex) {
-                        throw new UpdateException(ex);
-                    } catch (ClassNotFoundException ex) {
-                        throw new UpdateException(ex);
-                    } finally {
-                        boolean deleted = false;
-                        try {
-                            if (outputPath != null && outputPath.exists()) {
-                                deleted = outputPath.delete();
-                            }
-                        } finally {
-                            if (outputPath != null && (outputPath.exists() || !deleted)) {
-                                outputPath.deleteOnExit();
-                            }
-                        }
-                        try {
-                            deleted = false;
-                            if (outputPath12 != null && outputPath12.exists()) {
-                                deleted = outputPath12.delete();
-                            }
-                        } finally {
-                            if (outputPath12 != null && (outputPath12.exists() || !deleted)) {
-                                outputPath12.deleteOnExit();
-                            }
-                        }
-                    }
-                }
-            }
-            if (maxUpdates >= 1) {
-                ensureModifiedIsInLastUpdatedProperties(update);
-                cveDB.cleanupDatabase();
-            }
-        } catch (MalformedURLException ex) {
-            throw new UpdateException(ex);
-        } catch (DownloadFailedException ex) {
-            throw new UpdateException(ex);
-        } finally {
-            closeDataStores();
-        }
-    }
-
-    /**
-     * Imports the NVD CVE XML File into the Lucene Index.
-     *
-     * @param file the file containing the NVD CVE XML
-     * @param oldVersion contains the file containing the NVD CVE XML 1.2
-     * @throws ParserConfigurationException is thrown if there is a parser
-     * configuration exception
-     * @throws SAXException is thrown if there is a SAXException
-     * @throws IOException is thrown if there is a ioexception
-     * @throws SQLException is thrown if there is a sql exception
-     * @throws DatabaseException is thrown if there is a database exception
-     * @throws ClassNotFoundException thrown if the h2 database driver cannot be
-     * loaded
-     */
-    private void importXML(File file, File oldVersion)
-            throws ParserConfigurationException, SAXException, IOException, SQLException, DatabaseException, ClassNotFoundException {
-
-        final SAXParserFactory factory = SAXParserFactory.newInstance();
-        final SAXParser saxParser = factory.newSAXParser();
-
-        final NvdCve12Handler cve12Handler = new NvdCve12Handler();
-        saxParser.parse(oldVersion, cve12Handler);
-        final Map<String, List<VulnerableSoftware>> prevVersionVulnMap = cve12Handler.getVulnerabilities();
-
-        final NvdCve20Handler cve20Handler = new NvdCve20Handler();
-        cve20Handler.setCveDB(cveDB);
-        cve20Handler.setPrevVersionVulnMap(prevVersionVulnMap);
-        cve20Handler.setCpeIndex(cpeIndex);
-        saxParser.parse(file, cve20Handler);
-    }
-
-    /**
-     * Closes the CVE and CPE data stores.
-     */
-    private void closeDataStores() {
-        if (cveDB != null) {
-            try {
-                cveDB.close();
-            } catch (Exception ignore) {
-                Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.FINEST, "Error closing the cveDB", ignore);
-            }
-        }
-        if (cpeIndex != null) {
-            try {
-                cpeIndex.close();
-            } catch (Exception ignore) {
-                Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.FINEST, "Error closing the cpeIndex", ignore);
-            }
-        }
-    }
-
-    /**
-     * Opens the CVE and CPE data stores.
-     *
-     * @throws UpdateException thrown if a data store cannot be opened
-     */
-    private void openDataStores() throws UpdateException {
-        //open the cve and cpe data stores
-        try {
-            cveDB = new CveDB();
-            cveDB.open();
-            cpeIndex = new Index();
-            cpeIndex.openIndexWriter();
-        } catch (IOException ex) {
-            closeDataStores();
-            Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.FINE, "IO Error opening databases", ex);
-            throw new UpdateException("Error updating the CPE/CVE data, please see the log file for more details.");
-        } catch (SQLException ex) {
-            closeDataStores();
-            Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.FINE, "SQL Exception opening databases", ex);
-            throw new UpdateException("Error updating the CPE/CVE data, please see the log file for more details.");
-        } catch (DatabaseException ex) {
-            closeDataStores();
-            Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.FINE, "Database Exception opening databases", ex);
-            throw new UpdateException("Error updating the CPE/CVE data, please see the log file for more details.");
-        } catch (ClassNotFoundException ex) {
-            closeDataStores();
-            Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.FINE, "Class not found exception opening databases", ex);
-            throw new UpdateException("Error updating the CPE/CVE data, please see the log file for more details.");
-        }
-    }
-
-    //<editor-fold defaultstate="collapsed" desc="Code to read/write properties files regarding the last update dates">
-    /**
-     * Writes a properties file containing the last updated date to the
-     * VULNERABLE_CPE directory.
-     *
-     * @param updatedValue the updated nvdcve entry
-     * @throws UpdateException is thrown if there is an update exception
-     */
-    private void writeLastUpdatedPropertyFile(NvdCveUrl updatedValue) throws UpdateException {
-        if (updatedValue == null) {
-            return;
-        }
-        String dir;
-        try {
-            dir = CveDB.getDataDirectory().getCanonicalPath();
-        } catch (IOException ex) {
-            Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.FINE, "Error updating the databases propterty file.", ex);
-            throw new UpdateException("Unable to locate last updated properties file.", ex);
-        }
-        final File cveProp = new File(dir, UPDATE_PROPERTIES_FILE);
-        final Properties prop = new Properties();
-        if (cveProp.exists()) {
-            FileInputStream in = null;
-            try {
-                in = new FileInputStream(cveProp);
-                prop.load(in);
-            } catch (Exception ignoreMe) {
-                Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.FINEST, null, ignoreMe);
-            } finally {
-                if (in != null) {
-                    try {
-                        in.close();
-                    } catch (Exception ignoreMeToo) {
-                        Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.FINEST, null, ignoreMeToo);
-                    }
-                }
-            }
-
-        }
-        prop.put("version", CveDB.DB_SCHEMA_VERSION);
-        prop.put(LAST_UPDATED_BASE + updatedValue.getId(), String.valueOf(updatedValue.getTimestamp()));
-
-        OutputStream os = null;
-        OutputStreamWriter out = null;
-        try {
-            os = new FileOutputStream(cveProp);
-            out = new OutputStreamWriter(os, "UTF-8");
-            prop.store(out, dir);
-        } catch (FileNotFoundException ex) {
-            Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.FINE, null, ex);
-            throw new UpdateException("Unable to find last updated properties file.", ex);
-        } catch (IOException ex) {
-            Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.FINE, null, ex);
-            throw new UpdateException("Unable to update last updated properties file.", ex);
-        } finally {
-            if (out != null) {
-                try {
-                    out.close();
-                } catch (IOException ex) {
-                    Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.FINEST, null, ex);
-                }
-            }
-            if (os != null) {
-                try {
-                    os.close();
-                } catch (IOException ex) {
-                    Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.FINEST, null, ex);
-                }
-            }
-        }
-    }
-
-    /**
-     * Determines if the index needs to be updated. This is done by fetching the
-     * nvd cve meta data and checking the last update date. If the data needs to
-     * be refreshed this method will return the NvdCveUrl for the files that
-     * need to be updated.
-     *
-     * @return the NvdCveUrl of the files that need to be updated.
-     * @throws MalformedURLException is thrown if the URL for the NVD CVE Meta
-     * data is incorrect.
-     * @throws DownloadFailedException is thrown if there is an error.
-     * downloading the nvd cve download data file.
-     * @throws UpdateException Is thrown if there is an issue with the last
-     * updated properties file.
-     */
-    public Map<String, NvdCveUrl> updateNeeded() throws MalformedURLException, DownloadFailedException, UpdateException {
-
-        Map<String, NvdCveUrl> currentlyPublished;
-        try {
-            currentlyPublished = retrieveCurrentTimestampsFromWeb();
-        } catch (InvalidDataException ex) {
-            final String msg = "Unable to retrieve valid timestamp from nvd cve downloads page";
-            Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.FINE, msg, ex);
-            throw new DownloadFailedException(msg, ex);
-
-        } catch (InvalidSettingException ex) {
-            Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.FINE, "Invalid setting found when retrieving timestamps", ex);
-            throw new DownloadFailedException("Invalid settings", ex);
-        }
-
-        if (currentlyPublished == null) {
-            throw new DownloadFailedException("Unable to retrieve valid timestamp from nvd cve downloads page");
-        }
-        String dir;
-        try {
-            dir = CveDB.getDataDirectory().getCanonicalPath();
-        } catch (IOException ex) {
-            Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.FINE, "CveDB data directory doesn't exist?", ex);
-            throw new UpdateException("Unable to locate last updated properties file.", ex);
-        }
-
-        final File f = new File(dir);
-        if (f.exists()) {
-            final File cveProp = new File(dir, UPDATE_PROPERTIES_FILE);
-            if (cveProp.exists()) {
-                final Properties prop = new Properties();
-                InputStream is = null;
-                try {
-                    is = new FileInputStream(cveProp);
-                    prop.load(is);
-
-                    boolean deleteAndRecreate = false;
-                    float version;
-
-                    if (prop.getProperty("version") == null) {
-                        deleteAndRecreate = true;
-                    } else {
-                        try {
-                            version = Float.parseFloat(prop.getProperty("version"));
-                            final float currentVersion = Float.parseFloat(CveDB.DB_SCHEMA_VERSION);
-                            if (currentVersion > version) {
-                                deleteAndRecreate = true;
-                            }
-                        } catch (NumberFormatException ex) {
-                            deleteAndRecreate = true;
-                        }
-                    }
-                    if (deleteAndRecreate) {
-                        Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.INFO, "The database version is old. Rebuilding the database.");
-                        is.close();
-                        //this is an old version of the lucene index - just delete it
-                        FileUtils.delete(f);
-
-                        //this importer also updates the CPE index and it is also using an old version
-                        final Index cpeId = new Index();
-                        final File cpeDir = cpeId.getDataDirectory();
-                        FileUtils.delete(cpeDir);
-                        return currentlyPublished;
-                    }
-
-                    final long lastUpdated = Long.parseLong(prop.getProperty(LAST_UPDATED_MODIFIED, "0"));
-                    final Date now = new Date();
-                    final int days = Settings.getInt(Settings.KEYS.CVE_MODIFIED_VALID_FOR_DAYS, 7);
-                    final int start = Settings.getInt(Settings.KEYS.CVE_START_YEAR, 2002);
-                    final int end = Calendar.getInstance().get(Calendar.YEAR);
-                    if (lastUpdated == currentlyPublished.get(MODIFIED).timestamp) {
-                        currentlyPublished.clear(); //we don't need to update anything.
-                    } else if (withinRange(lastUpdated, now.getTime(), days)) {
-                        currentlyPublished.get(MODIFIED).setNeedsUpdate(true);
-                        for (int i = start; i <= end; i++) {
-                            currentlyPublished.get(String.valueOf(i)).setNeedsUpdate(false);
-                        }
-                    } else { //we figure out which of the several XML files need to be downloaded.
-                        currentlyPublished.get(MODIFIED).setNeedsUpdate(false);
-                        for (int i = start; i <= end; i++) {
-                            final NvdCveUrl cve = currentlyPublished.get(String.valueOf(i));
-                            long currentTimestamp = 0;
-                            try {
-                                currentTimestamp = Long.parseLong(prop.getProperty(LAST_UPDATED_BASE + String.valueOf(i), "0"));
-                            } catch (NumberFormatException ex) {
-                                final String msg = String.format("Error parsing '%s' '%s' from nvdcve.lastupdated",
-                                        LAST_UPDATED_BASE, String.valueOf(i));
-                                Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.FINE, msg, ex);
-                            }
-                            if (currentTimestamp == cve.getTimestamp()) {
-                                cve.setNeedsUpdate(false); //they default to true.
-                            }
-                        }
-                    }
-                } catch (FileNotFoundException ex) {
-                    Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.FINEST, null, ex);
-                } catch (IOException ex) {
-                    Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.FINEST, null, ex);
-                } catch (NumberFormatException ex) {
-                    Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.FINEST, null, ex);
-                } finally {
-                    if (is != null) {
-                        try {
-                            is.close();
-                        } catch (IOException ex) {
-                            Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.FINEST, null, ex);
-                        }
-                    }
-                }
-            }
-        }
-        return currentlyPublished;
-    }
-
-    /**
-     * Determines if the epoch date is within the range specified of the
-     * compareTo epoch time. This takes the (compareTo-date)/1000/60/60/24 to
-     * get the number of days. If the calculated days is less then the range the
-     * date is considered valid.
-     *
-     * @param date the date to be checked.
-     * @param compareTo the date to compare to.
-     * @param range the range in days to be considered valid.
-     * @return whether or not the date is within the range.
-     */
-    private boolean withinRange(long date, long compareTo, int range) {
-        final double differenceInDays = (compareTo - date) / 1000.0 / 60.0 / 60.0 / 24.0;
-        return differenceInDays < range;
-    }
-
-    /**
-     * Retrieves the timestamps from the NVD CVE meta data file.
-     *
-     * @return the timestamp from the currently published nvdcve downloads page
-     * @throws MalformedURLException thrown if the URL for the NVD CCE Meta data
-     * is incorrect.
-     * @throws DownloadFailedException thrown if there is an error downloading
-     * the nvd cve meta data file
-     * @throws InvalidDataException thrown if there is an exception parsing the
-     * timestamps
-     * @throws InvalidSettingException thrown if the settings are invalid
-     */
-    protected Map<String, NvdCveUrl> retrieveCurrentTimestampsFromWeb()
-            throws MalformedURLException, DownloadFailedException, InvalidDataException, InvalidSettingException {
-
-        final Map<String, NvdCveUrl> map = new HashMap<String, NvdCveUrl>();
-        String retrieveUrl = Settings.getString(Settings.KEYS.CVE_MODIFIED_20_URL);
-
-        NvdCveUrl item = new NvdCveUrl();
-        item.setNeedsUpdate(false); //the others default to true, to make life easier later this should default to false.
-        item.setId(MODIFIED);
-        item.setUrl(retrieveUrl);
-        item.setOldSchemaVersionUrl(Settings.getString(Settings.KEYS.CVE_MODIFIED_12_URL));
-
-        item.timestamp = Downloader.getLastModified(new URL(retrieveUrl));
-        map.put(MODIFIED, item);
-
-        final int start = Settings.getInt(Settings.KEYS.CVE_START_YEAR);
-        final int end = Calendar.getInstance().get(Calendar.YEAR);
-        final String baseUrl20 = Settings.getString(Settings.KEYS.CVE_SCHEMA_2_0);
-        final String baseUrl12 = Settings.getString(Settings.KEYS.CVE_SCHEMA_1_2);
-        for (int i = start; i <= end; i++) {
-            retrieveUrl = String.format(baseUrl20, i);
-            item = new NvdCveUrl();
-            item.setId(Integer.toString(i));
-            item.setUrl(retrieveUrl);
-            item.setOldSchemaVersionUrl(String.format(baseUrl12, i));
-            item.setTimestamp(Downloader.getLastModified(new URL(retrieveUrl)));
-            map.put(item.id, item);
-        }
-        return map;
-    }
-
-    /**
-     * Method to double check that the "modified" nvdcve file is listed and has
-     * a timestamp in the last updated properties file.
-     *
-     * @param update a set of updated NvdCveUrl objects
-     */
-    private void ensureModifiedIsInLastUpdatedProperties(Map<String, NvdCveUrl> update) {
-        try {
-            writeLastUpdatedPropertyFile(update.get(MODIFIED));
-        } catch (UpdateException ex) {
-            Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.FINE, null, ex);
-        }
-    }
-
-    /**
-     * A pojo that contains the Url and timestamp of the current NvdCve XML
-     * files.
-     */
-    protected static class NvdCveUrl {
-
-        /**
-         * an id.
-         */
-        private String id;
-
-        /**
-         * Get the value of id.
-         *
-         * @return the value of id
-         */
-        public String getId() {
-            return id;
-        }
-
-        /**
-         * Set the value of id.
-         *
-         * @param id new value of id
-         */
-        public void setId(String id) {
-            this.id = id;
-        }
-        /**
-         * a url.
-         */
-        private String url;
-
-        /**
-         * Get the value of url.
-         *
-         * @return the value of url
-         */
-        public String getUrl() {
-            return url;
-        }
-
-        /**
-         * Set the value of url.
-         *
-         * @param url new value of url
-         */
-        public void setUrl(String url) {
-            this.url = url;
-        }
-        /**
-         * The 1.2 schema URL.
-         */
-        private String oldSchemaVersionUrl;
-
-        /**
-         * Get the value of oldSchemaVersionUrl.
-         *
-         * @return the value of oldSchemaVersionUrl
-         */
-        public String getOldSchemaVersionUrl() {
-            return oldSchemaVersionUrl;
-        }
-
-        /**
-         * Set the value of oldSchemaVersionUrl.
-         *
-         * @param oldSchemaVersionUrl new value of oldSchemaVersionUrl
-         */
-        public void setOldSchemaVersionUrl(String oldSchemaVersionUrl) {
-            this.oldSchemaVersionUrl = oldSchemaVersionUrl;
-        }
-        /**
-         * a timestamp - epoch time.
-         */
-        private long timestamp;
-
-        /**
-         * Get the value of timestamp - epoch time.
-         *
-         * @return the value of timestamp - epoch time
-         */
-        public long getTimestamp() {
-            return timestamp;
-        }
-
-        /**
-         * Set the value of timestamp - epoch time.
-         *
-         * @param timestamp new value of timestamp - epoch time
-         */
-        public void setTimestamp(long timestamp) {
-            this.timestamp = timestamp;
-        }
-        /**
-         * indicates whether or not this item should be updated.
-         */
-        private boolean needsUpdate = true;
-
-        /**
-         * Get the value of needsUpdate.
-         *
-         * @return the value of needsUpdate
-         */
-        public boolean getNeedsUpdate() {
-            return needsUpdate;
-        }
-
-        /**
-         * Set the value of needsUpdate.
-         *
-         * @param needsUpdate new value of needsUpdate
-         */
-        public void setNeedsUpdate(boolean needsUpdate) {
-            this.needsUpdate = needsUpdate;
-        }
-    }
-    //</editor-fold>
-}

dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/CallableDownloadTask.java

@@ -0,0 +1,179 @@
+/*
+ * This file is part of dependency-check-core.
+ *
+ * Dependency-check-core is free software: you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation, either version 3 of the License, or (at your option) any
+ * later version.
+ *
+ * Dependency-check-core is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+ * details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * dependency-check-core. If not, see http://www.gnu.org/licenses/.
+ *
+ * Copyright (c) 2013 Jeremy Long. All Rights Reserved.
+ */
+package org.owasp.dependencycheck.data.update;
+
+import java.io.File;
+import java.net.URL;
+import java.util.concurrent.Callable;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+import org.owasp.dependencycheck.utils.DownloadFailedException;
+import org.owasp.dependencycheck.utils.Downloader;
+
+/**
+ * A callable object to download two files.
+ *
+ * @author Jeremy Long (jeremy.long@owasp.org)
+ */
+public class CallableDownloadTask implements Callable<CallableDownloadTask> {
+
+    /**
+     * Simple constructor for the callable download task.
+     *
+     * @param nvdCveInfo the nvd cve info
+     * @param first the first file
+     * @param second the second file
+     */
+    public CallableDownloadTask(NvdCveInfo nvdCveInfo, File first, File second) {
+        this.nvdCveInfo = nvdCveInfo;
+        this.first = first;
+        this.second = second;
+    }
+    /**
+     * The NVD CVE Meta Data.
+     */
+    private NvdCveInfo nvdCveInfo;
+
+    /**
+     * Get the value of nvdCveInfo.
+     *
+     * @return the value of nvdCveInfo
+     */
+    public NvdCveInfo getNvdCveInfo() {
+        return nvdCveInfo;
+    }
+
+    /**
+     * Set the value of nvdCveInfo.
+     *
+     * @param nvdCveInfo new value of nvdCveInfo
+     */
+    public void setNvdCveInfo(NvdCveInfo nvdCveInfo) {
+        this.nvdCveInfo = nvdCveInfo;
+    }
+    /**
+     * a file.
+     */
+    private File first;
+
+    /**
+     * Get the value of first.
+     *
+     * @return the value of first
+     */
+    public File getFirst() {
+        return first;
+    }
+
+    /**
+     * Set the value of first.
+     *
+     * @param first new value of first
+     */
+    public void setFirst(File first) {
+        this.first = first;
+    }
+    /**
+     * a file.
+     */
+    private File second;
+
+    /**
+     * Get the value of second.
+     *
+     * @return the value of second
+     */
+    public File getSecond() {
+        return second;
+    }
+
+    /**
+     * Set the value of second.
+     *
+     * @param second new value of second
+     */
+    public void setSecond(File second) {
+        this.second = second;
+    }
+    /**
+     * A placeholder for an exception.
+     */
+    private Exception exception = null;
+
+    /**
+     * Get the value of exception.
+     *
+     * @return the value of exception
+     */
+    public Exception getException() {
+        return exception;
+    }
+
+    /**
+     * returns whether or not an exception occurred during download.
+     *
+     * @return whether or not an exception occurred during download
+     */
+    public boolean hasException() {
+        return exception != null;
+    }
+
+    @Override
+    public CallableDownloadTask call() throws Exception {
+        try {
+            final URL url1 = new URL(nvdCveInfo.getUrl());
+            final URL url2 = new URL(nvdCveInfo.getOldSchemaVersionUrl());
+            String msg = String.format("Download Started for NVD CVE - %s", nvdCveInfo.getId());
+            Logger.getLogger(CallableDownloadTask.class.getName()).log(Level.INFO, msg);
+            Downloader.fetchFile(url1, first);
+            Downloader.fetchFile(url2, second);
+            msg = String.format("Download Complete for NVD CVE - %s", nvdCveInfo.getId());
+            Logger.getLogger(CallableDownloadTask.class.getName()).log(Level.INFO, msg);
+        } catch (DownloadFailedException ex) {
+            this.exception = ex;
+        }
+        return this;
+    }
+
+    /**
+     * Attempts to delete the files that were downloaded.
+     */
+    public void cleanup() {
+        boolean deleted = false;
+        try {
+            if (first != null && first.exists()) {
+                deleted = first.delete();
+            }
+        } finally {
+            if (first != null && (first.exists() || !deleted)) {
+                first.deleteOnExit();
+            }
+        }
+        try {
+            deleted = false;
+            if (second != null && second.exists()) {
+                deleted = second.delete();
+            }
+        } finally {
+            if (second != null && (second.exists() || !deleted)) {
+                second.deleteOnExit();
+            }
+        }
+    }
+}

dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/DataStoreMetaInfo.java

@@ -0,0 +1,241 @@
+/*
+ * This file is part of dependency-check-core.
+ *
+ * Dependency-check-core is free software: you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation, either version 3 of the License, or (at your option) any
+ * later version.
+ *
+ * Dependency-check-core is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+ * details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * dependency-check-core. If not, see http://www.gnu.org/licenses/.
+ *
+ * Copyright (c) 2013 Jeremy Long. All Rights Reserved.
+ */
+package org.owasp.dependencycheck.data.update;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.io.OutputStreamWriter;
+import java.util.Properties;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+import org.owasp.dependencycheck.data.UpdateException;
+import org.owasp.dependencycheck.data.nvdcve.CveDB;
+import org.owasp.dependencycheck.utils.Settings;
+
+/**
+ *
+ * @author Jeremy Long (jeremy.long@owasp.org)
+ */
+public class DataStoreMetaInfo {
+
+    /**
+     * Batch key word, used as key to store information about batch mode.
+     */
+    public static final String BATCH = "batch";
+    /**
+     * Modified key word, used as a key to store information about the modified
+     * file (i.e. the containing the last 8 days of updates)..
+     */
+    public static final String MODIFIED = "modified";
+    /**
+     * The name of the properties file containing the timestamp of the last
+     * update.
+     */
+    private static final String UPDATE_PROPERTIES_FILE = "data.properties";
+    /**
+     * The properties file key for the last updated field - used to store the
+     * last updated time of the Modified NVD CVE xml file.
+     */
+    public static final String LAST_UPDATED = "lastupdated.modified";
+    /**
+     * Stores the last updated time for each of the NVD CVE files. These
+     * timestamps should be updated if we process the modified file within 7
+     * days of the last update.
+     */
+    public static final String LAST_UPDATED_BASE = "lastupdated.";
+    /**
+     * A collection of properties about the data.
+     */
+    private Properties properties = new Properties();
+    /**
+     * Indicates whether or not the updates are using a batch update mode or
+     * not.
+     */
+    private boolean batchUpdateMode;
+
+    /**
+     * Get the value of batchUpdateMode.
+     *
+     * @return the value of batchUpdateMode
+     */
+    protected boolean isBatchUpdateMode() {
+        return batchUpdateMode;
+    }
+
+    /**
+     * Set the value of batchUpdateMode.
+     *
+     * @param batchUpdateMode new value of batchUpdateMode
+     */
+    protected void setBatchUpdateMode(boolean batchUpdateMode) {
+        this.batchUpdateMode = batchUpdateMode;
+    }
+
+    /**
+     * Constructs a new data properties object.
+     */
+    public DataStoreMetaInfo() {
+        batchUpdateMode = !Settings.getString(Settings.KEYS.BATCH_UPDATE_URL, "").isEmpty();
+        loadProperties();
+    }
+
+    /**
+     * Loads the data's meta properties.
+     */
+    private void loadProperties() {
+        final File file = getPropertiesFile();
+        if (file.exists()) {
+            InputStream is = null;
+            try {
+                is = new FileInputStream(file);
+            } catch (FileNotFoundException ignore) {
+                //we will never get here as we check for existence above.
+                Logger.getLogger(DataStoreMetaInfo.class.getName()).log(Level.FINEST, null, ignore);
+            }
+            try {
+                properties.load(is);
+            } catch (IOException ex) {
+                final String msg = String.format("Unable to load properties file '%s'", file.getPath());
+                Logger.getLogger(DataStoreMetaInfo.class.getName()).log(Level.WARNING, msg);
+                Logger.getLogger(DataStoreMetaInfo.class.getName()).log(Level.FINE, null, ex);
+            } finally {
+                if (is != null) {
+                    try {
+                        is.close();
+                    } catch (IOException ex) {
+                        final String msg = String.format("Unable to close properties file '%s'", file.getPath());
+                        Logger.getLogger(DataStoreMetaInfo.class.getName()).log(Level.WARNING, msg);
+                        Logger.getLogger(DataStoreMetaInfo.class.getName()).log(Level.FINE, null, ex);
+                    }
+                }
+            }
+        }
+    }
+
+    /**
+     * Returns whether or not any properties are set.
+     *
+     * @return whether or not any properties are set
+     */
+    public boolean isEmpty() {
+        return properties.isEmpty();
+    }
+
+    /**
+     * Writes a properties file containing the last updated date to the
+     * VULNERABLE_CPE directory.
+     *
+     * @param updatedValue the updated nvdcve entry
+     * @throws UpdateException is thrown if there is an update exception
+     */
+    public void save(NvdCveInfo updatedValue) throws UpdateException {
+        if (updatedValue == null) {
+            return;
+        }
+        final File cveProp = getPropertiesFile();
+        final Properties prop = new Properties();
+        if (cveProp.exists()) {
+            FileInputStream in = null;
+            try {
+                in = new FileInputStream(cveProp);
+                prop.load(in);
+            } catch (Exception ignoreMe) {
+                Logger.getLogger(DataStoreMetaInfo.class.getName()).log(Level.FINEST, null, ignoreMe);
+            } finally {
+                if (in != null) {
+                    try {
+                        in.close();
+                    } catch (Exception ignoreMeToo) {
+                        Logger.getLogger(DataStoreMetaInfo.class.getName()).log(Level.FINEST, null, ignoreMeToo);
+                    }
+                }
+            }
+        }
+        prop.put("version", CveDB.DB_SCHEMA_VERSION);
+        prop.put(LAST_UPDATED_BASE + updatedValue.getId(), String.valueOf(updatedValue.getTimestamp()));
+
+        OutputStream os = null;
+        OutputStreamWriter out = null;
+        try {
+            os = new FileOutputStream(cveProp);
+            out = new OutputStreamWriter(os, "UTF-8");
+            prop.store(out, "Meta data about data and data sources used by dependency-check");
+        } catch (FileNotFoundException ex) {
+            Logger.getLogger(DataStoreMetaInfo.class.getName()).log(Level.FINE, null, ex);
+            throw new UpdateException("Unable to find last updated properties file.", ex);
+        } catch (IOException ex) {
+            Logger.getLogger(DataStoreMetaInfo.class.getName()).log(Level.FINE, null, ex);
+            throw new UpdateException("Unable to update last updated properties file.", ex);
+        } finally {
+            if (out != null) {
+                try {
+                    out.close();
+                } catch (IOException ex) {
+                    Logger.getLogger(DataStoreMetaInfo.class.getName()).log(Level.FINEST, null, ex);
+                }
+            }
+            if (os != null) {
+                try {
+                    os.close();
+                } catch (IOException ex) {
+                    Logger.getLogger(DataStoreMetaInfo.class.getName()).log(Level.FINEST, null, ex);
+                }
+            }
+        }
+    }
+
+    /**
+     * Returns the property value for the given key. If the key is not contained
+     * in the underlying properties null is returned.
+     *
+     * @param key the property key
+     * @return the value of the property
+     */
+    public String getProperty(String key) {
+        return properties.getProperty(key);
+    }
+
+    /**
+     * Returns the property value for the given key. If the key is not contained
+     * in the underlying properties the default value is returned.
+     *
+     * @param key the property key
+     * @param defaultValue the default value
+     * @return the value of the property
+     */
+    public String getProperty(String key, String defaultValue) {
+        return properties.getProperty(key, defaultValue);
+    }
+
+    /**
+     * Retrieves the properties file.
+     *
+     * @return the properties file
+     */
+    public static File getPropertiesFile() {
+        final File dataDirectory = Settings.getDataFile(Settings.KEYS.DATA_DIRECTORY);
+        final File file = new File(dataDirectory, UPDATE_PROPERTIES_FILE);
+        return file;
+    }
+}

dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/DatabaseUpdater.java

@@ -0,0 +1,87 @@
+/*
+ * This file is part of dependency-check-core.
+ *
+ * Dependency-check-core is free software: you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation, either version 3 of the License, or (at your option) any
+ * later version.
+ *
+ * Dependency-check-core is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+ * details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * dependency-check-core. If not, see http://www.gnu.org/licenses/.
+ *
+ * Copyright (c) 2012 Jeremy Long. All Rights Reserved.
+ */
+package org.owasp.dependencycheck.data.update;
+
+import java.io.File;
+import java.io.IOException;
+import org.owasp.dependencycheck.data.CachedWebDataSource;
+import java.net.MalformedURLException;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+import org.owasp.dependencycheck.data.UpdateException;
+import org.owasp.dependencycheck.utils.DownloadFailedException;
+import org.owasp.dependencycheck.utils.FileUtils;
+import org.owasp.dependencycheck.utils.Settings;
+
+/**
+ * Class responsible for updating the CPE and NVDCVE data stores.
+ *
+ * @author Jeremy Long (jeremy.long@owasp.org)
+ */
+public class DatabaseUpdater implements CachedWebDataSource {
+
+    /**
+     * <p>Downloads the latest NVD CVE XML file from the web and imports it into
+     * the current CVE Database.</p>
+     *
+     * @throws UpdateException is thrown if there is an error updating the
+     * database
+     */
+    @Override
+    public void update() throws UpdateException {
+        try {
+            final StandardUpdate task = new StandardUpdate();
+            if (task.isUpdateNeeded()) {
+                if (task.shouldDeleteAndRecreate()) {
+                    try {
+                        deleteExistingData();
+                    } catch (IOException ex) {
+                        Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.WARNING, "Unable to delete the existing data directory");
+                        Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.FINE, null, ex);
+                    }
+                }
+                task.update();
+            }
+        } catch (MalformedURLException ex) {
+            Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.WARNING,
+                    "NVD CVE properties files contain an invalid URL, unable to update the data to use the most current data.");
+            Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.FINE, null, ex);
+        } catch (DownloadFailedException ex) {
+            Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.WARNING,
+                    "Unable to download the NVD CVE data, unable to update the data to use the most current data.");
+            Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.FINE, null, ex);
+        }
+    }
+
+    /**
+     * Deletes the existing data directories.
+     *
+     * @throws IOException thrown if the directory cannot be deleted
+     */
+    protected void deleteExistingData() throws IOException {
+        File data = Settings.getDataFile(Settings.KEYS.CVE_DATA_DIRECTORY);
+        if (data.exists()) {
+            FileUtils.delete(data);
+        }
+        data = DataStoreMetaInfo.getPropertiesFile();
+        if (data.exists()) {
+            FileUtils.delete(data);
+        }
+    }
+}

dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/NvdCveInfo.java

@@ -0,0 +1,138 @@
+/*
+ * This file is part of dependency-check-core.
+ *
+ * Dependency-check-core is free software: you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation, either version 3 of the License, or (at your option) any
+ * later version.
+ *
+ * Dependency-check-core is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+ * details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * dependency-check-core. If not, see http://www.gnu.org/licenses/.
+ *
+ * Copyright (c) 2013 Jeremy Long. All Rights Reserved.
+ */
+package org.owasp.dependencycheck.data.update;
+
+/**
+ * A pojo that contains the Url and timestamp of the current NvdCve XML files.
+ *
+ * @author Jeremy Long (jeremy.long@owasp.org)
+ */
+public class NvdCveInfo {
+
+    /**
+     * an id.
+     */
+    private String id;
+
+    /**
+     * Get the value of id.
+     *
+     * @return the value of id
+     */
+    public String getId() {
+        return id;
+    }
+
+    /**
+     * Set the value of id.
+     *
+     * @param id new value of id
+     */
+    public void setId(String id) {
+        this.id = id;
+    }
+    /**
+     * a url.
+     */
+    private String url;
+
+    /**
+     * Get the value of url.
+     *
+     * @return the value of url
+     */
+    public String getUrl() {
+        return url;
+    }
+
+    /**
+     * Set the value of url.
+     *
+     * @param url new value of url
+     */
+    public void setUrl(String url) {
+        this.url = url;
+    }
+    /**
+     * The 1.2 schema URL.
+     */
+    private String oldSchemaVersionUrl;
+
+    /**
+     * Get the value of oldSchemaVersionUrl.
+     *
+     * @return the value of oldSchemaVersionUrl
+     */
+    public String getOldSchemaVersionUrl() {
+        return oldSchemaVersionUrl;
+    }
+
+    /**
+     * Set the value of oldSchemaVersionUrl.
+     *
+     * @param oldSchemaVersionUrl new value of oldSchemaVersionUrl
+     */
+    public void setOldSchemaVersionUrl(String oldSchemaVersionUrl) {
+        this.oldSchemaVersionUrl = oldSchemaVersionUrl;
+    }
+    /**
+     * a timestamp - epoch time.
+     */
+    private long timestamp;
+
+    /**
+     * Get the value of timestamp - epoch time.
+     *
+     * @return the value of timestamp - epoch time
+     */
+    public long getTimestamp() {
+        return timestamp;
+    }
+
+    /**
+     * Set the value of timestamp - epoch time.
+     *
+     * @param timestamp new value of timestamp - epoch time
+     */
+    public void setTimestamp(long timestamp) {
+        this.timestamp = timestamp;
+    }
+    /**
+     * indicates whether or not this item should be updated.
+     */
+    private boolean needsUpdate = true;
+
+    /**
+     * Get the value of needsUpdate.
+     *
+     * @return the value of needsUpdate
+     */
+    public boolean getNeedsUpdate() {
+        return needsUpdate;
+    }
+
+    /**
+     * Set the value of needsUpdate.
+     *
+     * @param needsUpdate new value of needsUpdate
+     */
+    public void setNeedsUpdate(boolean needsUpdate) {
+        this.needsUpdate = needsUpdate;
+    }
+}

dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/ProcessTask.java

@@ -0,0 +1,148 @@
+/*
+ * This file is part of dependency-check-core.
+ *
+ * Dependency-check-core is free software: you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation, either version 3 of the License, or (at your option) any
+ * later version.
+ *
+ * Dependency-check-core is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+ * details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * dependency-check-core. If not, see http://www.gnu.org/licenses/.
+ *
+ * Copyright (c) 2013 Jeremy Long. All Rights Reserved.
+ */
+package org.owasp.dependencycheck.data.update;
+
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.sql.SQLException;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.Callable;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+import javax.xml.parsers.ParserConfigurationException;
+import javax.xml.parsers.SAXParser;
+import javax.xml.parsers.SAXParserFactory;
+import org.owasp.dependencycheck.data.UpdateException;
+import org.owasp.dependencycheck.data.nvdcve.CveDB;
+import org.owasp.dependencycheck.data.nvdcve.DatabaseException;
+import org.owasp.dependencycheck.data.nvdcve.NvdCve12Handler;
+import org.owasp.dependencycheck.data.nvdcve.NvdCve20Handler;
+import org.owasp.dependencycheck.dependency.VulnerableSoftware;
+import org.xml.sax.SAXException;
+
+/**
+ * A callable task that will process a given set of NVD CVE xml files and update
+ * the Cve Database accordingly.
+ *
+ * @author Jeremy Long (jeremy.long@owasp.org)
+ */
+public class ProcessTask implements Callable<ProcessTask> {
+
+    /**
+     * A field to store any update exceptions that occur during the "call".
+     */
+    private UpdateException exception = null;
+
+    /**
+     * Get the value of exception.
+     *
+     * @return the value of exception
+     */
+    public UpdateException getException() {
+        return exception;
+    }
+
+    /**
+     * Set the value of exception.
+     *
+     * @param exception new value of exception
+     */
+    public void setException(UpdateException exception) {
+        this.exception = exception;
+    }
+    private final CveDB cveDB;
+    private final CallableDownloadTask filePair;
+    private final DataStoreMetaInfo properties;
+
+    public ProcessTask(final CveDB cveDB, final DataStoreMetaInfo properties, final CallableDownloadTask filePair) {
+        this.cveDB = cveDB;
+        this.filePair = filePair;
+        this.properties = properties;
+    }
+
+    @Override
+    public ProcessTask call() throws Exception {
+        try {
+            processFiles();
+        } catch (UpdateException ex) {
+            this.exception = ex;
+        }
+        return this;
+    }
+
+    /**
+     * Imports the NVD CVE XML File into the Lucene Index.
+     *
+     * @param file the file containing the NVD CVE XML
+     * @param oldVersion contains the file containing the NVD CVE XML 1.2
+     * @throws ParserConfigurationException is thrown if there is a parser
+     * configuration exception
+     * @throws SAXException is thrown if there is a SAXException
+     * @throws IOException is thrown if there is a IO Exception
+     * @throws SQLException is thrown if there is a SQL exception
+     * @throws DatabaseException is thrown if there is a database exception
+     * @throws ClassNotFoundException thrown if the h2 database driver cannot be
+     * loaded
+     */
+    protected void importXML(File file, File oldVersion) throws ParserConfigurationException,
+            SAXException, IOException, SQLException, DatabaseException, ClassNotFoundException {
+
+        final SAXParserFactory factory = SAXParserFactory.newInstance();
+        final SAXParser saxParser = factory.newSAXParser();
+
+        final NvdCve12Handler cve12Handler = new NvdCve12Handler();
+        saxParser.parse(oldVersion, cve12Handler);
+        final Map<String, List<VulnerableSoftware>> prevVersionVulnMap = cve12Handler.getVulnerabilities();
+
+        final NvdCve20Handler cve20Handler = new NvdCve20Handler();
+        cve20Handler.setCveDB(cveDB);
+        cve20Handler.setPrevVersionVulnMap(prevVersionVulnMap);
+        saxParser.parse(file, cve20Handler);
+    }
+
+    private void processFiles() throws UpdateException {
+        String msg = String.format("Processing Started for NVD CVE - %s", filePair.getNvdCveInfo().getId());
+        Logger.getLogger(StandardUpdate.class.getName()).log(Level.INFO, msg);
+        try {
+            importXML(filePair.getFirst(), filePair.getSecond());
+            cveDB.commit();
+            properties.save(filePair.getNvdCveInfo());
+        } catch (FileNotFoundException ex) {
+            throw new UpdateException(ex);
+        } catch (ParserConfigurationException ex) {
+            throw new UpdateException(ex);
+        } catch (SAXException ex) {
+            throw new UpdateException(ex);
+        } catch (IOException ex) {
+            throw new UpdateException(ex);
+        } catch (SQLException ex) {
+            throw new UpdateException(ex);
+        } catch (DatabaseException ex) {
+            throw new UpdateException(ex);
+        } catch (ClassNotFoundException ex) {
+            throw new UpdateException(ex);
+        } finally {
+            filePair.cleanup();
+        }
+        msg = String.format("Processing Complete for NVD CVE - %s", filePair.getNvdCveInfo().getId());
+        Logger.getLogger(StandardUpdate.class.getName()).log(Level.INFO, msg);
+    }
+}

dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/StandardUpdate.java

@@ -0,0 +1,557 @@
+/*
+ * This file is part of dependency-check-core.
+ *
+ * Dependency-check-core is free software: you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation, either version 3 of the License, or (at your option) any
+ * later version.
+ *
+ * Dependency-check-core is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+ * details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * dependency-check-core. If not, see http://www.gnu.org/licenses/.
+ *
+ * Copyright (c) 2012 Jeremy Long. All Rights Reserved.
+ */
+package org.owasp.dependencycheck.data.update;
+
+import org.owasp.dependencycheck.data.nvdcve.InvalidDataException;
+import java.io.File;
+import java.io.IOException;
+import java.net.MalformedURLException;
+import java.sql.SQLException;
+import java.util.Calendar;
+import java.util.Date;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.Set;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+import org.owasp.dependencycheck.data.UpdateException;
+import org.owasp.dependencycheck.data.nvdcve.CveDB;
+import org.owasp.dependencycheck.utils.DownloadFailedException;
+import org.owasp.dependencycheck.utils.Settings;
+import org.owasp.dependencycheck.data.nvdcve.DatabaseException;
+import org.owasp.dependencycheck.utils.InvalidSettingException;
+import static org.owasp.dependencycheck.data.update.DataStoreMetaInfo.MODIFIED;
+import org.owasp.dependencycheck.utils.FileUtils;
+
+/**
+ * Class responsible for updating the NVDCVE data store.
+ *
+ * @author Jeremy Long (jeremy.long@owasp.org)
+ */
+public class StandardUpdate {
+
+    /**
+     * The max thread pool size to use when downloading files.
+     */
+    public static final int MAX_THREAD_POOL_SIZE = Settings.getInt(Settings.KEYS.MAX_DOWNLOAD_THREAD_POOL_SIZE, 3);
+    /**
+     * Information about the timestamps and URLs for data that needs to be
+     * updated.
+     */
+    private DataStoreMetaInfo properties;
+    /**
+     * A collection of updateable NVD CVE items.
+     */
+    private Updateable updateable;
+    /**
+     * A flag indicating whether or not the current data store should be
+     * deleted.
+     */
+    private boolean deleteAndRecreate = false;
+    /**
+     * Reference to the Cve Database.
+     */
+    private CveDB cveDB = null;
+
+    /**
+     * Gets whether or not an update is needed.
+     *
+     * @return true or false depending on whether an update is needed
+     */
+    public boolean isUpdateNeeded() {
+        return updateable.isUpdateNeeded();
+    }
+
+    /**
+     * Set the value of deleteAndRecreate.
+     *
+     * @param deleteAndRecreate new value of deleteAndRecreate
+     */
+    protected void setDeleteAndRecreate(boolean deleteAndRecreate) {
+        this.deleteAndRecreate = deleteAndRecreate;
+    }
+
+    /**
+     * Get the value of deleteAndRecreate.
+     *
+     * @return the value of deleteAndRecreate
+     */
+    public boolean shouldDeleteAndRecreate() {
+        return deleteAndRecreate;
+    }
+
+    /**
+     * Constructs a new Standard Update Task.
+     *
+     * @throws MalformedURLException thrown if a configured URL is malformed
+     * @throws DownloadFailedException thrown if a timestamp cannot be checked
+     * on a configured URL
+     * @throws UpdateException thrown if there is an exception generating the
+     * update task
+     */
+    public StandardUpdate() throws MalformedURLException, DownloadFailedException, UpdateException {
+        properties = new DataStoreMetaInfo();
+        updateable = updatesNeeded();
+    }
+
+    /**
+     * <p>Downloads the latest NVD CVE XML file from the web and imports it into
+     * the current CVE Database.</p>
+     *
+     * @throws UpdateException is thrown if there is an error updating the
+     * database
+     */
+    public void update() throws UpdateException {
+        int maxUpdates = 0;
+        try {
+            for (NvdCveInfo cve : updateable) {
+                if (cve.getNeedsUpdate()) {
+                    maxUpdates += 1;
+                }
+            }
+            if (maxUpdates <= 0) {
+                return;
+            }
+            if (maxUpdates > 3) {
+                Logger.getLogger(StandardUpdate.class.getName()).log(Level.INFO,
+                        "NVD CVE requires several updates; this could take a couple of minutes.");
+            }
+            if (maxUpdates > 0) {
+                openDataStores();
+            }
+
+            final int poolSize = (MAX_THREAD_POOL_SIZE > maxUpdates) ? MAX_THREAD_POOL_SIZE : maxUpdates;
+            final ExecutorService downloadExecutor = Executors.newFixedThreadPool(poolSize);
+            final ExecutorService processExecutor = Executors.newSingleThreadExecutor();
+            final Set<Future<CallableDownloadTask>> downloadFutures = new HashSet<Future<CallableDownloadTask>>(maxUpdates);
+            final Set<Future<ProcessTask>> processFutures = new HashSet<Future<ProcessTask>>(maxUpdates);
+            int ctr = 0;
+            for (NvdCveInfo cve : updateable) {
+                if (cve.getNeedsUpdate()) {
+                    ctr += 1;
+                    final File file1;
+                    final File file2;
+                    try {
+                        file1 = File.createTempFile("cve" + cve.getId() + "_", ".xml");
+                        file2 = File.createTempFile("cve_1_2_" + cve.getId() + "_", ".xml");
+                    } catch (IOException ex) {
+                        throw new UpdateException(ex);
+                    }
+                    final CallableDownloadTask call = new CallableDownloadTask(cve, file1, file2);
+                    downloadFutures.add(downloadExecutor.submit(call));
+
+                    boolean waitForFuture = ctr % 2 == 0;
+
+                    final Iterator<Future<CallableDownloadTask>> itr = downloadFutures.iterator();
+                    while (itr.hasNext()) {
+                        final Future<CallableDownloadTask> future = itr.next();
+                        if (waitForFuture) { //only allow two NVD/CVE files to be downloaded at a time
+                            spinWaitForFuture(future);
+                        }
+                        if (future.isDone()) { //if we find something complete, add it to the process queue
+                            try {
+                                final CallableDownloadTask filePair = future.get();
+                                itr.remove();
+                                final ProcessTask task = new ProcessTask(cveDB, properties, filePair);
+                                processFutures.add(processExecutor.submit(task));
+                            } catch (InterruptedException ex) {
+                                downloadExecutor.shutdownNow();
+                                Logger.getLogger(StandardUpdate.class.getName()).log(Level.FINE, "Thread was interupted", ex);
+                                throw new UpdateException(ex);
+                            } catch (ExecutionException ex) {
+                                downloadExecutor.shutdownNow();
+                                Logger.getLogger(StandardUpdate.class.getName()).log(Level.SEVERE, null, ex);
+                                throw new UpdateException(ex);
+                            }
+                        }
+                    }
+
+                }
+            }
+
+            try {
+                final Iterator<Future<CallableDownloadTask>> itr = downloadFutures.iterator();
+                while (itr.hasNext()) {
+                    final Future<CallableDownloadTask> future = itr.next();
+                    final CallableDownloadTask filePair = future.get();
+                    final ProcessTask task = new ProcessTask(cveDB, properties, filePair);
+                    processFutures.add(processExecutor.submit(task));
+                }
+            } catch (InterruptedException ex) {
+                downloadExecutor.shutdownNow();
+                Logger.getLogger(StandardUpdate.class.getName()).log(Level.FINE, "Thread was interupted during download", ex);
+                throw new UpdateException(ex);
+            } catch (ExecutionException ex) {
+                downloadExecutor.shutdownNow();
+                Logger.getLogger(StandardUpdate.class.getName()).log(Level.FINE, "Execution Exception during download", ex);
+                throw new UpdateException(ex);
+            } finally {
+                downloadExecutor.shutdown();
+            }
+
+            for (Future<ProcessTask> future : processFutures) {
+                try {
+                    final ProcessTask task = future.get();
+                    if (task.getException() != null) {
+                        throw task.getException();
+                    }
+                } catch (InterruptedException ex) {
+                    processExecutor.shutdownNow();
+                    Logger.getLogger(StandardUpdate.class.getName()).log(Level.FINE, "Thread was interupted during processing", ex);
+                    throw new UpdateException(ex);
+                } catch (ExecutionException ex) {
+                    processExecutor.shutdownNow();
+                    Logger.getLogger(StandardUpdate.class.getName()).log(Level.FINE, "Execution Exception during process", ex);
+                    throw new UpdateException(ex);
+                } finally {
+                    processExecutor.shutdown();
+                }
+            }
+
+            if (maxUpdates >= 1) { //ensure the modified file date gets written
+                properties.save(updateable.get(MODIFIED));
+                cveDB.cleanupDatabase();
+            }
+        } finally {
+            closeDataStores();
+        }
+    }
+
+    //<editor-fold defaultstate="collapsed" desc="OLD version of update() - not multithreaded">
+    /*
+     * TODO - remove this
+     public void update() throws UpdateException {
+     try {
+     int maxUpdates = 0;
+     for (NvdCveInfo cve : getUpdateable()) {
+     if (cve.getNeedsUpdate()) {
+     maxUpdates += 1;
+     }
+     }
+     if (maxUpdates > 3) {
+     Logger.getLogger(StandardUpdate.class.getName()).log(Level.INFO,
+     "NVD CVE requires several updates; this could take a couple of minutes.");
+     }
+     if (maxUpdates > 0) {
+     openDataStores();
+     }
+
+     int count = 0;
+     for (NvdCveInfo cve : getUpdateable()) {
+     if (cve.getNeedsUpdate()) {
+     count += 1;
+     Logger.getLogger(StandardUpdate.class.getName()).log(Level.INFO,
+     "Updating NVD CVE ({0} of {1})", new Object[]{count, maxUpdates});
+     URL url = new URL(cve.getUrl());
+     File outputPath = null;
+     File outputPath12 = null;
+     try {
+     Logger.getLogger(StandardUpdate.class.getName()).log(Level.INFO,
+     "Downloading {0}", cve.getUrl());
+     outputPath = File.createTempFile("cve" + cve.getId() + "_", ".xml");
+     Downloader.fetchFile(url, outputPath);
+
+     url = new URL(cve.getOldSchemaVersionUrl());
+     outputPath12 = File.createTempFile("cve_1_2_" + cve.getId() + "_", ".xml");
+     Downloader.fetchFile(url, outputPath12);
+
+     Logger.getLogger(StandardUpdate.class.getName()).log(Level.INFO,
+     "Processing {0}", cve.getUrl());
+
+     importXML(outputPath, outputPath12);
+
+     getCveDB().commit();
+     getProperties().save(cve);
+
+     Logger.getLogger(StandardUpdate.class.getName()).log(Level.INFO,
+     "Completed update {0} of {1}", new Object[]{count, maxUpdates});
+     } catch (FileNotFoundException ex) {
+     throw new UpdateException(ex);
+     } catch (ParserConfigurationException ex) {
+     throw new UpdateException(ex);
+     } catch (SAXException ex) {
+     throw new UpdateException(ex);
+     } catch (IOException ex) {
+     throw new UpdateException(ex);
+     } catch (SQLException ex) {
+     throw new UpdateException(ex);
+     } catch (DatabaseException ex) {
+     throw new UpdateException(ex);
+     } catch (ClassNotFoundException ex) {
+     throw new UpdateException(ex);
+     } finally {
+     boolean deleted = false;
+     try {
+     if (outputPath != null && outputPath.exists()) {
+     deleted = outputPath.delete();
+     }
+     } finally {
+     if (outputPath != null && (outputPath.exists() || !deleted)) {
+     outputPath.deleteOnExit();
+     }
+     }
+     try {
+     deleted = false;
+     if (outputPath12 != null && outputPath12.exists()) {
+     deleted = outputPath12.delete();
+     }
+     } finally {
+     if (outputPath12 != null && (outputPath12.exists() || !deleted)) {
+     outputPath12.deleteOnExit();
+     }
+     }
+     }
+     }
+     }
+     if (maxUpdates >= 1) { //ensure the modified file date gets written
+     getProperties().save(getUpdateable().get(MODIFIED));
+     getCveDB().cleanupDatabase();
+     }
+     } catch (MalformedURLException ex) {
+     throw new UpdateException(ex);
+     } finally {
+     closeDataStores();
+     }
+     }
+     */
+    //</editor-fold>
+    /**
+     * Determines if the index needs to be updated. This is done by fetching the
+     * NVD CVE meta data and checking the last update date. If the data needs to
+     * be refreshed this method will return the NvdCveUrl for the files that
+     * need to be updated.
+     *
+     * @return the collection of files that need to be updated
+     * @throws MalformedURLException is thrown if the URL for the NVD CVE Meta
+     * data is incorrect
+     * @throws DownloadFailedException is thrown if there is an error.
+     * downloading the NVD CVE download data file
+     * @throws UpdateException Is thrown if there is an issue with the last
+     * updated properties file
+     */
+    protected final Updateable updatesNeeded() throws MalformedURLException, DownloadFailedException, UpdateException {
+        Updateable updates = null;
+        try {
+            updates = retrieveCurrentTimestampsFromWeb();
+        } catch (InvalidDataException ex) {
+            final String msg = "Unable to retrieve valid timestamp from nvd cve downloads page";
+            Logger
+                    .getLogger(StandardUpdate.class
+                    .getName()).log(Level.FINE, msg, ex);
+            throw new DownloadFailedException(msg, ex);
+        } catch (InvalidSettingException ex) {
+            Logger.getLogger(StandardUpdate.class
+                    .getName()).log(Level.FINE, "Invalid setting found when retrieving timestamps", ex);
+            throw new DownloadFailedException(
+                    "Invalid settings", ex);
+        }
+
+        if (updates == null) {
+            throw new DownloadFailedException("Unable to retrieve the timestamps of the currently published NVD CVE data");
+        }
+        if (!properties.isEmpty()) {
+            try {
+                float version;
+
+                if (properties.getProperty("version") == null) {
+                    deleteAndRecreate = true;
+                } else {
+                    try {
+                        version = Float.parseFloat(properties.getProperty("version"));
+                        final float currentVersion = Float.parseFloat(CveDB.DB_SCHEMA_VERSION);
+                        if (currentVersion > version) {
+                            deleteAndRecreate = true;
+                        }
+                    } catch (NumberFormatException ex) {
+                        deleteAndRecreate = true;
+                    }
+                }
+
+                if (deleteAndRecreate) {
+                    return updates;
+                }
+
+                final long lastUpdated = Long.parseLong(properties.getProperty(DataStoreMetaInfo.LAST_UPDATED, "0"));
+                final Date now = new Date();
+                final int days = Settings.getInt(Settings.KEYS.CVE_MODIFIED_VALID_FOR_DAYS, 7);
+                if (lastUpdated == updates.getTimeStamp(MODIFIED)) {
+                    updates.clear(); //we don't need to update anything.
+                } else if (withinRange(lastUpdated, now.getTime(), days)) {
+                    for (NvdCveInfo entry : updates) {
+                        if (MODIFIED.equals(entry.getId())) {
+                            entry.setNeedsUpdate(true);
+                        } else {
+                            entry.setNeedsUpdate(false);
+                        }
+                    }
+                } else { //we figure out which of the several XML files need to be downloaded.
+                    for (NvdCveInfo entry : updates) {
+                        if (MODIFIED.equals(entry.getId())) {
+                            entry.setNeedsUpdate(true);
+                        } else {
+                            long currentTimestamp = 0;
+                            try {
+                                currentTimestamp = Long.parseLong(properties.getProperty(DataStoreMetaInfo.LAST_UPDATED_BASE + entry.getId(), "0"));
+                            } catch (NumberFormatException ex) {
+                                final String msg = String.format("Error parsing '%s' '%s' from nvdcve.lastupdated",
+                                        DataStoreMetaInfo.LAST_UPDATED_BASE, entry.getId());
+                                Logger
+                                        .getLogger(StandardUpdate.class
+                                        .getName()).log(Level.FINE, msg, ex);
+                            }
+                            if (currentTimestamp == entry.getTimestamp()) {
+                                entry.setNeedsUpdate(false);
+                            }
+                        }
+                    }
+                }
+            } catch (NumberFormatException ex) {
+                final String msg = "An invalid schema version or timestamp exists in the data.properties file.";
+                Logger
+                        .getLogger(StandardUpdate.class
+                        .getName()).log(Level.WARNING, msg);
+                Logger.getLogger(StandardUpdate.class
+                        .getName()).log(Level.FINE, null, ex);
+            }
+        }
+        return updates;
+    }
+
+    /**
+     * Retrieves the timestamps from the NVD CVE meta data file.
+     *
+     * @return the timestamp from the currently published nvdcve downloads page
+     * @throws MalformedURLException thrown if the URL for the NVD CCE Meta data
+     * is incorrect.
+     * @throws DownloadFailedException thrown if there is an error downloading
+     * the nvd cve meta data file
+     * @throws InvalidDataException thrown if there is an exception parsing the
+     * timestamps
+     * @throws InvalidSettingException thrown if the settings are invalid
+     */
+    private Updateable retrieveCurrentTimestampsFromWeb()
+            throws MalformedURLException, DownloadFailedException, InvalidDataException, InvalidSettingException {
+
+        final Updateable updates = new Updateable();
+        updates.add(MODIFIED, Settings.getString(Settings.KEYS.CVE_MODIFIED_20_URL),
+                Settings.getString(Settings.KEYS.CVE_MODIFIED_12_URL),
+                false);
+
+        final int start = Settings.getInt(Settings.KEYS.CVE_START_YEAR);
+        final int end = Calendar.getInstance().get(Calendar.YEAR);
+        final String baseUrl20 = Settings.getString(Settings.KEYS.CVE_SCHEMA_2_0);
+        final String baseUrl12 = Settings.getString(Settings.KEYS.CVE_SCHEMA_1_2);
+        for (int i = start; i <= end; i++) {
+            updates.add(Integer.toString(i), String.format(baseUrl20, i),
+                    String.format(baseUrl12, i),
+                    true);
+        }
+
+        return updates;
+    }
+
+    /**
+     * Deletes the existing data directories.
+     *
+     * @throws IOException thrown if the directory cannot be deleted
+     */
+    protected void deleteExistingData() throws IOException {
+        File data = Settings.getDataFile(Settings.KEYS.CVE_DATA_DIRECTORY);
+        if (data.exists()) {
+            FileUtils.delete(data);
+        }
+        data = DataStoreMetaInfo.getPropertiesFile();
+        if (data.exists()) {
+            FileUtils.delete(data);
+        }
+    }
+
+    /**
+     * Closes the CVE and CPE data stores.
+     */
+    protected void closeDataStores() {
+        if (cveDB != null) {
+            try {
+                cveDB.close();
+            } catch (Exception ignore) {
+                Logger.getLogger(StandardUpdate.class.getName()).log(Level.FINEST, "Error closing the cveDB", ignore);
+            }
+        }
+    }
+
+    /**
+     * Opens the CVE and CPE data stores.
+     *
+     * @throws UpdateException thrown if a data store cannot be opened
+     */
+    protected void openDataStores() throws UpdateException {
+        //open the cve and cpe data stores
+        try {
+            cveDB = new CveDB();
+            cveDB.open();
+        } catch (IOException ex) {
+            closeDataStores();
+            Logger.getLogger(StandardUpdate.class.getName()).log(Level.FINE, "IO Error opening databases", ex);
+            throw new UpdateException("Error updating the CPE/CVE data, please see the log file for more details.");
+        } catch (SQLException ex) {
+            closeDataStores();
+            Logger.getLogger(StandardUpdate.class.getName()).log(Level.FINE, "SQL Exception opening databases", ex);
+            throw new UpdateException("Error updating the CPE/CVE data, please see the log file for more details.");
+        } catch (DatabaseException ex) {
+            closeDataStores();
+            Logger.getLogger(StandardUpdate.class.getName()).log(Level.FINE, "Database Exception opening databases", ex);
+            throw new UpdateException("Error updating the CPE/CVE data, please see the log file for more details.");
+        } catch (ClassNotFoundException ex) {
+            closeDataStores();
+            Logger.getLogger(StandardUpdate.class.getName()).log(Level.FINE, "Class not found exception opening databases", ex);
+            throw new UpdateException("Error updating the CPE/CVE data, please see the log file for more details.");
+        }
+    }
+
+    /**
+     * Determines if the epoch date is within the range specified of the
+     * compareTo epoch time. This takes the (compareTo-date)/1000/60/60/24 to
+     * get the number of days. If the calculated days is less then the range the
+     * date is considered valid.
+     *
+     * @param date the date to be checked.
+     * @param compareTo the date to compare to.
+     * @param range the range in days to be considered valid.
+     * @return whether or not the date is within the range.
+     */
+    protected boolean withinRange(long date, long compareTo, int range) {
+        final double differenceInDays = (compareTo - date) / 1000.0 / 60.0 / 60.0 / 24.0;
+        return differenceInDays < range;
+    }
+
+    private void spinWaitForFuture(final Future<CallableDownloadTask> future) {
+        //then wait for downloads to finish
+        while (!future.isDone()) {
+            try {
+                Thread.sleep(1000);
+            } catch (InterruptedException ex) {
+                Logger.getLogger(StandardUpdate.class.getName()).log(Level.FINE, null, ex);
+            }
+        }
+    }
+}

dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/Updateable.java

@@ -0,0 +1,181 @@
+/*
+ * This file is part of dependency-check-core.
+ *
+ * Dependency-check-core is free software: you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation, either version 3 of the License, or (at your option) any
+ * later version.
+ *
+ * Dependency-check-core is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+ * details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * dependency-check-core. If not, see http://www.gnu.org/licenses/.
+ *
+ * Copyright (c) 2012 Jeremy Long. All Rights Reserved.
+ */
+package org.owasp.dependencycheck.data.update;
+
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.TreeMap;
+import org.owasp.dependencycheck.utils.DownloadFailedException;
+import org.owasp.dependencycheck.utils.Downloader;
+
+/**
+ * Contains a collection of updateable NvdCveInfo objects. This is used to
+ * determine which files need to be downloaded and processed.
+ *
+ * @author Jeremy Long (jeremy.long@owasp.org)
+ */
+public class Updateable implements java.lang.Iterable<NvdCveInfo>, Iterator<NvdCveInfo> {
+
+    /**
+     * A collection of sources of data.
+     */
+    private Map<String, NvdCveInfo> collection = new TreeMap<String, NvdCveInfo>();
+
+    /**
+     * Returns the collection of NvdCveInfo objects. This method is mainly used
+     * for testing.
+     *
+     * @return the collection of NvdCveInfo objects
+     */
+    protected Map<String, NvdCveInfo> getCollection() {
+        return collection;
+    }
+
+    /**
+     * Gets whether or not an update is needed.
+     *
+     * @return true or false depending on whether an update is needed
+     */
+    public boolean isUpdateNeeded() {
+        for (NvdCveInfo item : this) {
+            if (item.getNeedsUpdate()) {
+                return true;
+            }
+        }
+        return false;
+    }
+
+    /**
+     * Adds a new entry of updateable information to the contained collection.
+     *
+     * @param id the key for the item to be added
+     * @param url the URL to download the item
+     * @param oldUrl the URL for the old version of the item (the NVD CVE old
+     * schema still contains useful data we need).
+     * @throws MalformedURLException thrown if the URL provided is invalid
+     * @throws DownloadFailedException thrown if the download fails.
+     */
+    public void add(String id, String url, String oldUrl) throws MalformedURLException, DownloadFailedException {
+        add(id, url, oldUrl, false);
+    }
+
+    /**
+     * Adds a new entry of updateable information to the contained collection.
+     *
+     * @param id the key for the item to be added
+     * @param url the URL to download the item
+     * @param oldUrl the URL for the old version of the item (the NVD CVE old
+     * schema still contains useful data we need).
+     * @param needsUpdate whether or not the data needs to be updated
+     * @throws MalformedURLException thrown if the URL provided is invalid
+     * @throws DownloadFailedException thrown if the download fails.
+     */
+    public void add(String id, String url, String oldUrl, boolean needsUpdate) throws MalformedURLException, DownloadFailedException {
+        final NvdCveInfo item = new NvdCveInfo();
+        item.setNeedsUpdate(needsUpdate); //the others default to true, to make life easier later this should default to false.
+        item.setId(id);
+        item.setUrl(url);
+        item.setOldSchemaVersionUrl(oldUrl);
+        item.setTimestamp(Downloader.getLastModified(new URL(url)));
+        collection.put(id, item);
+    }
+
+    /**
+     * Clears the contained collection of NvdCveInfo entries.
+     */
+    public void clear() {
+        collection.clear();
+    }
+
+    /**
+     * Returns the timestamp for the given entry.
+     *
+     * @param key the key to lookup in the collection of NvdCveInfo items
+     * @return the timestamp for the given entry
+     */
+    public long getTimeStamp(String key) {
+        return collection.get(key).getTimestamp();
+    }
+    /**
+     * An internal iterator used to implement iterable.
+     */
+    private Iterator<Entry<String, NvdCveInfo>> iterableContent = null;
+
+    /**
+     * <p>Returns an iterator for the NvdCveInfo contained.</p>
+     * <p><b>This method is not thread safe.</b></p>
+     *
+     * @return an NvdCveInfo Iterator
+     */
+    @Override
+    public Iterator<NvdCveInfo> iterator() {
+        iterableContent = collection.entrySet().iterator();
+        return this;
+    }
+
+    /**
+     * <p>Returns whether or not there is another item in the collection.</p>
+     * <p><b>This method is not thread safe.</b></p>
+     *
+     * @return true or false depending on whether or not another item exists in
+     * the collection
+     */
+    @Override
+    public boolean hasNext() {
+        return iterableContent.hasNext();
+    }
+
+    /**
+     * <p>Returns the next item in the collection.</p>
+     * <p><b>This method is not thread safe.</b></p>
+     *
+     * @return the next NvdCveInfo item in the collection
+     */
+    @Override
+    public NvdCveInfo next() {
+        return iterableContent.next().getValue();
+    }
+
+    /**
+     * <p>Removes the current NvdCveInfo object from the collection.</p>
+     * <p><b>This method is not thread safe.</b></p>
+     */
+    @Override
+    public void remove() {
+        iterableContent.remove();
+    }
+
+    /**
+     * Returns the specified item from the collection.
+     *
+     * @param key the key to lookup the return value
+     * @return the NvdCveInfo object stored using the specified key
+     */
+    NvdCveInfo get(String key) {
+        return collection.get(key);
+    }
+
+    @Override
+    public String toString() {
+        return "Updateable{" + "size=" + collection.size() + '}';
+    }
+}

dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/package-info.java

@@ -15,4 +15,4 @@
  * </html>
 */
 
-package org.owasp.dependencycheck.data.nvdcve.xml;
+package org.owasp.dependencycheck.data.update;

dependency-check-core/src/main/java/org/owasp/dependencycheck/suppression/PropertyType.java

@@ -0,0 +1,186 @@
+/*
+ * This file is part of dependency-check-core.
+ *
+ * Dependency-check-core is free software: you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation, either version 3 of the License, or (at your option) any
+ * later version.
+ *
+ * Dependency-check-core is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+ * details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * dependency-check-core. If not, see http://www.gnu.org/licenses/.
+ *
+ * Copyright (c) 2013 Jeremy Long. All Rights Reserved.
+ */
+package org.owasp.dependencycheck.suppression;
+
+import java.util.regex.Pattern;
+
+/**
+ * A simple PropertyType used to represent a string value that could be used as
+ * a regular expression or could be case insensitive. The equals method has been
+ * over-ridden so that the object will correctly compare to strings.
+ *
+ * @author Jeremy Long (jeremy.long@owasp.org)
+ */
+public class PropertyType {
+
+    //<editor-fold defaultstate="collapsed" desc="properties">
+    /**
+     * The value.
+     */
+    private String value;
+
+    /**
+     * Gets the value of the value property.
+     *
+     * @return the value of the value property
+     *
+     */
+    public String getValue() {
+        return value;
+    }
+
+    /**
+     * Sets the value of the value property.
+     *
+     * @param value the value of the value property
+     */
+    public void setValue(String value) {
+        this.value = value;
+    }
+    /**
+     * Whether or not the expression is a regex.
+     */
+    private boolean regex = false;
+
+    /**
+     * Returns whether or not the value is a regex.
+     *
+     * @return true if the value is a regex, otherwise false
+     *
+     */
+    public boolean isRegex() {
+        return regex;
+    }
+
+    /**
+     * Sets whether the value property is a regex.
+     *
+     * @param value true if the value is a regex, otherwise false
+     *
+     */
+    public void setRegex(boolean value) {
+        this.regex = value;
+    }
+    /**
+     * Indicates case sensitivity.
+     */
+    private boolean caseSensitive = false;
+
+    /**
+     * Gets the value of the caseSensitive property.
+     *
+     * @return true if the value is case sensitive
+     *
+     */
+    public boolean isCaseSensitive() {
+        return caseSensitive;
+    }
+
+    /**
+     * Sets the value of the caseSensitive property.
+     *
+     * @param value whether the value is case sensitive
+     *
+     */
+    public void setCaseSensitive(boolean value) {
+        this.caseSensitive = value;
+    }
+    //</editor-fold>
+
+    /**
+     * Uses the object's properties to determine if the supplied string matches
+     * the value of this property.
+     *
+     * @param text the String to validate
+     * @return whether the text supplied is matched by the value of the property
+     */
+    public boolean matches(String text) {
+        if (text == null) {
+            return false;
+        }
+        if (this.regex) {
+            Pattern rx;
+            if (this.caseSensitive) {
+                rx = Pattern.compile(this.value);
+            } else {
+                rx = Pattern.compile(this.value, Pattern.CASE_INSENSITIVE);
+            }
+            return rx.matcher(text).matches();
+        } else {
+            if (this.caseSensitive) {
+                return value.equals(text);
+            } else {
+                return value.equalsIgnoreCase(text);
+            }
+        }
+    }
+
+    //<editor-fold defaultstate="collapsed" desc="standard implmentations of hashCode, equals, and toString">
+    /**
+     * Default implementation of hashCode.
+     *
+     * @return the hash code
+     */
+    @Override
+    public int hashCode() {
+        int hash = 3;
+        hash = 59 * hash + (this.value != null ? this.value.hashCode() : 0);
+        hash = 59 * hash + (this.regex ? 1 : 0);
+        hash = 59 * hash + (this.caseSensitive ? 1 : 0);
+        return hash;
+    }
+
+    /**
+     * Default implementation of equals.
+     *
+     * @param obj the object to compare
+     * @return whether the objects are equivalent
+     */
+    @Override
+    public boolean equals(Object obj) {
+        if (obj == null) {
+            return false;
+        }
+        if (getClass() != obj.getClass()) {
+            return false;
+        }
+        final PropertyType other = (PropertyType) obj;
+        if ((this.value == null) ? (other.value != null) : !this.value.equals(other.value)) {
+            return false;
+        }
+        if (this.regex != other.regex) {
+            return false;
+        }
+        if (this.caseSensitive != other.caseSensitive) {
+            return false;
+        }
+        return true;
+    }
+
+    /**
+     * Default implementation of toString().
+     *
+     * @return the string representation of the object
+     */
+    @Override
+    public String toString() {
+        return "PropertyType{" + "value=" + value + ", regex=" + regex + ", caseSensitive=" + caseSensitive + '}';
+    }
+    //</editor-fold>
+}

dependency-check-core/src/main/java/org/owasp/dependencycheck/suppression/SuppressionErrorHandler.java

@@ -0,0 +1,93 @@
+/*
+ * This file is part of dependency-check-core.
+ *
+ * Dependency-check-core is free software: you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation, either version 3 of the License, or (at your option) any
+ * later version.
+ *
+ * Dependency-check-core is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+ * details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * dependency-check-core. If not, see http://www.gnu.org/licenses/.
+ *
+ * Copyright (c) 2013 Jeremy Long. All Rights Reserved.
+ */
+package org.owasp.dependencycheck.suppression;
+
+import java.util.logging.Level;
+import java.util.logging.Logger;
+import org.xml.sax.ErrorHandler;
+import org.xml.sax.SAXException;
+import org.xml.sax.SAXParseException;
+
+/**
+ * An XML parsing error handler.
+ *
+ * @author Jeremy Long (jeremy.long@owasp.org)
+ */
+public class SuppressionErrorHandler implements ErrorHandler {
+
+    /**
+     * Builds a prettier exception message.
+     *
+     * @param ex the SAXParseException
+     * @return an easier to read exception message
+     */
+    private String getPrettyParseExceptionInfo(SAXParseException ex) {
+
+        final StringBuffer sb = new StringBuffer();
+
+        if (ex.getSystemId() != null) {
+            sb.append("systemId=").append(ex.getSystemId()).append(", ");
+        }
+        if (ex.getPublicId() != null) {
+            sb.append("publicId=").append(ex.getPublicId()).append(", ");
+        }
+        if (ex.getLineNumber() > 0) {
+            sb.append("Line=").append(ex.getLineNumber());
+        }
+        if (ex.getColumnNumber() > 0) {
+            sb.append(", Column=").append(ex.getColumnNumber());
+        }
+        sb.append(": ").append(ex.getMessage());
+
+        return sb.toString();
+    }
+
+    /**
+     * Logs warnings.
+     *
+     * @param ex the warning to log
+     * @throws SAXException is never thrown
+     */
+    @Override
+    public void warning(SAXParseException ex) throws SAXException {
+        Logger.getLogger(SuppressionErrorHandler.class.getName()).log(Level.FINE, null, ex);
+    }
+
+    /**
+     * Handles errors.
+     *
+     * @param ex the error to handle
+     * @throws SAXException is always thrown
+     */
+    @Override
+    public void error(SAXParseException ex) throws SAXException {
+        throw new SAXException(getPrettyParseExceptionInfo(ex));
+    }
+
+    /**
+     * Handles fatal exceptions.
+     *
+     * @param ex a fatal exception
+     * @throws SAXException is always
+     */
+    @Override
+    public void fatalError(SAXParseException ex) throws SAXException {
+        throw new SAXException(getPrettyParseExceptionInfo(ex));
+    }
+}

dependency-check-core/src/main/java/org/owasp/dependencycheck/suppression/SuppressionHandler.java

@@ -0,0 +1,174 @@
+/*
+ * This file is part of dependency-check-core.
+ *
+ * Dependency-check-core is free software: you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation, either version 3 of the License, or (at your option) any
+ * later version.
+ *
+ * Dependency-check-core is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+ * details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * dependency-check-core. If not, see http://www.gnu.org/licenses/.
+ *
+ * Copyright (c) 2013 Jeremy Long. All Rights Reserved.
+ */
+package org.owasp.dependencycheck.suppression;
+
+import java.util.ArrayList;
+import java.util.List;
+import org.xml.sax.Attributes;
+import org.xml.sax.SAXException;
+import org.xml.sax.helpers.DefaultHandler;
+
+/**
+ * A handler to load suppression rules.
+ *
+ * @author Jeremy Long (jeremy.long@owasp.org)
+ */
+public class SuppressionHandler extends DefaultHandler {
+
+    /**
+     * The suppress node, indicates the start of a new rule.
+     */
+    public static final String SUPPRESS = "suppress";
+    /**
+     * The file path element name.
+     */
+    public static final String FILE_PATH = "filePath";
+    /**
+     * The sha1 hash element name.
+     */
+    public static final String SHA1 = "sha1";
+    /**
+     * The CVE element name.
+     */
+    public static final String CVE = "cve";
+    /**
+     * The CPE element name.
+     */
+    public static final String CPE = "cpe";
+    /**
+     * The CWE element name.
+     */
+    public static final String CWE = "cwe";
+    /**
+     * The cvssBelow element name.
+     */
+    public static final String CVSS_BELOW = "cvssBelow";
+    /**
+     * A list of suppression rules.
+     */
+    private List<SuppressionRule> supressionRules = new ArrayList<SuppressionRule>();
+
+    /**
+     * Get the value of supressionRules.
+     *
+     * @return the value of supressionRules
+     */
+    public List<SuppressionRule> getSupressionRules() {
+        return supressionRules;
+    }
+    /**
+     * The current rule being read.
+     */
+    private SuppressionRule rule;
+    /**
+     * The attributes of the node being read.
+     */
+    private Attributes currentAttributes;
+    /**
+     * The current node text being extracted from the element.
+     */
+    private StringBuffer currentText;
+
+    /**
+     * Handles the start element event.
+     *
+     * @param uri the uri of the element being processed
+     * @param localName the local name of the element being processed
+     * @param qName the qName of the element being processed
+     * @param attributes the attributes of the element being processed
+     * @throws SAXException thrown if there is an exception processing
+     */
+    @Override
+    public void startElement(String uri, String localName, String qName, Attributes attributes) throws SAXException {
+        currentAttributes = null;
+        currentText = new StringBuffer();
+
+        if (SUPPRESS.equals(qName)) {
+            rule = new SuppressionRule();
+        } else if (FILE_PATH.equals(qName)) {
+            currentAttributes = attributes;
+        }
+    }
+
+    /**
+     * Handles the end element event.
+     *
+     * @param uri the uri of the element
+     * @param localName the local name of the element
+     * @param qName the qName of the element
+     * @throws SAXException thrown if there is an exception processing
+     */
+    @Override
+    public void endElement(String uri, String localName, String qName) throws SAXException {
+        if (SUPPRESS.equals(qName)) {
+            supressionRules.add(rule);
+            rule = null;
+        } else if (FILE_PATH.equals(qName)) {
+            final PropertyType pt = processPropertyType();
+            rule.setFilePath(pt);
+        } else if (SHA1.equals(qName)) {
+            rule.setSha1(currentText.toString());
+        } else if (CPE.equals(qName)) {
+            final PropertyType pt = processPropertyType();
+            rule.addCpe(pt);
+        } else if (CWE.equals(qName)) {
+            rule.addCwe(currentText.toString());
+        } else if (CVE.equals(qName)) {
+            rule.addCve(currentText.toString());
+        } else if (CVSS_BELOW.equals(qName)) {
+            final float cvss = Float.parseFloat(currentText.toString());
+            rule.addCvssBelow(cvss);
+        }
+    }
+
+    /**
+     * Collects the body text of the node being processed.
+     *
+     * @param ch the char array of text
+     * @param start the start position to copy text from in the char array
+     * @param length the number of characters to copy from the char array
+     * @throws SAXException thrown if there is a parsing exception
+     */
+    @Override
+    public void characters(char[] ch, int start, int length) throws SAXException {
+        currentText.append(ch, start, length);
+    }
+
+    /**
+     * Processes field members that have been collected during the characters
+     * and startElement method to construct a PropertyType object.
+     *
+     * @return a PropertyType object
+     */
+    private PropertyType processPropertyType() {
+        final PropertyType pt = new PropertyType();
+        pt.setValue(currentText.toString());
+        if (currentAttributes != null && currentAttributes.getLength() > 0) {
+            final String regex = currentAttributes.getValue("regex");
+            if (regex != null) {
+                pt.setRegex(Boolean.parseBoolean(regex));
+            }
+            final String caseSensitive = currentAttributes.getValue("caseSensitive");
+            if (regex != null) {
+                pt.setCaseSensitive(Boolean.parseBoolean(caseSensitive));
+            }
+        }
+        return pt;
+    }
+}

dependency-check-core/src/main/java/org/owasp/dependencycheck/suppression/SuppressionParseException.java

@@ -0,0 +1,69 @@
+/*
+ * This file is part of dependency-check-core.
+ *
+ * Dependency-check-core is free software: you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation, either version 3 of the License, or (at your option) any
+ * later version.
+ *
+ * Dependency-check-core is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+ * details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * dependency-check-core. If not, see http://www.gnu.org/licenses/.
+ *
+ * Copyright (c) 2013 Jeremy Long. All Rights Reserved.
+ */
+package org.owasp.dependencycheck.suppression;
+
+import java.io.IOException;
+
+/**
+ * An exception used when parsing a suppression rule file fails.
+ *
+ * @author Jeremy Long (jeremy.long@owasp.org)
+ */
+public class SuppressionParseException extends IOException {
+
+    /**
+     * The serial version UID.
+     */
+    private static final long serialVersionUID = 1L;
+
+    /**
+     * Creates a new SuppressionParseException.
+     */
+    public SuppressionParseException() {
+        super();
+    }
+
+    /**
+     * Creates a new SuppressionParseException.
+     *
+     * @param msg a message for the exception.
+     */
+    public SuppressionParseException(String msg) {
+        super(msg);
+    }
+
+    /**
+     * Creates a new SuppressionParseException.
+     *
+     * @param ex the cause of the download failure.
+     */
+    public SuppressionParseException(Throwable ex) {
+        super(ex);
+    }
+
+    /**
+     * Creates a new SuppressionParseException.
+     *
+     * @param msg a message for the exception.
+     * @param ex the cause of the download failure.
+     */
+    public SuppressionParseException(String msg, Throwable ex) {
+        super(msg, ex);
+    }
+}

dependency-check-core/src/main/java/org/owasp/dependencycheck/suppression/SuppressionParser.java

@@ -0,0 +1,107 @@
+/*
+ * This file is part of dependency-check-core.
+ *
+ * Dependency-check-core is free software: you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation, either version 3 of the License, or (at your option) any
+ * later version.
+ *
+ * Dependency-check-core is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+ * details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * dependency-check-core. If not, see http://www.gnu.org/licenses/.
+ *
+ * Copyright (c) 2013 Jeremy Long. All Rights Reserved.
+ */
+package org.owasp.dependencycheck.suppression;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.io.Reader;
+import java.util.List;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+import javax.xml.parsers.ParserConfigurationException;
+import javax.xml.parsers.SAXParser;
+import javax.xml.parsers.SAXParserFactory;
+import org.xml.sax.InputSource;
+import org.xml.sax.SAXException;
+import org.xml.sax.XMLReader;
+
+/**
+ * A simple validating parser for XML Suppression Rules.
+ *
+ * @author Jeremy Long (jeremy.long@owasp.org)
+ */
+public class SuppressionParser {
+
+    /**
+     * JAXP Schema Language. Source:
+     * http://docs.oracle.com/javase/tutorial/jaxp/sax/validation.html
+     */
+    public static final String JAXP_SCHEMA_LANGUAGE = "http://java.sun.com/xml/jaxp/properties/schemaLanguage";
+    /**
+     * W3C XML Schema. Source:
+     * http://docs.oracle.com/javase/tutorial/jaxp/sax/validation.html
+     */
+    public static final String W3C_XML_SCHEMA = "http://www.w3.org/2001/XMLSchema";
+    /**
+     * JAXP Schema Source. Source:
+     * http://docs.oracle.com/javase/tutorial/jaxp/sax/validation.html
+     */
+    public static final String JAXP_SCHEMA_SOURCE = "http://java.sun.com/xml/jaxp/properties/schemaSource";
+
+    /**
+     * Parses the given xml file and returns a list of the suppression rules
+     * contained.
+     *
+     * @param file an xml file containing suppression rules
+     * @return a list of suppression rules
+     * @throws SuppressionParseException thrown if the xml file cannot be parsed
+     */
+    public List<SuppressionRule> parseSuppressionRules(File file) throws SuppressionParseException {
+        try {
+            final InputStream schemaStream = this.getClass().getClassLoader().getResourceAsStream("schema/suppression.xsd");
+            final SuppressionHandler handler = new SuppressionHandler();
+
+            final SAXParserFactory factory = SAXParserFactory.newInstance();
+            factory.setNamespaceAware(true);
+            factory.setValidating(true);
+            final SAXParser saxParser = factory.newSAXParser();
+            saxParser.setProperty(SuppressionParser.JAXP_SCHEMA_LANGUAGE, SuppressionParser.W3C_XML_SCHEMA);
+            saxParser.setProperty(SuppressionParser.JAXP_SCHEMA_SOURCE, new InputSource(schemaStream));
+            final XMLReader xmlReader = saxParser.getXMLReader();
+            xmlReader.setErrorHandler(new SuppressionErrorHandler());
+            xmlReader.setContentHandler(handler);
+
+            final InputStream inputStream = new FileInputStream(file);
+            final Reader reader = new InputStreamReader(inputStream, "UTF-8");
+            final InputSource in = new InputSource(reader);
+            //in.setEncoding("UTF-8");
+
+            xmlReader.parse(in);
+
+
+            return handler.getSupressionRules();
+        } catch (ParserConfigurationException ex) {
+            Logger.getLogger(SuppressionParser.class.getName()).log(Level.FINE, null, ex);
+            throw new SuppressionParseException(ex);
+        } catch (SAXException ex) {
+            Logger.getLogger(SuppressionParser.class.getName()).log(Level.FINE, null, ex);
+            throw new SuppressionParseException(ex);
+        } catch (FileNotFoundException ex) {
+            Logger.getLogger(SuppressionParser.class.getName()).log(Level.FINE, null, ex);
+            throw new SuppressionParseException(ex);
+        } catch (IOException ex) {
+            Logger.getLogger(SuppressionParser.class.getName()).log(Level.FINE, null, ex);
+            throw new SuppressionParseException(ex);
+        }
+    }
+}

dependency-check-core/src/main/java/org/owasp/dependencycheck/suppression/SuppressionRule.java

@@ -0,0 +1,365 @@
+/*
+ * This file is part of dependency-check-core.
+ *
+ * Dependency-check-core is free software: you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation, either version 3 of the License, or (at your option) any
+ * later version.
+ *
+ * Dependency-check-core is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+ * details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * dependency-check-core. If not, see http://www.gnu.org/licenses/.
+ *
+ * Copyright (c) 2013 Jeremy Long. All Rights Reserved.
+ */
+package org.owasp.dependencycheck.suppression;
+
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+import org.owasp.dependencycheck.dependency.Dependency;
+import org.owasp.dependencycheck.dependency.Identifier;
+import org.owasp.dependencycheck.dependency.Vulnerability;
+
+/**
+ *
+ * @author Jeremy Long (jeremy.long@owasp.org)
+ */
+public class SuppressionRule {
+
+    /**
+     * The file path for the suppression.
+     */
+    private PropertyType filePath;
+
+    /**
+     * Get the value of filePath.
+     *
+     * @return the value of filePath
+     */
+    public PropertyType getFilePath() {
+        return filePath;
+    }
+
+    /**
+     * Set the value of filePath.
+     *
+     * @param filePath new value of filePath
+     */
+    public void setFilePath(PropertyType filePath) {
+        this.filePath = filePath;
+    }
+    /**
+     * The sha1 hash.
+     */
+    private String sha1;
+
+    /**
+     * Get the value of sha1.
+     *
+     * @return the value of sha1
+     */
+    public String getSha1() {
+        return sha1;
+    }
+
+    /**
+     * Set the value of sha1.
+     *
+     * @param sha1 new value of sha1
+     */
+    public void setSha1(String sha1) {
+        this.sha1 = sha1;
+    }
+    /**
+     * A list of CPEs to suppression
+     */
+    private List<PropertyType> cpe = new ArrayList<PropertyType>();
+
+    /**
+     * Get the value of cpe.
+     *
+     * @return the value of cpe
+     */
+    public List<PropertyType> getCpe() {
+        return cpe;
+    }
+
+    /**
+     * Set the value of cpe.
+     *
+     * @param cpe new value of cpe
+     */
+    public void setCpe(List<PropertyType> cpe) {
+        this.cpe = cpe;
+    }
+
+    /**
+     * Adds the cpe to the cpe list.
+     *
+     * @param cpe the cpe to add
+     */
+    public void addCpe(PropertyType cpe) {
+        this.cpe.add(cpe);
+    }
+
+    /**
+     * Returns whether or not this suppression rule as CPE entries.
+     *
+     * @return whether or not this suppression rule as CPE entries
+     */
+    public boolean hasCpe() {
+        return cpe.size() > 0;
+    }
+    /**
+     * The list of cvssBelow scores.
+     */
+    private List<Float> cvssBelow = new ArrayList<Float>();
+
+    /**
+     * Get the value of cvssBelow.
+     *
+     * @return the value of cvssBelow
+     */
+    public List<Float> getCvssBelow() {
+        return cvssBelow;
+    }
+
+    /**
+     * Set the value of cvssBelow.
+     *
+     * @param cvssBelow new value of cvssBelow
+     */
+    public void setCvssBelow(List<Float> cvssBelow) {
+        this.cvssBelow = cvssBelow;
+    }
+
+    /**
+     * Adds the cvss to the cvssBelow list.
+     *
+     * @param cvss the cvss to add
+     */
+    public void addCvssBelow(Float cvss) {
+        this.cvssBelow.add(cvss);
+    }
+
+    /**
+     * Returns whether or not this suppression rule has cvss suppressions.
+     *
+     * @return whether or not this suppression rule has cvss suppressions
+     */
+    public boolean hasCvssBelow() {
+        return cvssBelow.size() > 0;
+    }
+    /**
+     * The list of cwe entries to suppress.
+     */
+    private List<String> cwe = new ArrayList<String>();
+
+    /**
+     * Get the value of cwe.
+     *
+     * @return the value of cwe
+     */
+    public List<String> getCwe() {
+        return cwe;
+    }
+
+    /**
+     * Set the value of cwe.
+     *
+     * @param cwe new value of cwe
+     */
+    public void setCwe(List<String> cwe) {
+        this.cwe = cwe;
+    }
+
+    /**
+     * Adds the cwe to the cwe list.
+     *
+     * @param cwe the cwe to add
+     */
+    public void addCwe(String cwe) {
+        this.cwe.add(cwe);
+    }
+
+    /**
+     * Returns whether this suppression rule has CWE entries.
+     *
+     * @return whether this suppression rule has CWE entries
+     */
+    public boolean hasCwe() {
+        return cwe.size() > 0;
+    }
+    /**
+     * The list of cve entries to suppress.
+     */
+    private List<String> cve = new ArrayList<String>();
+
+    /**
+     * Get the value of cve.
+     *
+     * @return the value of cve
+     */
+    public List<String> getCve() {
+        return cve;
+    }
+
+    /**
+     * Set the value of cve.
+     *
+     * @param cve new value of cve
+     */
+    public void setCve(List<String> cve) {
+        this.cve = cve;
+    }
+
+    /**
+     * Adds the cve to the cve list.
+     *
+     * @param cve the cve to add
+     */
+    public void addCve(String cve) {
+        this.cve.add(cve);
+    }
+
+    /**
+     * Returns whether this suppression rule has CVE entries.
+     *
+     * @return whether this suppression rule has CVE entries
+     */
+    public boolean hasCve() {
+        return cve.size() > 0;
+    }
+
+    /**
+     * Processes a given dependency to determine if any CPE, CVE, CWE, or CVSS
+     * scores should be suppressed. If any should be, they are removed from the
+     * dependency.
+     *
+     * @param dependency a project dependency to analyze
+     */
+    public void process(Dependency dependency) {
+        if (filePath != null && !filePath.matches(dependency.getFilePath())) {
+            return;
+        }
+        if (sha1 != null && !sha1.equalsIgnoreCase(dependency.getSha1sum())) {
+            return;
+        }
+        if (this.hasCpe()) {
+            final Iterator<Identifier> itr = dependency.getIdentifiers().iterator();
+            while (itr.hasNext()) {
+                final Identifier i = itr.next();
+                for (PropertyType c : this.cpe) {
+                    if (cpeMatches(c, i)) {
+                        itr.remove();
+                        break;
+                    }
+                }
+            }
+        }
+        if (hasCve() || hasCwe() || hasCvssBelow()) {
+            final Iterator<Vulnerability> itr = dependency.getVulnerabilities().iterator();
+            while (itr.hasNext()) {
+                boolean remove = false;
+                final Vulnerability v = itr.next();
+                for (String entry : this.cve) {
+                    if (entry.equalsIgnoreCase(v.getName())) {
+                        remove = true;
+                        break;
+                    }
+                }
+                if (!remove) {
+                    for (String entry : this.cwe) {
+                        if (v.getCwe() != null) {
+                            final String toMatch = String.format("CWE-%s ", entry);
+                            final String toTest = v.getCwe().substring(0, toMatch.length()).toUpperCase();
+                            if (toTest.equals(toMatch)) {
+                                remove = true;
+                                break;
+                            }
+                        }
+                    }
+                }
+                if (!remove) {
+                    for (float cvss : this.cvssBelow) {
+                        if (v.getCvssScore() < cvss) {
+                            remove = true;
+                            break;
+                        }
+                    }
+                }
+                if (remove) {
+                    itr.remove();
+                }
+            }
+        }
+    }
+
+    /**
+     * Identifies if the cpe specified by the cpe suppression rule does not
+     * specify a version.
+     *
+     * @param c a suppression rule identifier
+     * @return true if the property type does not specify a version; otherwise
+     * false
+     */
+    boolean cpeHasNoVersion(PropertyType c) {
+        if (c.isRegex()) {
+            return false;
+        } // cpe:/a:jboss:jboss:1.0.0:
+        if (countCharacter(c.getValue(), ':') == 3) {
+            return true;
+        }
+        return false;
+    }
+
+    /**
+     * Counts the number of occurrences of the character found within the
+     * string.
+     *
+     * @param str the string to check
+     * @param c the character to count
+     * @return the number of times the character is found in the string
+     */
+    int countCharacter(String str, char c) {
+        int count = 0;
+        int pos = str.indexOf(c) + 1;
+        while (pos > 0) {
+            count += 1;
+            pos = str.indexOf(c, pos) + 1;
+        }
+        return count;
+    }
+
+    /**
+     * Determines if the cpeEntry specified as a PropertyType matches the given
+     * Identifier.
+     *
+     * @param cpeEntry a suppression rule entry
+     * @param identifier a CPE identifier to check
+     * @return true if the entry matches; otherwise false
+     */
+    boolean cpeMatches(PropertyType cpeEntry, Identifier identifier) {
+        if (cpeEntry.matches(identifier.getValue())) {
+            return true;
+        } else if (cpeHasNoVersion(cpeEntry)) {
+            if (cpeEntry.isCaseSensitive()) {
+                if (identifier.getValue().startsWith(cpeEntry.getValue())) {
+                    return true;
+                }
+            } else {
+                final String id = identifier.getValue().toLowerCase();
+                final String check = cpeEntry.getValue().toLowerCase();
+                if (id.startsWith(check)) {
+                    return true;
+                }
+            }
+        }
+        return false;
+    }
+}

dependency-check-core/src/main/java/org/owasp/dependencycheck/suppression/package-info.java

@@ -0,0 +1,11 @@
+/**
+ * <html>
+ * <head>
+ * <title>org.owasp.dependencycheck.suppression</title>
+ * </head>
+ * <body>
+ * Contains classes used to suppress findings.
+ * </body>
+ * </html>
+ */
+package org.owasp.dependencycheck.suppression;

dependency-check-core/src/main/java/org/owasp/dependencycheck/utils/Downloader.java

@@ -23,10 +23,13 @@ import java.io.File;
 import java.io.FileOutputStream;
 import java.io.IOException;
 import java.io.InputStream;
+import java.net.Authenticator;
 import java.net.HttpURLConnection;
 import java.net.InetSocketAddress;
+import java.net.PasswordAuthentication;
 import java.net.Proxy;
 import java.net.SocketAddress;
+import java.net.URISyntaxException;
 import java.net.URL;
 import java.util.logging.Level;
 import java.util.logging.Logger;
@@ -46,33 +49,6 @@ public final class Downloader {
     private Downloader() {
     }
 
-    /**
-     * Retrieves a file from a given URL and saves it to the outputPath.
-     *
-     * @param url the URL of the file to download.
-     * @param outputPath the path to the save the file to.
-     * @throws DownloadFailedException is thrown if there is an error
-     * downloading the file.
-     */
-    public static void fetchFile(URL url, String outputPath) throws DownloadFailedException {
-        fetchFile(url, outputPath, false);
-    }
-
-    /**
-     * Retrieves a file from a given URL and saves it to the outputPath.
-     *
-     * @param url the URL of the file to download.
-     * @param outputPath the path to the save the file to.
-     * @param unzip true/false indicating that the file being retrieved is
-     * gzipped and if true, should be uncompressed before writing to the file.
-     * @throws DownloadFailedException is thrown if there is an error
-     * downloading the file.
-     */
-    public static void fetchFile(URL url, String outputPath, boolean unzip) throws DownloadFailedException {
-        final File f = new File(outputPath);
-        fetchFile(url, f, unzip);
-    }
-
     /**
      * Retrieves a file from a given URL and saves it to the outputPath.
      *
@@ -82,20 +58,6 @@ public final class Downloader {
      * downloading the file.
      */
     public static void fetchFile(URL url, File outputPath) throws DownloadFailedException {
-        fetchFile(url, outputPath, false);
-    }
-
-    /**
-     * Retrieves a file from a given URL and saves it to the outputPath.
-     *
-     * @param url the URL of the file to download.
-     * @param outputPath the path to the save the file to.
-     * @param unzip true/false indicating that the file being retrieved is
-     * gzipped and if true, should be uncompressed before writing to the file.
-     * @throws DownloadFailedException is thrown if there is an error
-     * downloading the file.
-     */
-    public static void fetchFile(URL url, File outputPath, boolean unzip) throws DownloadFailedException {
         HttpURLConnection conn = null;
         try {
             conn = Downloader.getConnection(url);
@@ -116,7 +78,7 @@ public final class Downloader {
         BufferedOutputStream writer = null;
         InputStream reader = null;
         try {
-            if (unzip || (encoding != null && "gzip".equalsIgnoreCase(encoding))) {
+            if (encoding != null && "gzip".equalsIgnoreCase(encoding)) {
                 reader = new GZIPInputStream(conn.getInputStream());
             } else if (encoding != null && "deflate".equalsIgnoreCase(encoding)) {
                 reader = new InflaterInputStream(conn.getInputStream());
@@ -136,7 +98,6 @@ public final class Downloader {
             if (writer != null) {
                 try {
                     writer.close();
-                    writer = null;
                 } catch (Exception ex) {
                     Logger.getLogger(Downloader.class.getName()).log(Level.FINEST,
                             "Error closing the writer in Downloader.", ex);
@@ -145,9 +106,7 @@ public final class Downloader {
             if (reader != null) {
                 try {
                     reader.close();
-                    reader = null;
                 } catch (Exception ex) {
-
                     Logger.getLogger(Downloader.class.getName()).log(Level.FINEST,
                             "Error closing the reader in Downloader.", ex);
                 }
@@ -162,7 +121,8 @@ public final class Downloader {
 
     /**
      * Makes an HTTP Head request to retrieve the last modified date of the
-     * given URL.
+     * given URL. If the file:// protocol is specified, then the lastTimestamp
+     * of the file is returned.
      *
      * @param url the URL to retrieve the timestamp from
      * @return an epoch timestamp
@@ -170,21 +130,42 @@ public final class Downloader {
      * the HTTP request
      */
     public static long getLastModified(URL url) throws DownloadFailedException {
-        HttpURLConnection conn = null;
         long timestamp = 0;
-        try {
-            conn = Downloader.getConnection(url);
-            conn.setRequestMethod("HEAD");
-            conn.connect();
-            timestamp = conn.getLastModified();
-        } catch (Exception ex) {
-            throw new DownloadFailedException("Error making HTTP HEAD request.", ex);
-        } finally {
-            if (conn != null) {
-                try {
-                    conn.disconnect();
-                } finally {
-                    conn = null;
+        //TODO add the FPR protocol?
+        if ("file".equalsIgnoreCase(url.getProtocol())) {
+            File lastModifiedFile;
+            try {
+//                if (System.getProperty("os.name").toLowerCase().startsWith("windows")) {
+//                    String filePath = url.toString();
+//                    if (filePath.matches("file://[a-zA-Z]:.*")) {
+//                        f = new File(filePath.substring(7));
+//                    } else {
+//                        f = new File(url.toURI());
+//                    }
+//                } else {
+                lastModifiedFile = new File(url.toURI());
+//                }
+            } catch (URISyntaxException ex) {
+                final String msg = String.format("Unable to locate '%s'; is the cve.url-2.0.modified property set correctly?", url.toString());
+                throw new DownloadFailedException(msg);
+            }
+            timestamp = lastModifiedFile.lastModified();
+        } else {
+            HttpURLConnection conn = null;
+            try {
+                conn = Downloader.getConnection(url);
+                conn.setRequestMethod("HEAD");
+                conn.connect();
+                timestamp = conn.getLastModified();
+            } catch (Exception ex) {
+                throw new DownloadFailedException("Error making HTTP HEAD request.", ex);
+            } finally {
+                if (conn != null) {
+                    try {
+                        conn.disconnect();
+                    } finally {
+                        conn = null;
+                    }
                 }
             }
         }
@@ -208,16 +189,29 @@ public final class Downloader {
             if (proxyUrl != null) {
                 final int proxyPort = Settings.getInt(Settings.KEYS.PROXY_PORT);
                 final SocketAddress addr = new InetSocketAddress(proxyUrl, proxyPort);
+
+                final String username = Settings.getString(Settings.KEYS.PROXY_USERNAME);
+                final String password = Settings.getString(Settings.KEYS.PROXY_PASSWORD);
+                if (username != null && password != null) {
+                    final Authenticator auth = new Authenticator() {
+                        @Override
+                        public PasswordAuthentication getPasswordAuthentication() {
+                            if (getRequestorType().equals(RequestorType.PROXY)) {
+                                return new PasswordAuthentication(username, password.toCharArray());
+                            }
+                            return super.getPasswordAuthentication();
+                        }
+                    };
+                    Authenticator.setDefault(auth);
+                }
+
                 proxy = new Proxy(Proxy.Type.HTTP, addr);
                 conn = (HttpURLConnection) url.openConnection(proxy);
             } else {
                 conn = (HttpURLConnection) url.openConnection();
             }
-            //added a default timeout of 20000
-            //if (Settings.getString(Settings.KEYS.CONNECTION_TIMEOUT) != null) {
             final int timeout = Settings.getInt(Settings.KEYS.CONNECTION_TIMEOUT, 60000);
             conn.setConnectTimeout(timeout);
-            //}
         } catch (IOException ex) {
             if (conn != null) {
                 try {

dependency-check-core/src/main/java/org/owasp/dependencycheck/utils/ExtractionException.java

@@ -0,0 +1,69 @@
+/*
+ * This file is part of dependency-check-core.
+ *
+ * Dependency-check-core is free software: you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation, either version 3 of the License, or (at your option) any
+ * later version.
+ *
+ * Dependency-check-core is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+ * details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * dependency-check-core. If not, see http://www.gnu.org/licenses/.
+ *
+ * Copyright (c) 2013 Jeremy Long. All Rights Reserved.
+ */
+package org.owasp.dependencycheck.utils;
+
+import java.io.IOException;
+
+/**
+ * An exception used when a file is unable to be un-zipped.
+ *
+ * @author Jeremy Long (jeremy.long@owasp.org)
+ */
+public class ExtractionException extends IOException {
+
+    /**
+     * The serial version UID.
+     */
+    private static final long serialVersionUID = 1L;
+
+    /**
+     * Creates a new ExtractionException.
+     */
+    public ExtractionException() {
+        super();
+    }
+
+    /**
+     * Creates a new ExtractionException.
+     *
+     * @param msg a message for the exception.
+     */
+    public ExtractionException(String msg) {
+        super(msg);
+    }
+
+    /**
+     * Creates a new ExtractionException.
+     *
+     * @param ex the cause of the download failure.
+     */
+    public ExtractionException(Throwable ex) {
+        super(ex);
+    }
+
+    /**
+     * Creates a new ExtractionException.
+     *
+     * @param msg a message for the exception.
+     * @param ex the cause of the download failure.
+     */
+    public ExtractionException(String msg, Throwable ex) {
+        super(msg, ex);
+    }
+}

dependency-check-core/src/main/java/org/owasp/dependencycheck/utils/FileUtils.java

@@ -18,11 +18,20 @@
  */
 package org.owasp.dependencycheck.utils;
 
+import java.io.BufferedInputStream;
+import java.io.BufferedOutputStream;
 import java.io.File;
+import java.io.FileInputStream;
 import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
 import java.io.IOException;
 import java.io.UnsupportedEncodingException;
 import java.net.URLDecoder;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipInputStream;
+import org.owasp.dependencycheck.Engine;
 
 /**
  * A collection of utilities for processing information about files.
@@ -31,6 +40,11 @@ import java.net.URLDecoder;
  */
 public final class FileUtils {
 
+    /**
+     * The buffer size to use when extracting files from the archive.
+     */
+    private static final int BUFFER_SIZE = 4096;
+
     /**
      * Private constructor for a utility class.
      */
@@ -65,9 +79,15 @@ public final class FileUtils {
                 delete(c);
             }
         }
-        if (!file.delete()) {
+        if (!org.apache.commons.io.FileUtils.deleteQuietly(file)) {
             throw new FileNotFoundException("Failed to delete file: " + file);
         }
+        /* else {
+         //delete on exit was a bad idea. if for some reason the file can't be deleted
+         // this will cause a newly constructed file to be deleted and a subsequent run may fail.
+         // still not sure why a file fails to be deleted, but can be overwritten... odd.
+         file.deleteOnExit();
+         }*/
     }
 
     /**
@@ -114,4 +134,100 @@ public final class FileUtils {
         final File jarPath = new File(decodedPath);
         return jarPath.getParentFile();
     }
+
+    /**
+     * Extracts the contents of an archive into the specified directory.
+     *
+     * @param archive an archive file such as a WAR or EAR
+     * @param extractTo a directory to extract the contents to
+     * @throws ExtractionException thrown if an exception occurs while
+     * extracting the files
+     */
+    public static void extractFiles(File archive, File extractTo) throws ExtractionException {
+        extractFiles(archive, extractTo, null);
+    }
+
+    /**
+     * Extracts the contents of an archive into the specified directory. The
+     * files are only extracted if they are supported by the analyzers loaded
+     * into the specified engine. If the engine is specified as null then all
+     * files are extracted.
+     *
+     * @param archive an archive file such as a WAR or EAR
+     * @param extractTo a directory to extract the contents to
+     * @param engine the scanning engine
+     * @throws ExtractionException thrown if there is an error extracting the
+     * files
+     */
+    public static void extractFiles(File archive, File extractTo, Engine engine) throws ExtractionException {
+        if (archive == null || extractTo == null) {
+            return;
+        }
+
+        FileInputStream fis = null;
+        ZipInputStream zis = null;
+
+        try {
+            fis = new FileInputStream(archive);
+        } catch (FileNotFoundException ex) {
+            Logger.getLogger(FileUtils.class.getName()).log(Level.INFO, null, ex);
+            throw new ExtractionException("Archive file was not found.", ex);
+        }
+        zis = new ZipInputStream(new BufferedInputStream(fis));
+        ZipEntry entry;
+        try {
+            while ((entry = zis.getNextEntry()) != null) {
+                if (entry.isDirectory()) {
+                    final File d = new File(extractTo, entry.getName());
+                    if (!d.exists() && !d.mkdirs()) {
+                        final String msg = String.format("Unable to create '%s'.", d.getAbsolutePath());
+                        throw new ExtractionException(msg);
+                    }
+                } else {
+                    final File file = new File(extractTo, entry.getName());
+                    final String ext = getFileExtension(file.getName());
+                    if (engine == null || engine.supportsExtension(ext)) {
+                        BufferedOutputStream bos = null;
+                        FileOutputStream fos;
+                        try {
+                            fos = new FileOutputStream(file);
+                            bos = new BufferedOutputStream(fos, BUFFER_SIZE);
+                            int count;
+                            final byte data[] = new byte[BUFFER_SIZE];
+                            while ((count = zis.read(data, 0, BUFFER_SIZE)) != -1) {
+                                bos.write(data, 0, count);
+                            }
+                            bos.flush();
+                        } catch (FileNotFoundException ex) {
+                            Logger.getLogger(FileUtils.class.getName()).log(Level.FINE, null, ex);
+                            final String msg = String.format("Unable to find file '%s'.", file.getName());
+                            throw new ExtractionException(msg, ex);
+                        } catch (IOException ex) {
+                            Logger.getLogger(FileUtils.class.getName()).log(Level.FINE, null, ex);
+                            final String msg = String.format("IO Exception while parsing file '%s'.", file.getName());
+                            throw new ExtractionException(msg, ex);
+                        } finally {
+                            if (bos != null) {
+                                try {
+                                    bos.close();
+                                } catch (IOException ex) {
+                                    Logger.getLogger(FileUtils.class.getName()).log(Level.FINEST, null, ex);
+                                }
+                            }
+                        }
+                    }
+                }
+            }
+        } catch (IOException ex) {
+            final String msg = String.format("Exception reading archive '%s'.", archive.getName());
+            Logger.getLogger(FileUtils.class.getName()).log(Level.FINE, msg, ex);
+            throw new ExtractionException(msg, ex);
+        } finally {
+            try {
+                zis.close();
+            } catch (IOException ex) {
+                Logger.getLogger(FileUtils.class.getName()).log(Level.FINEST, null, ex);
+            }
+        }
+    }
 }

dependency-check-core/src/main/java/org/owasp/dependencycheck/utils/LogFilter.java

@@ -0,0 +1,44 @@
+/*
+ * This file is part of dependency-check-core.
+ *
+ * Dependency-check-core is free software: you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation, either version 3 of the License, or (at your option) any
+ * later version.
+ *
+ * Dependency-check-core is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+ * details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * dependency-check-core. If not, see http://www.gnu.org/licenses/.
+ *
+ * Copyright (c) 2013 Jeremy Long. All Rights Reserved.
+ */
+package org.owasp.dependencycheck.utils;
+
+import java.util.logging.Filter;
+import java.util.logging.LogRecord;
+
+/**
+ * A simple log filter to limit the entries written to the verbose log file. The
+ * verbose log file uses the root logger as I couldn't get anything else to
+ * work; as such, this filter limits the log entries to specific classes.
+ *
+ * @author Jeremy Long (jeremy.long@owasp.org)
+ */
+public class LogFilter implements Filter {
+
+    /**
+     * Determines if the record should be logged.
+     *
+     * @param record a log record to examine
+     * @return true if the record should be logged, otherwise false
+     */
+    @Override
+    public boolean isLoggable(LogRecord record) {
+        final String name = record.getSourceClassName();
+        return name.startsWith("org.owasp.dependencycheck") && !name.contains("generated") && !name.contains("VelocityLoggerRedirect");
+    }
+}

dependency-check-core/src/main/java/org/owasp/dependencycheck/utils/LogUtils.java

@@ -0,0 +1,89 @@
+/*
+ * This file is part of dependency-check-core.
+ *
+ * Dependency-check-core is free software: you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation, either version 3 of the License, or (at your option) any
+ * later version.
+ *
+ * Dependency-check-core is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+ * details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * dependency-check-core. If not, see http://www.gnu.org/licenses/.
+ *
+ * Copyright (c) 2013 Jeremy Long. All Rights Reserved.
+ */
+package org.owasp.dependencycheck.utils;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.logging.FileHandler;
+import java.util.logging.Level;
+import java.util.logging.LogManager;
+import java.util.logging.Logger;
+import java.util.logging.SimpleFormatter;
+
+/**
+ * A utility class to aide in the setup of the logging mechanism.
+ *
+ * @author Jeremy Long (jeremy.long@owasp.org)
+ */
+public final class LogUtils {
+
+    /**
+     * Private constructor for a utility class.
+     */
+    private LogUtils() {
+    }
+
+    /**
+     * Configures the logger for use by the application.
+     *
+     * @param in the input stream to read the log settings from
+     * @param verboseLogFile the file path for the verbose log
+     */
+    public static void prepareLogger(InputStream in, String verboseLogFile) {
+        try {
+            LogManager.getLogManager().reset();
+            LogManager.getLogManager().readConfiguration(in);
+            if (verboseLogFile != null && !verboseLogFile.isEmpty()) {
+                verboseLoggingEnabled = true;
+                final Logger logger = Logger.getLogger("");
+                final FileHandler handler = new FileHandler(verboseLogFile, true);
+                handler.setFormatter(new SimpleFormatter());
+                handler.setLevel(Level.FINE);
+                handler.setFilter(new LogFilter());
+                logger.addHandler(handler);
+                logger.setLevel(Level.FINE);
+            }
+        } catch (IOException ex) {
+            Logger.getLogger(LogUtils.class.getName()).log(Level.FINE, "IO Error preparing the logger", ex);
+        } catch (SecurityException ex) {
+            Logger.getLogger(LogUtils.class.getName()).log(Level.FINE, "Error preparing the logger", ex);
+        } finally {
+            if (in != null) {
+                try {
+                    in.close();
+                } catch (Exception ex) {
+                    Logger.getLogger(LogUtils.class.getName()).log(Level.FINEST, "Error closing resource stream", ex);
+                }
+            }
+        }
+    }
+    /**
+     * Whether or not verbose logging is enabled.
+     */
+    private static boolean verboseLoggingEnabled = false;
+
+    /**
+     * Get the value of verboseLoggingEnabled.
+     *
+     * @return the value of verboseLoggingEnabled
+     */
+    public static boolean isVerboseLoggingEnabled() {
+        return verboseLoggingEnabled;
+    }
+}

dependency-check-core/src/main/java/org/owasp/dependencycheck/utils/Settings.java

@@ -68,23 +68,15 @@ public final class Settings {
          */
         public static final String DATA_DIRECTORY = "data.directory";
         /**
-         * The properties key for the path where the CPE Lucene Index will be
-         * stored.
+         * The location of the batch update URL. This is a zip file that
+         * contains the contents of the data directory.
          */
-        public static final String CPE_DATA_DIRECTORY = "data.cpe";
+        public static final String BATCH_UPDATE_URL = "batch.update.url";
         /**
          * The properties key for the path where the CVE H2 database will be
          * stored.
          */
         public static final String CVE_DATA_DIRECTORY = "data.cve";
-        /**
-         * The properties key for the URL to the CPE.
-         */
-        public static final String CPE_URL = "cpe.url";
-        /**
-         * The properties key for the URL to the CPE.
-         */
-        public static final String CPE_META_URL = "cpe.meta.url";
         /**
          * The properties key for the URL to retrieve the "meta" data from about
          * the CVE entries.
@@ -128,6 +120,14 @@ public final class Settings {
          * value.
          */
         public static final String PROXY_PORT = "proxy.port";
+        /**
+         * The properties key for the proxy username.
+         */
+        public static final String PROXY_USERNAME = "proxy.username";
+        /**
+         * The properties key for the proxy password.
+         */
+        public static final String PROXY_PASSWORD = "proxy.password";
         /**
          * The properties key for the connection timeout.
          */
@@ -136,6 +136,14 @@ public final class Settings {
          * The location of the temporary directory.
          */
         public static final String TEMP_DIRECTORY = "temp.directory";
+        /**
+         * The maximum number of threads to allocate when downloading files.
+         */
+        public static final String MAX_DOWNLOAD_THREAD_POOL_SIZE = "max.download.threads";
+        /**
+         * The key for a list of suppression files.
+         */
+        public static final String SUPPRESSION_FILE = "suppression.file";
     }
     /**
      * The properties file location.
@@ -198,6 +206,23 @@ public final class Settings {
         }
     }
 
+    /**
+     * Merges a new properties file into the current properties. This method
+     * allows for the loading of a user provided properties file.<br/><br/>
+     * Note: even if using this method - system properties will be loaded before
+     * properties loaded from files.
+     *
+     * @param filePath the path to the properties file to merge.
+     * @throws FileNotFoundException is thrown when the filePath points to a
+     * non-existent file
+     * @throws IOException is thrown when there is an exception loading/merging
+     * the properties
+     */
+    public static void mergeProperties(File filePath) throws FileNotFoundException, IOException {
+        final FileInputStream fis = new FileInputStream(filePath);
+        mergeProperties(fis);
+    }
+
     /**
      * Merges a new properties file into the current properties. This method
      * allows for the loading of a user provided properties file.<br/><br/>
@@ -236,16 +261,14 @@ public final class Settings {
      * before the values in the contained configuration file.
      *
      * @param key the key to lookup within the properties file
-     * @param defaultValue the default value for the requested property
-     * @return the property from the properties file as a File object
+     * @return the property from the properties file converted to a File object
      */
-    public static File getFile(String key, String defaultValue) {
-        final String baseDir = getString(Settings.KEYS.DATA_DIRECTORY);
-        final String str = getString(key, defaultValue);
-        if (baseDir != null) {
-            return new File(baseDir, str);
+    public static File getFile(String key) {
+        final String file = getString(key);
+        if (file == null) {
+            return null;
         }
-        return new File(str);
+        return new File(file);
     }
 
     /**
@@ -254,22 +277,29 @@ public final class Settings {
      * argument - this method will return the value from the system properties
      * before the values in the contained configuration file.
      *
-     * This method will also replace a leading "[JAR]\" sequence with the path
-     * to the folder containing the JAR file containing this class.
+     * This method will check the configured base directory and will use this as
+     * the base of the file path. Additionally, if the base directory begins
+     * with a leading "[JAR]\" sequence with the path to the folder containing
+     * the JAR file containing this class.
      *
      * @param key the key to lookup within the properties file
      * @return the property from the properties file converted to a File object
-     * @throws IOException thrown if the file path to the JAR cannot be found
      */
-    public static File getFile(String key) throws IOException {
+    public static File getDataFile(String key) {
         final String file = getString(key);
         final String baseDir = getString(Settings.KEYS.DATA_DIRECTORY);
         if (baseDir != null) {
             if (baseDir.startsWith("[JAR]/")) {
                 final File jarPath = getJarPath();
-                final File newBase = new File(jarPath.getCanonicalPath(), baseDir.substring(6));
+                final File newBase = new File(jarPath, baseDir.substring(6));
+                if (Settings.KEYS.DATA_DIRECTORY.equals(key)) {
+                    return newBase;
+                }
                 return new File(newBase, file);
             }
+            if (Settings.KEYS.DATA_DIRECTORY.equals(key)) {
+                return new File(baseDir);
+            }
             return new File(baseDir, file);
         }
         return new File(file);
@@ -313,6 +343,15 @@ public final class Settings {
         return str;
     }
 
+    /**
+     * Returns the temporary directory.
+     *
+     * @return the temporary directory
+     */
+    public static File getTempDirectory() {
+        return new File(Settings.getString(Settings.KEYS.TEMP_DIRECTORY, System.getProperty("java.io.tmpdir")));
+    }
+
     /**
      * Returns a value from the properties file. If the value was specified as a
      * system property or passed in via the -Dprop=value argument - this method
@@ -326,6 +365,16 @@ public final class Settings {
         return System.getProperty(key, INSTANCE.props.getProperty(key));
     }
 
+    /**
+     * Removes a property from the local properties collection. This is mainly
+     * used in test cases.
+     *
+     * @param key the property key to remove
+     */
+    public static void removeProperty(String key) {
+        INSTANCE.props.remove(key);
+    }
+
     /**
      * Returns an int value from the properties file. If the value was specified
      * as a system property or passed in via the -Dprop=value argument - this

dependency-check-core/src/main/java/org/owasp/dependencycheck/utils/UrlStringUtils.java

@@ -21,6 +21,8 @@ package org.owasp.dependencycheck.utils;
 import java.net.MalformedURLException;
 import java.net.URL;
 import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashSet;
 import java.util.List;
 import java.util.regex.Pattern;
 
@@ -64,6 +66,12 @@ public final class UrlStringUtils {
     public static boolean isUrl(String text) {
         return IS_URL_TEST.matcher(text).matches();
     }
+    /**
+     * A listing of domain parts that shold not be used as evidence. Yes, this
+     * is an incomplete list.
+     */
+    private static final HashSet<String> IGNORE_LIST = new HashSet<String>(
+            Arrays.asList("www", "com", "org", "gov", "info", "name", "net", "pro", "tel", "mobi", "xxx"));
 
     /**
      * <p>Takes a URL, in String format, and adds the important parts of the URL
@@ -84,7 +92,7 @@ public final class UrlStringUtils {
         //add the domain except www and the tld.
         for (int i = 0; i < domain.length - 1; i++) {
             final String sub = domain[i];
-            if (!"www".equalsIgnoreCase(sub)) {
+            if (!IGNORE_LIST.contains(sub.toLowerCase())) {
                 importantParts.add(sub);
             }
         }

dependency-check-core/src/main/resources/META-INF/licenses/commons-compress/LICENSE.txt

@@ -0,0 +1,202 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

dependency-check-core/src/main/resources/META-INF/services/org.owasp.dependencycheck.analyzer.Analyzer

@@ -1,8 +1,10 @@
 org.owasp.dependencycheck.analyzer.ArchiveAnalyzer
-org.owasp.dependencycheck.analyzer.JarAnalyzer
 org.owasp.dependencycheck.analyzer.FileNameAnalyzer
+org.owasp.dependencycheck.analyzer.JarAnalyzer
 org.owasp.dependencycheck.analyzer.HintAnalyzer
-org.owasp.dependencycheck.analyzer.DependencyBundlingAnalyzer
+org.owasp.dependencycheck.analyzer.CPEAnalyzer
 org.owasp.dependencycheck.analyzer.FalsePositiveAnalyzer
-org.owasp.dependencycheck.data.cpe.CPEAnalyzer
-org.owasp.dependencycheck.data.nvdcve.NvdCveAnalyzer
+org.owasp.dependencycheck.analyzer.CpeSuppressionAnalyzer
+org.owasp.dependencycheck.analyzer.DependencyBundlingAnalyzer
+org.owasp.dependencycheck.analyzer.NvdCveAnalyzer
+org.owasp.dependencycheck.analyzer.VulnerabilitySuppressionAnalyzer

dependency-check-core/src/main/resources/META-INF/services/org.owasp.dependencycheck.data.CachedWebDataSource

@@ -1 +1 @@
-org.owasp.dependencycheck.data.nvdcve.xml.DatabaseUpdater
+org.owasp.dependencycheck.data.update.DatabaseUpdater

dependency-check-core/src/main/resources/data/initialize.sql

@@ -8,6 +8,7 @@ DROP TABLE IF EXISTS vulnerability;
 DROP TABLE IF EXISTS reference;
 DROP TABLE IF EXISTS cpeEntry;
 DROP TABLE IF EXISTS software;
+DROP TABLE IF EXISTS settings;
 
 CREATE TABLE settings (id varchar(50) PRIMARY KEY, value varchar(200));
 

dependency-check-core/src/main/resources/dependencycheck.properties

@@ -1,14 +1,13 @@
 application.name=${pom.name}
 application.version=${pom.version}
 autoupdate=true
+max.download.threads=3
 
 #temp.directory defaults to System.getProperty("java.io.tmpdir")
 #temp.directory=[path to temp directory]
 
 # the path to the data directory; if tis
 data.directory=[JAR]/data
-# the path to the lucene index to store the cpe data
-data.cpe=cpe
 # the path to the h2 database to store the nvd cve data
 data.cve=cve
 
@@ -23,6 +22,15 @@ cpe.meta.url=http://static.nvd.nist.gov/feeds/xml/cpe/dictionary/official-cpe-di
 # holds 8 days of updates, we are using 7 just to be safe.
 cve.url.modified.validfordays=7
 
+# The location of the zipped CVE H2 database and CPE Lucene index. If specified and
+# a full download of data is required this URL will be used and the data extracted
+# into the specified "data" directory. Additionally, after pulling the data the
+# system will attempt to update the modified. Thus, if one were maintaining an
+# internal copy of the data one would not need to update it nightly.
+# If the 'modified' URLs below for the CVE are removed and a batch url is provided
+# then if an update is required, the entre zip file will be downloaded.
+#batch.update.url=file:///C:/path/to/data.zip
+
 # the path to the modified nvd cve xml file.
 cve.url-1.2.modified=http://nvd.nist.gov/download/nvdcve-modified.xml
 cve.url-2.0.modified=http://static.nvd.nist.gov/feeds/xml/cve/nvdcve-2.0-modified.xml

dependency-check-core/src/main/resources/schema/DependencyCheck.xsd

@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="utf-8"?>
-<xs:schema id="analysis" xmlns="https://www.owasp.org/index.php/OWASP_Dependency_Check" xmlns:xs="http://www.w3.org/2001/XMLSchema">
+<xs:schema id="analysis" xmlns:xs="http://www.w3.org/2001/XMLSchema" elementFormDefault="qualified" targetNamespace="https://www.owasp.org/index.php/OWASP_Dependency_Check">
     <xs:element name="analysis">
         <xs:complexType>
             <xs:sequence minOccurs="0" maxOccurs="unbounded">

dependency-check-core/src/main/resources/schema/suppression.xsd

@@ -0,0 +1,57 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<xs:schema id="suppressions"
+           xmlns:xs="http://www.w3.org/2001/XMLSchema"
+           elementFormDefault="qualified"
+           targetNamespace="https://www.owasp.org/index.php/OWASP_Dependency_Check_Suppression"
+           xmlns:dc="https://www.owasp.org/index.php/OWASP_Dependency_Check_Suppression">
+
+    <xs:complexType name="regexStringType">
+        <xs:simpleContent>
+            <xs:extension base="xs:string">
+                <xs:attribute name="regex" use="optional" type="xs:boolean" default="false"/>
+                <xs:attribute name="caseSensitive" use="optional" type="xs:boolean" default="false"/>
+            </xs:extension>
+        </xs:simpleContent>
+    </xs:complexType>
+    <xs:simpleType name="cvssScoreType">
+        <xs:restriction base="xs:decimal">
+            <xs:minInclusive value="0"/>
+            <xs:maxInclusive value="10"/>
+        </xs:restriction>
+    </xs:simpleType>
+    <xs:simpleType name="cveType">
+        <xs:restriction base="xs:string">
+            <xs:pattern value="CVE\-\d\d\d\d\-\d+"/>
+        </xs:restriction>
+    </xs:simpleType>
+    <xs:simpleType name="sha1Type">
+        <xs:restriction base="xs:string">
+            <xs:pattern value="[a-fA-F0-9]{40}"/>
+        </xs:restriction>
+    </xs:simpleType>
+    <xs:element name="suppressions">
+        <xs:complexType>
+            <xs:sequence minOccurs="0" maxOccurs="unbounded">
+                <xs:element name="suppress">
+                    <xs:complexType>
+                        <xs:sequence minOccurs="1" maxOccurs="1">
+                            <xs:sequence minOccurs="0" maxOccurs="1">
+                                <xs:element name="notes" type="xs:string"/>
+                            </xs:sequence>
+                            <xs:choice minOccurs="0" maxOccurs="1">
+                                <xs:element name="filePath" type="dc:regexStringType"/>
+                                <xs:element name="sha1" type="dc:sha1Type"/>
+                            </xs:choice>
+                            <xs:choice minOccurs="0" maxOccurs="unbounded">
+                                <xs:element name="cpe" type="dc:regexStringType"/>
+                                <xs:element name="cve" type="dc:cveType"/>
+                                <xs:element name="cwe" type="xs:positiveInteger"/>
+                                <xs:element name="cvssBelow" type="dc:cvssScoreType"/>
+                            </xs:choice>
+                        </xs:sequence>
+                    </xs:complexType>
+                </xs:element>
+            </xs:sequence>
+        </xs:complexType>
+    </xs:element>
+</xs:schema>

dependency-check-core/src/main/resources/templates/HtmlReport.vsl

@@ -51,9 +51,132 @@ Copyright (c) 2012 Jeremy Long. All Rights Reserved.
 
                 });
             });
+
+            $(function(){
+                $("#modal-background, #modal-close").click(function () {
+                    $("#modal-content,#modal-background").toggleClass("active");
+                });
+                $("#modal-text").bind('copy cut', function() {
+                    setTimeout('$("#modal-content,#modal-background").toggleClass("active");',100);
+                });
+                $("#modal-add-header").click(function () {
+                    xml = '<?xml version="1.0" encoding="UTF-8"?>\n<suppressions xmlns="https://www.owasp.org/index.php/OWASP_Dependency_Check_Suppression">\n   ';
+                    xml += $("#modal-text").text().replace(/\n/g,'\n   ');
+                    xml += '\n</suppressions>';
+                    $("#modal-text").text(xml).focus().select();
+                });
+            });
+
+            function copyText(name, sha1, type, val) {
+                xml =  '<suppress>\n';
+                xml += '   <notes><!'+'[CDATA[\n   file name: ' + name + '\n   ]]'+'></notes>\n';
+                xml += '   <sha1>' + sha1 + '</sha1>\n';
+                xml += '   <'+type+'>' + val + '</'+type+'>\n';
+                xml += '</suppress>';
+                $("#modal-text").text(xml);
+                $("#modal-content,#modal-background").toggleClass("active");
+                $("#modal-text").focus();
+                $("#modal-text").select();
+            }
+
+            function toggleVuln() {
+                $(".notvulnerable").toggle();
+            }
         </script>
         <style type="text/css">
+            #modal-background {
+                display: none;
+                position: fixed;
+                top: 0;
+                left: 0;
+                width: 100%;
+                height: 100%;
+                background-color: white;
+                opacity: .50;
+                -webkit-opacity: .5;
+                -moz-opacity: .5;
+                filter: alpha(opacity=50);
+                z-index: 1000;
+            }
 
+            #modal-content {
+                background-color: white;
+                border-radius: 10px;
+                -webkit-border-radius: 10px;
+                -moz-border-radius: 10px;
+                box-shadow: 0 0 20px 0 #222;
+                -webkit-box-shadow: 0 0 20px 0 #222;
+                -moz-box-shadow: 0 0 20px 0 #222;
+                display: none;
+                height: 240px;
+                left: 50%;
+                margin: -120px 0 0 -160px;
+                padding: 10px;
+                position: absolute;
+                top: 50%;
+                z-index: 1000;
+            }
+
+            #modal-background.active, #modal-content.active {
+                display: block;
+            }
+            #modal-text {
+                border: 0;
+                overflow: hidden
+            }
+            #modal-text:focus {
+                outline: none;
+            }
+            .copybutton {
+                padding:1px;
+                background-color: #eeeeee;
+                border: 1px solid #555555;
+                color:#555555;
+                text-decoration:none;
+                -moz-border-radius: 3px;
+                -webkit-border-radius: 3px;
+                -khtml-border-radius: 3px;
+                -o-border-radius: 3px;
+                border-radius: 3px;
+            }
+            .copybutton:hover {
+                padding:1px;
+                background-color: #dddddd;
+                border: 1px solid #444444;
+                color:#444444;
+                text-decoration:none;
+                -moz-border-radius: 3px;
+                -webkit-border-radius: 3px;
+                -khtml-border-radius: 3px;
+                -o-border-radius: 3px;
+                border-radius: 3px;
+            }
+            .modal-button {
+                padding:1px;
+                float:right;
+                background-color: #eeeeee;
+                border: 1px solid #555555;
+                color:#555555;
+                text-decoration:none;
+                -moz-border-radius: 3px;
+                -webkit-border-radius: 3px;
+                -khtml-border-radius: 3px;
+                -o-border-radius: 3px;
+                border-radius: 3px;
+            }
+            .modal-button:hover {
+                padding:1px;
+                float:right;
+                background-color: #dddddd;
+                border: 1px solid #333333;
+                color:#333333;
+                text-decoration:none;
+                -moz-border-radius: 3px;
+                -webkit-border-radius: 3px;
+                -khtml-border-radius: 3px;
+                -o-border-radius: 3px;
+                border-radius: 3px;
+            }
             .rounded-corners {
                 -moz-border-radius: 20px;
                 -webkit-border-radius: 20px;
@@ -275,9 +398,24 @@ Copyright (c) 2012 Jeremy Long. All Rights Reserved.
                 margin-top:3px;
                 margin-bottom:3px;
             }
+            .vulnerable {
+            	color: #f00;
+            }
+            .vulnerable li {
+            	color: #000;
+            }
+            .notvulnerable {
+                display:none;
+            }
         </style>
     </head>
     <body>
+        <div id="modal-background"></div>
+        <div id="modal-content">
+            <div>Press CTR-C to copy XML<button id="modal-add-header" class="modal-button">Complete XML Doc</button></div>
+                <textarea id="modal-text" cols="50" rows="10"></textarea><br/>
+                <button id="modal-close" class="modal-button">Close</button>
+        </div>
         <div class="wrapper">
             <h1>Dependency Report</h1>
 ]]#
@@ -292,13 +430,14 @@ Copyright (c) 2012 Jeremy Long. All Rights Reserved.
                         #set($vulnCount=$vulnCount+1)
                     #end
                 #end
-                Dependencies Scanned:&nbsp;$depCount<br/>
+                Dependencies Scanned:&nbsp;$depCount&nbsp;(<a href="#" onclick="toggleVuln()">show all</a>)<br/>
                 Vulnerable Dependencies:&nbsp;$vulnCount<br/><br/>
-                <div class="indent">
+                <ul class="indent">
                 #set($lnkcnt=0)
                 #foreach($dependency in $dependencies)
                     #set($lnkcnt=$lnkcnt+1)
-                    <a href="#l${lnkcnt}_$esc.html($esc.url($dependency.Sha1sum))">$esc.html($dependency.FileName)</a>#if($dependency.getVulnerabilities().size()>0)&nbsp;<b style="color:#ff0000;">&#8226;</b>#end<br/>
+                    <li class="#if($dependency.getVulnerabilities().size()==0)notvulnerable#else vulnerable#end">
+                    <a href="#l${lnkcnt}_$esc.html($esc.url($dependency.Sha1sum))">$esc.html($dependency.FileName)</a>
                     #if($dependency.getRelatedDependencies().size()>0)
                         <ul>
                         #foreach($related in $dependency.getRelatedDependencies())
@@ -306,15 +445,16 @@ Copyright (c) 2012 Jeremy Long. All Rights Reserved.
                         #end
                         </ul>
                     #end
+                    </li>
                 #end
-                </div>
+                </ul>
                 <h2>Dependencies</h2>
                 #set($lnkcnt=0)
                 #set($cnt=0)
                 #foreach($dependency in $dependencies)
                     #set($lnkcnt=$lnkcnt+1)
-                    <h3 class="subsectionheader standardsubsection"><a name="l${lnkcnt}_$esc.html($dependency.Sha1sum)"></a>$esc.html($dependency.FileName)</h3>
-                    <div class="subsectioncontent">
+                    <h3 class="subsectionheader standardsubsection#if($dependency.getVulnerabilities().size()==0) notvulnerable#end"><a name="l${lnkcnt}_$esc.html($dependency.Sha1sum)"></a>$esc.html($dependency.FileName)</h3>
+                    <div class="subsectioncontent#if($dependency.getVulnerabilities().size()==0) notvulnerable#end">
                         #if ($dependency.description)
                             <p><b>Description:</b>&nbsp;$esc.html($dependency.description)<br/></p>
                         #end
@@ -408,6 +548,8 @@ Copyright (c) 2012 Jeremy Long. All Rights Reserved.
                                 #else
                                     <li><b>$esc.html($id.type):</b>&nbsp;$esc.html($id.value)
                                 #end
+                                    ##yes, we are HTML Encoding into JavaScript... the escape utils don't have a JS Encode and I haven't written one yet
+                                    &nbsp;&nbsp;<button class="copybutton" onclick="copyText('$esc.html($dependency.FileName)', '$esc.html($dependency.Sha1sum)', 'cpe', '$esc.html($id.value)')">suppress</button>
                                 #if( $id.description )
                                     <br/>$esc.html($id.description)
                                 #end
@@ -421,7 +563,7 @@ Copyright (c) 2012 Jeremy Long. All Rights Reserved.
                     <h4 id="header$cnt" class="subsectionheader expandable collaspablesubsection white">Published Vulnerabilities</h4>
                     <div id="content$cnt" class="subsectioncontent standardsubsection">
                         #foreach($vuln in $dependency.getVulnerabilities())
-                            <p><b><a target="_blank" href="http://web.nvd.nist.gov/view/vuln/detail?vulnId=$esc.url($vuln.name)">$esc.html($vuln.name)</a></b></p>
+                            <p><b><a target="_blank" href="http://web.nvd.nist.gov/view/vuln/detail?vulnId=$esc.url($vuln.name)">$esc.html($vuln.name)</a></b>&nbsp;&nbsp;<button class="copybutton" onclick="copyText('$esc.html($dependency.FileName)', '$esc.html($dependency.Sha1sum)', 'cve', '$esc.html($vuln.name)')">suppress</button></p>
                             <p>Severity:
                             #if ($vuln.cvssScore<4.0)
                                 Low
@@ -443,7 +585,7 @@ Copyright (c) 2012 Jeremy Long. All Rights Reserved.
                                     </ul>
                                 #end
                             </p>
-                            <p>Vulnerably Software:<ul>
+                            <p>Vulnerable Software &amp; Versions:<ul>
                             #foreach($vs in $vuln.getVulnerableSoftware())
                                 <li><a target="_blank" href="http://web.nvd.nist.gov/view/vuln/search-results?cpe=$esc.url($vs.name)">$esc.html($vs.name)</a> #if($vs.hasPreviousVersion()) and all previous versions.#end</li>
                             #end

dependency-check-core/src/main/resources/templates/XmlReport.vsl

@@ -21,30 +21,30 @@ Copyright (c) 2012 Jeremy Long. All Rights Reserved.
 *#<?xml version="1.0"?>
 <analysis xmlns="https://www.owasp.org/index.php/OWASP_Dependency_Check">
     <projectInfo>
-        <name>$esc.html($applicationName)</name>
+        <name>$esc.xml($applicationName)</name>
         <reportDate>$date</reportDate>
         <credits>This report contains data retrieved from the National Vulnerability Database: http://nvd.nist.gov</credits>
     </projectInfo>
     <dependencies>
 #foreach($dependency in $dependencies)
         <dependency>
-            <fileName>$esc.html($dependency.FileName)</fileName>
-            <filePath>$esc.html($dependency.FilePath)</filePath>
-            <md5>$esc.html($dependency.Md5sum)</md5>
-            <sha1>$esc.html($dependency.Sha1sum)</sha1>
+            <fileName>$esc.xml($dependency.FileName)</fileName>
+            <filePath>$esc.xml($dependency.FilePath)</filePath>
+            <md5>$esc.xml($dependency.Md5sum)</md5>
+            <sha1>$esc.xml($dependency.Sha1sum)</sha1>
 #if ($dependency.description)
-            <description>$esc.html($dependency.description)</description>
+            <description>$esc.xml($dependency.description)</description>
 #end
 #if ($dependency.license)
-            <license>$esc.html($dependency.license)</license>
+            <license>$esc.xml($dependency.license)</license>
 #end
 #if ($dependency.getRelatedDependencies().size()>0)
             <relatedDependencies>
 #foreach($related in $dependency.getRelatedDependencies())
                 <relatedDependency>
-                    <filePath>$esc.html($related.FilePath)</filePath>
-                    <sha1>$esc.html($related.Sha1sum)</sha1>
-                    <md5>$esc.html($related.Md5sum)</md5>
+                    <filePath>$esc.xml($related.FilePath)</filePath>
+                    <sha1>$esc.xml($related.Sha1sum)</sha1>
+                    <md5>$esc.xml($related.Md5sum)</md5>
                 </relatedDependency>
 #end
             </relatedDependencies>
@@ -53,21 +53,21 @@ Copyright (c) 2012 Jeremy Long. All Rights Reserved.
             <analysisExceptions>
 #foreach($ex in $dependency.analysisExceptions)
                 <exception>
-                    <message>$esc.html($ex.message)</message>
+                    <message>$esc.xml($ex.message)</message>
 #if ( $ex.stackTrace )
                     <stackTrace>
 #foreach ($st in $ex.stackTrace)
-                        <trace>$esc.html($st)</trace>
+                        <trace>$esc.xml($st)</trace>
 #end
                     </stackTrace>
 #end
 #if ( $ex.cause )
                     <innerException>
-                        <message>$esc.html($ex.cause.message)</message>
+                        <message>$esc.xml($ex.cause.message)</message>
 #if ( $ex.cause.stackTrace )
                         <stackTrace>
 #foreach ($st in $ex.cause.stackTrace)
-                            <trace>$esc.html($st)</trace>
+                            <trace>$esc.xml($st)</trace>
 #end
                         </stackTrace>
 #end
@@ -80,22 +80,22 @@ Copyright (c) 2012 Jeremy Long. All Rights Reserved.
             <evidenceCollected>
 #foreach($evidence in $dependency.getEvidenceUsed())
                 <evidence>
-                    <source>$esc.html($evidence.getSource())</source>
-                    <name>$esc.html($evidence.getName())</name>
-                    <value>$esc.html($evidence.getValue().trim())</value>
+                    <source>$esc.xml($evidence.getSource())</source>
+                    <name>$esc.xml($evidence.getName())</name>
+                    <value>$esc.xml($evidence.getValue().trim())</value>
                 </evidence>
 #end
             </evidenceCollected>
 #if($dependency.getIdentifiers().size()>0)
             <identifiers>
 #foreach($id in $dependency.getIdentifiers())
-                <identifier type="$esc.html($id.type)">
-                    <name>$esc.html($id.value)</name>
+                <identifier type="$esc.xml($id.type)">
+                    <name>($id.value)</name>
 #if( $id.url )
-                    <url>$esc.html($id.url)</url>
+                    <url>$esc.xml($id.url)</url>
 #end
 #if( $id.description )
-                    <description>$esc.html($id.description)</description>
+                    <description>$esc.xml($id.description)</description>
 #end
                 </identifier>
 #end
@@ -105,7 +105,7 @@ Copyright (c) 2012 Jeremy Long. All Rights Reserved.
             <vulnerabilities>
 #foreach($vuln in $dependency.getVulnerabilities())
                 <vulnerability>
-                    <name>$esc.html($vuln.name)</name>
+                    <name>$esc.xml($vuln.name)</name>
                     <cvssScore>$vuln.cvssScore</cvssScore>
 #if ($vuln.cvssScore<4.0)
                     <severity>Low</severity>
@@ -115,21 +115,21 @@ Copyright (c) 2012 Jeremy Long. All Rights Reserved.
                     <severity>Medium</severity>
 #end
 #if ($vuln.cwe)
-                    <cwe>$esc.html($vuln.cwe)</cwe>
+                    <cwe>$esc.xml($vuln.cwe)</cwe>
 #end
-                    <description>$esc.html($vuln.description)</description>
+                    <description>$esc.xml($vuln.description)</description>
                     <references>
 #foreach($ref in $vuln.getReferences())
                         <reference>
-                            <source>$esc.html($ref.source)</source>
-                            <url>$esc.html($ref.url)</url>
-                            <name>$esc.html($ref.name)</name>
+                            <source>$esc.xml($ref.source)</source>
+                            <url>$esc.xml($ref.url)</url>
+                            <name>$esc.xml($ref.name)</name>
                         </reference>
 #end
                     </references>
                     <vulnerableSoftware>
 #foreach($vs in $vuln.getVulnerableSoftware())
-                        <software#if($vs.hasPreviousVersion()) allPreviousVersion="true"#end>$esc.html($vs.name)</software>
+                        <software#if($vs.hasPreviousVersion()) allPreviousVersion="true"#end>$esc.xml($vs.name)</software>
 #end
                     </vulnerableSoftware>
                 </vulnerability>

dependency-check-core/src/site/markdown/index.md

@@ -13,4 +13,4 @@ The engine is currently exposed via:
 - [Command Line Tool](../dependency-check-cli/installation.html)
 - [Maven Plugin](../dependency-check-maven/usage.html)
 - [Ant Task](../dependency-check-ant/installation.html)
-- Jenkins Plugin
+- [Jenkins Plugin](../dependency-check-jenkins/index.html)

dependency-check-core/src/test/java/org/owasp/dependencycheck/EngineIntegrationTest.java

@@ -44,7 +44,6 @@ public class EngineIntegrationTest {
     @Before
     public void setUp() throws Exception {
         org.owasp.dependencycheck.data.nvdcve.BaseDBTestCase.ensureDBExists();
-        org.owasp.dependencycheck.data.cpe.BaseIndexTestCase.ensureIndexExists();
     }
 
     @After
@@ -58,13 +57,13 @@ public class EngineIntegrationTest {
      */
     @Test
     public void testScan() throws Exception {
-        String path = "target/test-classes";
+        String testClasses = "target/test-classes";
         Engine instance = new Engine();
-        instance.scan(path);
+        instance.scan(testClasses);
         assertTrue(instance.getDependencies().size() > 0);
         instance.analyzeDependencies();
         ReportGenerator rg = new ReportGenerator("DependencyCheck",
                 instance.getDependencies(), instance.getAnalyzers());
-        rg.generateReports("./target/", "HTML");
+        rg.generateReports("./target/", "ALL");
     }
 }

dependency-check-core/src/test/java/org/owasp/dependencycheck/analyzer/ArchiveAnalyzerTest.java

@@ -28,6 +28,7 @@ import org.junit.BeforeClass;
 import org.junit.Test;
 import static org.junit.Assert.*;
 import org.owasp.dependencycheck.Engine;
+import org.owasp.dependencycheck.data.cpe.BaseIndexTestCase;
 import org.owasp.dependencycheck.dependency.Dependency;
 import org.owasp.dependencycheck.utils.Settings;
 
@@ -35,7 +36,7 @@ import org.owasp.dependencycheck.utils.Settings;
  *
  * @author Jeremy Long (jeremy.long@owasp.org)
  */
-public class ArchiveAnalyzerTest {
+public class ArchiveAnalyzerTest extends BaseIndexTestCase {
 
     public ArchiveAnalyzerTest() {
     }
@@ -49,11 +50,13 @@ public class ArchiveAnalyzerTest {
     }
 
     @Before
-    public void setUp() {
+    public void setUp() throws Exception {
+        super.setUp();
     }
 
     @After
-    public void tearDown() {
+    public void tearDown() throws Exception {
+        super.tearDown();
     }
 
     /**
@@ -66,6 +69,9 @@ public class ArchiveAnalyzerTest {
         expResult.add("zip");
         expResult.add("war");
         expResult.add("ear");
+        expResult.add("tar");
+        expResult.add("gz");
+        expResult.add("tgz");
         Set result = instance.getSupportedExtensions();
         assertEquals(expResult, result);
     }
@@ -86,7 +92,7 @@ public class ArchiveAnalyzerTest {
      */
     @Test
     public void testSupportsExtension() {
-        String extension = "tar"; //not supported
+        String extension = "7z"; //not supported
         ArchiveAnalyzer instance = new ArchiveAnalyzer();
         boolean expResult = false;
         boolean result = instance.supportsExtension(extension);
@@ -139,7 +145,7 @@ public class ArchiveAnalyzerTest {
         try {
             instance.initialize();
 
-            File file = new File(this.getClass().getClassLoader().getResource("opensso.war").getPath());
+            File file = new File(this.getClass().getClassLoader().getResource("daytrader-ear-2.1.7.ear").getPath());
             Dependency dependency = new Dependency(file);
             Settings.setBoolean(Settings.KEYS.AUTO_UPDATE, false);
             Engine engine = new Engine();
@@ -154,4 +160,109 @@ public class ArchiveAnalyzerTest {
             instance.close();
         }
     }
+
+    /**
+     * Test of analyze method, of class ArchiveAnalyzer.
+     */
+    @Test
+    public void testAnalyzeTar() throws Exception {
+        ArchiveAnalyzer instance = new ArchiveAnalyzer();
+        try {
+            instance.initialize();
+
+            File file = new File(this.getClass().getClassLoader().getResource("file.tar").getPath());
+            Dependency dependency = new Dependency(file);
+            Settings.setBoolean(Settings.KEYS.AUTO_UPDATE, false);
+            Engine engine = new Engine();
+
+            int initial_size = engine.getDependencies().size();
+            instance.analyze(dependency, engine);
+            int ending_size = engine.getDependencies().size();
+
+            assertTrue(initial_size < ending_size);
+
+        } finally {
+            instance.close();
+        }
+    }
+
+    /**
+     * Test of analyze method, of class ArchiveAnalyzer.
+     */
+    @Test
+    public void testAnalyzeTarGz() throws Exception {
+        ArchiveAnalyzer instance = new ArchiveAnalyzer();
+        try {
+            instance.initialize();
+
+            File file = new File(this.getClass().getClassLoader().getResource("file.tar.gz").getPath());
+            //Dependency dependency = new Dependency(file);
+            Settings.setBoolean(Settings.KEYS.AUTO_UPDATE, false);
+            Engine engine = new Engine();
+
+            int initial_size = engine.getDependencies().size();
+            //instance.analyze(dependency, engine);
+            engine.scan(file);
+            engine.analyzeDependencies();
+            int ending_size = engine.getDependencies().size();
+
+            assertTrue(initial_size < ending_size);
+
+        } finally {
+            instance.close();
+        }
+    }
+
+    /**
+     * Test of analyze method, of class ArchiveAnalyzer.
+     */
+    @Test
+    public void testAnalyzeTgz() throws Exception {
+        ArchiveAnalyzer instance = new ArchiveAnalyzer();
+        try {
+            instance.initialize();
+
+            File file = new File(this.getClass().getClassLoader().getResource("file.tgz").getPath());
+            Settings.setBoolean(Settings.KEYS.AUTO_UPDATE, false);
+            Engine engine = new Engine();
+
+            int initial_size = engine.getDependencies().size();
+            engine.scan(file);
+            engine.analyzeDependencies();
+            int ending_size = engine.getDependencies().size();
+
+            assertTrue(initial_size < ending_size);
+
+        } finally {
+            instance.close();
+        }
+    }
+
+    /**
+     * Test of analyze method, of class ArchiveAnalyzer.
+     */
+    @Test
+    public void testAnalyze_badZip() throws Exception {
+        ArchiveAnalyzer instance = new ArchiveAnalyzer();
+        try {
+            instance.initialize();
+
+            File file = new File(this.getClass().getClassLoader().getResource("test.zip").getPath());
+            Dependency dependency = new Dependency(file);
+            Settings.setBoolean(Settings.KEYS.AUTO_UPDATE, false);
+            Engine engine = new Engine();
+            int initial_size = engine.getDependencies().size();
+//            boolean failed = false;
+//            try {
+            instance.analyze(dependency, engine);
+//            } catch (java.lang.UnsupportedClassVersionError ex) {
+//                failed = true;
+//            }
+//            assertTrue(failed);
+            int ending_size = engine.getDependencies().size();
+            assertEquals(initial_size, ending_size);
+        } finally {
+            instance.close();
+        }
+    }
 }

dependency-check-core/src/test/java/org/owasp/dependencycheck/analyzer/CPEAnalyzerTest.java

@@ -16,10 +16,10 @@
  *
  * Copyright (c) 2012 Jeremy Long. All Rights Reserved.
  */
-package org.owasp.dependencycheck.data.cpe;
+package org.owasp.dependencycheck.analyzer;
 
 import org.owasp.dependencycheck.data.cpe.IndexEntry;
-import org.owasp.dependencycheck.data.cpe.CPEAnalyzer;
+import org.owasp.dependencycheck.analyzer.CPEAnalyzer;
 import java.io.File;
 import java.io.IOException;
 import java.util.HashSet;
@@ -30,15 +30,12 @@ import org.apache.lucene.queryparser.classic.ParseException;
 import org.junit.After;
 import org.junit.AfterClass;
 import org.owasp.dependencycheck.dependency.Dependency;
-import org.owasp.dependencycheck.analyzer.JarAnalyzer;
 import org.junit.Assert;
 import org.junit.Before;
 import org.junit.BeforeClass;
 import org.junit.Test;
-import org.owasp.dependencycheck.analyzer.FalsePositiveAnalyzer;
-import org.owasp.dependencycheck.analyzer.FileNameAnalyzer;
-import org.owasp.dependencycheck.analyzer.HintAnalyzer;
-import static org.owasp.dependencycheck.data.cpe.BaseIndexTestCase.ensureIndexExists;
+import org.owasp.dependencycheck.data.cpe.BaseIndexTestCase;
+import org.owasp.dependencycheck.data.cpe.IndexEntry;
 import org.owasp.dependencycheck.dependency.Identifier;
 
 /**
@@ -56,11 +53,13 @@ public class CPEAnalyzerTest extends BaseIndexTestCase {
     }
 
     @Before
+    @Override
     public void setUp() throws Exception {
         super.setUp();
     }
 
     @After
+    @Override
     public void tearDown() throws Exception {
         super.tearDown();
     }
@@ -102,21 +101,6 @@ public class CPEAnalyzerTest extends BaseIndexTestCase {
         Assert.assertTrue(expResult.equals(queryText));
     }
 
-    /**
-     * Test of open method, of class CPEAnalyzer.
-     *
-     * @throws Exception is thrown when an exception occurs
-     */
-    @Test
-    public void testOpen() throws Exception {
-        CPEAnalyzer instance = new CPEAnalyzer();
-        Assert.assertFalse(instance.isOpen());
-        instance.open();
-        Assert.assertTrue(instance.isOpen());
-        instance.close();
-        Assert.assertFalse(instance.isOpen());
-    }
-
     /**
      * Test of determineCPE method, of class CPEAnalyzer.
      *
@@ -124,6 +108,7 @@ public class CPEAnalyzerTest extends BaseIndexTestCase {
      */
     @Test
     public void testDetermineCPE_full() throws Exception {
+        callDetermineCPE_full("hazelcast-2.5.jar", null);
         callDetermineCPE_full("spring-context-support-2.5.5.jar", "cpe:/a:vmware:springsource_spring_framework:2.5.5");
         callDetermineCPE_full("spring-core-3.0.0.RELEASE.jar", "cpe:/a:vmware:springsource_spring_framework:3.0.0");
         callDetermineCPE_full("org.mortbay.jetty.jar", "cpe:/a:mortbay_jetty:jetty:4.2");
@@ -139,6 +124,7 @@ public class CPEAnalyzerTest extends BaseIndexTestCase {
     public void callDetermineCPE_full(String depName, String expResult) throws Exception {
 
         File file = new File(this.getClass().getClassLoader().getResource(depName).getPath());
+
         Dependency dep = new Dependency(file);
 
         FileNameAnalyzer fnAnalyzer = new FileNameAnalyzer();
@@ -163,8 +149,10 @@ public class CPEAnalyzerTest extends BaseIndexTestCase {
         if (expResult != null) {
             Identifier expIdentifier = new Identifier("cpe", expResult, expResult);
             Assert.assertTrue("Incorrect match: { dep:'" + dep.getFileName() + "' }", dep.getIdentifiers().contains(expIdentifier));
-        } else {
+        } else if (dep.getIdentifiers().isEmpty()) {
             Assert.assertTrue("Match found when an Identifier should not have been found: { dep:'" + dep.getFileName() + "' }", dep.getIdentifiers().isEmpty());
+        } else {
+            Assert.assertTrue("Match found when an Identifier should not have been found: { dep:'" + dep.getFileName() + "', identifier:'" + dep.getIdentifiers().iterator().next().getValue() + "' }", dep.getIdentifiers().isEmpty());
         }
     }
 

dependency-check-core/src/test/java/org/owasp/dependencycheck/analyzer/JarAnalyzerTest.java

@@ -118,6 +118,7 @@ public class JarAnalyzerTest {
         JarAnalyzer instance = new JarAnalyzer();
         Set expResult = new HashSet();
         expResult.add("jar");
+        expResult.add("war");
         Set result = instance.getSupportedExtensions();
         assertEquals(expResult, result);
     }

dependency-check-core/src/test/java/org/owasp/dependencycheck/concurrency/DirectorySpinLockTest.java

@@ -0,0 +1,116 @@
+/*
+ * This file is part of dependency-check-core.
+ *
+ * Dependency-check-core is free software: you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation, either version 3 of the License, or (at your option) any
+ * later version.
+ *
+ * Dependency-check-core is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+ * details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * dependency-check-core. If not, see http://www.gnu.org/licenses/.
+ *
+ * Copyright (c) 2013 Jeremy Long. All Rights Reserved.
+ */
+package org.owasp.dependencycheck.concurrency;
+
+import java.io.File;
+import java.net.URL;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import static org.junit.Assert.*;
+
+/**
+ *
+ * @author Jeremy Long (jeremy.long@owasp.org)
+ */
+public class DirectorySpinLockTest {
+
+    public DirectorySpinLockTest() {
+    }
+
+    @BeforeClass
+    public static void setUpClass() {
+    }
+
+    @AfterClass
+    public static void tearDownClass() {
+    }
+
+    @Before
+    public void setUp() {
+    }
+
+    @After
+    public void tearDown() {
+    }
+
+    /**
+     * Test of obtainSharedLock method, of class DirectorySpinLock.
+     * Specifically, this test uses the SpinLockTask to obtain an exclusive lock
+     * that is held for 5 seconds. We then try to obtain a shared lock while
+     * that task is running. It should take longer then 5 seconds to obtain the
+     * shared lock.
+     */
+    @Test
+    public void testObtainSharedLock_withContention() throws Exception {
+        URL location = this.getClass().getProtectionDomain().getCodeSource().getLocation();
+        File directory = new File(location.getFile());
+        DirectorySpinLock instance = new DirectorySpinLock(directory);
+        SpinLockTask task = new SpinLockTask(directory, 5000, false, 2);
+        long start = System.currentTimeMillis();
+        task.run();
+        instance.obtainSharedLock();
+        long end = System.currentTimeMillis();
+        instance.close();
+        if (task.getException() != null) {
+            throw task.getException();
+        }
+        long timeElapsed = end - start;
+        assertTrue("no lock contention occured?", timeElapsed >= 5000);
+        //no exceptions means everything worked.
+    }
+
+    /**
+     * Test of obtainSharedLock method, of class DirectorySpinLock. This method
+     * obtains two shared locks by using the SpinLockTask to obtain a lock in
+     * another thread.
+     */
+    @Test
+    public void testObtainSharedLock() throws Exception {
+        URL location = this.getClass().getProtectionDomain().getCodeSource().getLocation();
+        File directory = new File(location.getFile());
+        DirectorySpinLock instance = new DirectorySpinLock(directory);
+        SpinLockTask task = new SpinLockTask(directory, 1000, true, 2);
+        task.run();
+        instance.obtainSharedLock();
+        instance.close();
+        if (task.getException() != null) {
+            throw task.getException();
+        }
+        //no exceptions means everything worked.
+    }
+
+    /**
+     * Test of obtainExclusiveLock method, of class DirectorySpinLock.
+     */
+    @Test
+    public void testObtainExclusiveLock() throws Exception {
+        URL location = this.getClass().getProtectionDomain().getCodeSource().getLocation();
+        File directory = new File(location.getFile());
+        DirectorySpinLock instance = new DirectorySpinLock(directory);
+        SpinLockTask task = new SpinLockTask(directory, 1000, true, 1);
+        instance.obtainExclusiveLock();
+        task.run();
+        instance.close();
+        assertNotNull("No exception thrown due to exclusive lock failure?", task.getException());
+        assertEquals("Incorrect exception when obtaining exclusive lock", "Unable to obtain lock", task.getException().getMessage());
+    }
+}

dependency-check-core/src/test/java/org/owasp/dependencycheck/concurrency/SpinLockTask.java

@@ -0,0 +1,84 @@
+/*
+ * This file is part of dependency-check-core.
+ *
+ * Dependency-check-core is free software: you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation, either version 3 of the License, or (at your option) any
+ * later version.
+ *
+ * Dependency-check-core is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+ * details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * dependency-check-core. If not, see http://www.gnu.org/licenses/.
+ *
+ * Copyright (c) 2013 Jeremy Long. All Rights Reserved.
+ */
+package org.owasp.dependencycheck.concurrency;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+/**
+ * A simple task that obtains a lock on a directory. This is used in testing of
+ * the shared and exclusive locks.
+ *
+ * @author Jeremy Long (jeremy.long@owasp.org)
+ */
+public class SpinLockTask implements Runnable {
+
+    DirectorySpinLock lock = null;
+    int holdLockFor;
+    long maxWait;
+    boolean shared;
+    private Exception exception = null;
+
+    /**
+     * Get the value of exception
+     *
+     * @return the value of exception
+     */
+    public Exception getException() {
+        return exception;
+    }
+
+    /**
+     * Set the value of exception
+     *
+     * @param exception new value of exception
+     */
+    public void setException(Exception exception) {
+        this.exception = exception;
+    }
+
+    public SpinLockTask(File directory, int holdLockFor, boolean shared, long maxWait) throws InvalidDirectoryException, DirectoryLockException {
+        this.holdLockFor = holdLockFor;
+        this.shared = shared;
+        this.maxWait = maxWait;
+        lock = new DirectorySpinLock(directory);
+    }
+
+    @Override
+    public void run() {
+        try {
+            lock.obtainLock(shared, maxWait);
+            Thread.sleep(holdLockFor);
+        } catch (DirectoryLockException ex) {
+            exception = ex;
+        } catch (InterruptedException ex) {
+            exception = ex;
+        } finally {
+            if (lock != null) {
+                try {
+                    lock.close();
+                } catch (IOException ex) {
+                    exception = ex;
+                }
+            }
+        }
+    }
+}

dependency-check-core/src/test/java/org/owasp/dependencycheck/data/cpe/BaseIndexTestCase.java

@@ -18,30 +18,18 @@
  */
 package org.owasp.dependencycheck.data.cpe;
 
-import java.io.BufferedInputStream;
-import java.io.BufferedOutputStream;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.util.logging.Level;
-import java.util.logging.Logger;
-import java.util.zip.ZipEntry;
-import java.util.zip.ZipInputStream;
+import junit.framework.TestCase;
 import org.junit.After;
 import org.junit.AfterClass;
 import org.junit.Before;
 import org.junit.BeforeClass;
 import org.owasp.dependencycheck.data.nvdcve.BaseDBTestCase;
-import org.owasp.dependencycheck.utils.Settings;
 
 /**
  *
  * @author Jeremy Long (jeremy.long@owasp.org)
  */
-public abstract class BaseIndexTestCase {
-
-    protected static final int BUFFER_SIZE = 2048;
+public abstract class BaseIndexTestCase extends TestCase {
 
     @BeforeClass
     public static void setUpClass() throws Exception {
@@ -52,88 +40,15 @@ public abstract class BaseIndexTestCase {
     }
 
     @Before
+    @Override
     public void setUp() throws Exception {
-        ensureIndexExists();
         BaseDBTestCase.ensureDBExists();
+        super.setUp();
     }
 
     @After
+    @Override
     public void tearDown() throws Exception {
-    }
-
-    protected static File getDataDirectory() throws IOException {
-        final String fileName = Settings.getString(Settings.KEYS.CPE_DATA_DIRECTORY);
-        final String dataDirectory = Settings.getString(Settings.KEYS.DATA_DIRECTORY);
-        return new File(dataDirectory, fileName);
-        //return FileUtils.getDataDirectory(fileName, Index.class);
-    }
-
-    public static void ensureIndexExists() throws Exception {
-        //String indexPath = Settings.getString(Settings.KEYS.CPE_DATA_DIRECTORY);
-        String indexPath = getDataDirectory().getCanonicalPath();
-        java.io.File f = new File(indexPath);
-
-        if (!f.exists() || (f.isDirectory() && f.listFiles().length == 0)) {
-            f.mkdirs();
-            FileInputStream fis = null;
-            ZipInputStream zin = null;
-            try {
-                File path = new File(BaseIndexTestCase.class.getClassLoader().getResource("index.cpe.zip").getPath());
-                fis = new FileInputStream(path);
-                zin = new ZipInputStream(new BufferedInputStream(fis));
-                ZipEntry entry;
-                while ((entry = zin.getNextEntry()) != null) {
-                    if (entry.isDirectory()) {
-                        continue;
-                    }
-                    FileOutputStream fos = null;
-                    BufferedOutputStream dest = null;
-                    try {
-                        File o = new File(indexPath, entry.getName());
-                        o.createNewFile();
-                        fos = new FileOutputStream(o, false);
-                        dest = new BufferedOutputStream(fos, BUFFER_SIZE);
-                        byte data[] = new byte[BUFFER_SIZE];
-                        int count;
-                        while ((count = zin.read(data, 0, BUFFER_SIZE)) != -1) {
-                            dest.write(data, 0, count);
-                        }
-                    } catch (Exception ex) {
-                        Logger.getLogger(BaseIndexTestCase.class.getName()).log(Level.FINEST, null, ex);
-                    } finally {
-                        if (dest != null) {
-                            try {
-                                dest.flush();
-                                dest.close();
-                            } catch (Throwable ex) {
-                                Logger.getLogger(BaseIndexTestCase.class.getName()).log(Level.FINEST, null, ex);
-                            }
-                        }
-                        if (fos != null) {
-                            try {
-                                fos.close();
-                            } catch (Throwable ex) {
-                                Logger.getLogger(BaseIndexTestCase.class.getName()).log(Level.FINEST, null, ex);
-                            }
-                        }
-                    }
-                }
-            } finally {
-                try {
-                    if (zin != null) {
-                        zin.close();
-                    }
-                } catch (Throwable ex) {
-                    Logger.getLogger(BaseIndexTestCase.class.getName()).log(Level.FINEST, null, ex);
-                }
-                try {
-                    if (fis != null) {
-                        fis.close();
-                    }
-                } catch (Throwable ex) {
-                    Logger.getLogger(BaseIndexTestCase.class.getName()).log(Level.FINEST, null, ex);
-                }
-            }
-        }
+        super.tearDown();
     }
 }

dependency-check-core/src/test/java/org/owasp/dependencycheck/data/cpe/IndexEntryTest.java

@@ -19,6 +19,7 @@
 package org.owasp.dependencycheck.data.cpe;
 
 import org.owasp.dependencycheck.data.cpe.IndexEntry;
+import junit.framework.TestCase;
 import org.junit.After;
 import org.junit.AfterClass;
 import org.junit.Before;
@@ -30,7 +31,7 @@ import org.junit.Assert;
  *
  * @author Jeremy Long (jeremy.long@owasp.org)
  */
-public class IndexEntryTest {
+public class IndexEntryTest extends TestCase {
 
     @BeforeClass
     public static void setUpClass() throws Exception {
@@ -41,11 +42,15 @@ public class IndexEntryTest {
     }
 
     @Before
-    public void setUp() {
+    @Override
+    public void setUp() throws Exception {
+        super.setUp();
     }
 
     @After
-    public void tearDown() {
+    @Override
+    public void tearDown() throws Exception {
+        super.tearDown();
     }
 
     /**

dependency-check-core/src/test/java/org/owasp/dependencycheck/data/cpe/IndexTest.java

@@ -1,103 +0,0 @@
-/*
- * This file is part of dependency-check-core.
- *
- * Dependency-check-core is free software: you can redistribute it and/or modify it
- * under the terms of the GNU General Public License as published by the Free
- * Software Foundation, either version 3 of the License, or (at your option) any
- * later version.
- *
- * Dependency-check-core is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
- * details.
- *
- * You should have received a copy of the GNU General Public License along with
- * dependency-check-core. If not, see http://www.gnu.org/licenses/.
- *
- * Copyright (c) 2012 Jeremy Long. All Rights Reserved.
- */
-package org.owasp.dependencycheck.data.cpe;
-
-import org.owasp.dependencycheck.data.cpe.Index;
-import java.io.File;
-import java.io.IOException;
-import java.util.logging.Level;
-import java.util.logging.Logger;
-import org.apache.lucene.document.Document;
-import org.apache.lucene.queryparser.classic.ParseException;
-import org.apache.lucene.search.ScoreDoc;
-import org.apache.lucene.search.TopDocs;
-import org.apache.lucene.store.Directory;
-import org.junit.After;
-import org.junit.AfterClass;
-import org.junit.Before;
-import org.junit.BeforeClass;
-import org.junit.Test;
-import static org.junit.Assert.*;
-
-/**
- *
- * @author Jeremy Long (jeremy.long@owasp.org)
- */
-public class IndexTest {
-
-    @BeforeClass
-    public static void setUpClass() throws Exception {
-    }
-
-    @AfterClass
-    public static void tearDownClass() throws Exception {
-    }
-
-    @Before
-    public void setUp() {
-    }
-
-    @After
-    public void tearDown() {
-    }
-
-    /**
-     * Test of open method, of class Index.
-     */
-    @Test
-    public void testOpen() {
-        Index instance = new Index();
-        try {
-            instance.open();
-            //TODO research why are we getting multiple documents for the same documentId. is the update method not working?
-//            try {
-//                instance.createSearchingAnalyzer();
-//                TopDocs docs = instance.search("product:( project\\-open )", 20);
-//                for (ScoreDoc d : docs.scoreDocs) {
-//                    final Document doc = instance.getDocument(d.doc);
-//                    String vendor = doc.getField(Fields.VENDOR).stringValue();
-//                    String product = doc.getField(Fields.PRODUCT).stringValue();
-//                    System.out.print(d.doc);
-//                    System.out.print(" : ");
-//                    System.out.print(vendor + ":");
-//                    System.out.println(product);
-//                }
-//            } catch (ParseException ex) {
-//                Logger.getLogger(IndexTest.class.getName()).log(Level.SEVERE, null, ex);
-//            }
-        } catch (IOException ex) {
-            assertNull(ex.getMessage(), ex);
-        }
-        instance.close();
-    }
-
-    /**
-     * Test of getDirectory method, of class Index.
-     *
-     * @throws Exception
-     */
-    @Test
-    public void testGetDirectory() throws Exception {
-        Index index = new Index();
-        Directory result = index.getDirectory();
-
-        String exp = File.separatorChar + "target" + File.separatorChar + "data" + File.separatorChar + "cpe";
-        assertTrue(result.toString().contains(exp));
-    }
-}

dependency-check-core/src/test/java/org/owasp/dependencycheck/data/lucene/FieldAnalyzerTest.java

@@ -72,7 +72,7 @@ public class FieldAnalyzerTest {
     @Test
     public void testAnalyzers() throws Exception {
 
-        Analyzer analyzer = new FieldAnalyzer(Version.LUCENE_43);
+        Analyzer analyzer = new FieldAnalyzer(LuceneUtils.CURRENT_VERSION);
         Directory index = new RAMDirectory();
 
         String field1 = "product";
@@ -83,16 +83,16 @@ public class FieldAnalyzerTest {
 
         createIndex(analyzer, index, field1, text1, field2, text2);
 
-        //Analyzer searchingAnalyzer = new SearchFieldAnalyzer(Version.LUCENE_43);
+        //Analyzer searchingAnalyzer = new SearchFieldAnalyzer(LuceneUtils.CURRENT_VERSION);
         String querystr = "product:\"(Spring Framework Core)\" vendor:(SpringSource)";
 
-        SearchFieldAnalyzer searchAnalyzerProduct = new SearchFieldAnalyzer(Version.LUCENE_43);
-        SearchFieldAnalyzer searchAnalyzerVendor = new SearchFieldAnalyzer(Version.LUCENE_43);
+        SearchFieldAnalyzer searchAnalyzerProduct = new SearchFieldAnalyzer(LuceneUtils.CURRENT_VERSION);
+        SearchFieldAnalyzer searchAnalyzerVendor = new SearchFieldAnalyzer(LuceneUtils.CURRENT_VERSION);
         HashMap<String, Analyzer> map = new HashMap<String, Analyzer>();
         map.put(field1, searchAnalyzerProduct);
         map.put(field2, searchAnalyzerVendor);
-        PerFieldAnalyzerWrapper wrapper = new PerFieldAnalyzerWrapper(new StandardAnalyzer(Version.LUCENE_43), map);
-        QueryParser parser = new QueryParser(Version.LUCENE_43, field1, wrapper);
+        PerFieldAnalyzerWrapper wrapper = new PerFieldAnalyzerWrapper(new StandardAnalyzer(LuceneUtils.CURRENT_VERSION), map);
+        QueryParser parser = new QueryParser(LuceneUtils.CURRENT_VERSION, field1, wrapper);
 
         Query q = parser.parse(querystr);
         //System.out.println(q.toString());
@@ -116,7 +116,7 @@ public class FieldAnalyzerTest {
     }
 
     private void createIndex(Analyzer analyzer, Directory index, String field1, String text1, String field2, String text2) throws IOException {
-        IndexWriterConfig config = new IndexWriterConfig(Version.LUCENE_43, analyzer);
+        IndexWriterConfig config = new IndexWriterConfig(LuceneUtils.CURRENT_VERSION, analyzer);
         IndexWriter w = new IndexWriter(index, config);
         addDoc(w, field1, text1, field2, text2);
         w.close();

dependency-check-core/src/test/java/org/owasp/dependencycheck/data/lucene/TokenPairConcatenatingFilterTest.java

@@ -6,12 +6,18 @@ package org.owasp.dependencycheck.data.lucene;
 
 import java.io.IOException;
 import java.io.Reader;
+import java.io.StringReader;
 import org.apache.lucene.analysis.Analyzer;
 import org.apache.lucene.analysis.BaseTokenStreamTestCase;
 import static org.apache.lucene.analysis.BaseTokenStreamTestCase.assertAnalyzesTo;
+import static org.apache.lucene.analysis.BaseTokenStreamTestCase.assertTokenStreamContents;
 import static org.apache.lucene.analysis.BaseTokenStreamTestCase.checkOneTerm;
 import org.apache.lucene.analysis.MockTokenizer;
+import org.apache.lucene.analysis.TokenStream;
 import org.apache.lucene.analysis.Tokenizer;
+import org.apache.lucene.analysis.core.WhitespaceTokenizer;
+import org.apache.lucene.analysis.tokenattributes.TypeAttributeImpl;
+import org.apache.lucene.util.Version;
 import org.junit.After;
 import org.junit.AfterClass;
 import org.junit.Before;
@@ -25,19 +31,6 @@ import static org.junit.Assert.*;
  */
 public class TokenPairConcatenatingFilterTest extends BaseTokenStreamTestCase {
 
-    private Analyzer analyzer;
-
-    public TokenPairConcatenatingFilterTest() {
-        analyzer = new Analyzer() {
-            @Override
-            protected Analyzer.TokenStreamComponents createComponents(String fieldName,
-                    Reader reader) {
-                Tokenizer source = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
-                return new Analyzer.TokenStreamComponents(source, new TokenPairConcatenatingFilter(source));
-            }
-        };
-    }
-
     @BeforeClass
     public static void setUpClass() {
     }
@@ -60,21 +53,25 @@ public class TokenPairConcatenatingFilterTest extends BaseTokenStreamTestCase {
      * test some examples
      */
     public void testExamples() throws IOException {
-        //TODO figure outwhy I am getting "Failed: incrementtoken() called while in wrong state"
-//        String[] expected = new String[3];
-//        expected[0] = "one";
-//        expected[1] = "onetwo";
-//        expected[2] = "two";
-//        checkOneTerm(analyzer, "one", "one");
-//        assertAnalyzesTo(analyzer, "two", new String[]{"onetwo", "two"});
-        //checkOneTerm(analyzer, "two", "onetwo");
-        //checkOneTerm(analyzer, "three", "two");
+        Tokenizer wsTokenizer = new WhitespaceTokenizer(LuceneUtils.CURRENT_VERSION, new StringReader("one two three"));
+        TokenStream filter = new TokenPairConcatenatingFilter(wsTokenizer);
+        assertTokenStreamContents(filter,
+                new String[]{"one", "onetwo", "two", "twothree", "three"});
     }
 
     /**
      * Test of clear method, of class TokenPairConcatenatingFilter.
      */
     @Test
-    public void testClear() {
+    public void testClear() throws IOException {
+
+        TokenStream ts = new WhitespaceTokenizer(LuceneUtils.CURRENT_VERSION, new StringReader("one two three"));
+        TokenPairConcatenatingFilter filter = new TokenPairConcatenatingFilter(ts);
+        assertTokenStreamContents(filter, new String[]{"one", "onetwo", "two", "twothree", "three"});
+
+        assertNotNull(filter.getPreviousWord());
+        filter.clear();
+        assertNull(filter.getPreviousWord());
+        assertTrue(filter.getWords().isEmpty());
     }
 }

dependency-check-core/src/test/java/org/owasp/dependencycheck/data/nvdcve/BaseDBTestCase.java

@@ -30,6 +30,7 @@ import java.util.logging.Logger;
 import java.util.zip.ZipEntry;
 import java.util.zip.ZipInputStream;
 import junit.framework.TestCase;
+import org.owasp.dependencycheck.data.update.DataStoreMetaInfo;
 import org.owasp.dependencycheck.utils.Settings;
 
 /**
@@ -46,32 +47,28 @@ public abstract class BaseDBTestCase extends TestCase {
         ensureDBExists();
     }
 
-    protected static File getDataDirectory() throws IOException {
-        final String fileName = Settings.getString(Settings.KEYS.CVE_DATA_DIRECTORY);
-        final String dataDirectory = Settings.getString(Settings.KEYS.DATA_DIRECTORY);
-        return new File(dataDirectory, fileName);
-    }
-
     public static void ensureDBExists() throws Exception {
-        String indexPath = getDataDirectory().getCanonicalPath();
-        java.io.File f = new File(indexPath);
-        if (!f.exists() || (f.isDirectory() && f.listFiles().length == 0)) {
-            f.mkdirs();
+
+        java.io.File dataPath = Settings.getDataFile(Settings.KEYS.DATA_DIRECTORY);
+        if (!dataPath.exists() || (dataPath.isDirectory() && dataPath.listFiles().length < 3)) {
+            dataPath.mkdirs();
             FileInputStream fis = null;
             ZipInputStream zin = null;
             try {
-                File path = new File(BaseDBTestCase.class.getClassLoader().getResource("db.cve.zip").getPath());
+                File path = new File(BaseDBTestCase.class.getClassLoader().getResource("data.zip").getPath());
                 fis = new FileInputStream(path);
                 zin = new ZipInputStream(new BufferedInputStream(fis));
                 ZipEntry entry;
                 while ((entry = zin.getNextEntry()) != null) {
                     if (entry.isDirectory()) {
+                        final File d = new File(dataPath, entry.getName());
+                        d.mkdir();
                         continue;
                     }
                     FileOutputStream fos = null;
                     BufferedOutputStream dest = null;
                     try {
-                        File o = new File(indexPath, entry.getName());
+                        File o = new File(dataPath, entry.getName());
                         o.createNewFile();
                         fos = new FileOutputStream(o, false);
                         dest = new BufferedOutputStream(fos, BUFFER_SIZE);
@@ -81,7 +78,7 @@ public abstract class BaseDBTestCase extends TestCase {
                             dest.write(data, 0, count);
                         }
                     } catch (Exception ex) {
-                        Logger.getLogger(BaseDBTestCase.class.getName()).log(Level.FINEST, null, ex);
+                        Logger.getLogger(BaseDBTestCase.class.getName()).log(Level.SEVERE, null, ex);
                     } finally {
                         try {
                             if (dest != null) {

dependency-check-core/src/test/java/org/owasp/dependencycheck/data/nvdcve/NvdCve_1_2_HandlerTest.java

@@ -16,9 +16,9 @@
  *
  * Copyright (c) 2012 Jeremy Long. All Rights Reserved.
  */
-package org.owasp.dependencycheck.data.nvdcve.xml;
+package org.owasp.dependencycheck.data.nvdcve;
 
-import org.owasp.dependencycheck.data.nvdcve.xml.NvdCve12Handler;
+import org.owasp.dependencycheck.data.nvdcve.NvdCve12Handler;
 import java.io.File;
 import java.util.List;
 import java.util.Map;

dependency-check-core/src/test/java/org/owasp/dependencycheck/data/nvdcve/NvdCve_2_0_HandlerTest.java

@@ -16,9 +16,9 @@
  *
  * Copyright (c) 2012 Jeremy Long. All Rights Reserved.
  */
-package org.owasp.dependencycheck.data.nvdcve.xml;
+package org.owasp.dependencycheck.data.nvdcve;
 
-import org.owasp.dependencycheck.data.nvdcve.xml.NvdCve20Handler;
+import org.owasp.dependencycheck.data.nvdcve.NvdCve20Handler;
 import java.io.File;
 import javax.xml.parsers.SAXParser;
 import javax.xml.parsers.SAXParserFactory;

dependency-check-core/src/test/java/org/owasp/dependencycheck/data/update/DataStoreMetaInfoTest.java

@@ -0,0 +1,118 @@
+/*
+ * This file is part of dependency-check-core.
+ *
+ * Dependency-check-core is free software: you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation, either version 3 of the License, or (at your option) any
+ * later version.
+ *
+ * Dependency-check-core is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+ * details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * dependency-check-core. If not, see http://www.gnu.org/licenses/.
+ *
+ * Copyright (c) 2013 Jeremy Long. All Rights Reserved.
+ */
+package org.owasp.dependencycheck.data.update;
+
+import java.io.File;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import static org.junit.Assert.*;
+
+/**
+ *
+ * @author Jeremy Long (jeremy.long@owasp.org)
+ */
+public class DataStoreMetaInfoTest {
+
+    public DataStoreMetaInfoTest() {
+    }
+
+    @BeforeClass
+    public static void setUpClass() {
+    }
+
+    @AfterClass
+    public static void tearDownClass() {
+    }
+
+    @Before
+    public void setUp() {
+    }
+
+    @After
+    public void tearDown() {
+    }
+
+    /**
+     * Test of isBatchUpdateMode method, of class DataStoreMetaInfo.
+     */
+    @Test
+    public void testIsBatchUpdateMode() {
+        DataStoreMetaInfo instance = new DataStoreMetaInfo();
+        boolean expResult = false;
+        instance.setBatchUpdateMode(expResult);
+        boolean result = instance.isBatchUpdateMode();
+        assertEquals(expResult, result);
+    }
+
+    /**
+     * Test of isEmpty method, of class DataStoreMetaInfo.
+     */
+    @Test
+    public void testIsEmpty() {
+        DataStoreMetaInfo instance = new DataStoreMetaInfo();
+        boolean expResult = false;
+        boolean result = instance.isEmpty();
+        assertEquals(expResult, result);
+    }
+
+    /**
+     * Test of save method, of class DataStoreMetaInfo.
+     */
+    @Test
+    public void testSave() throws Exception {
+        NvdCveInfo updatedValue = new NvdCveInfo();
+        String key = "test";
+        long expected = 1337;
+        updatedValue.setId(key);
+        updatedValue.setTimestamp(expected);
+        DataStoreMetaInfo instance = new DataStoreMetaInfo();
+        instance.save(updatedValue);
+        //reload the properties
+        instance = new DataStoreMetaInfo();
+        long results = Long.parseLong(instance.getProperty("lastupdated." + key));
+        assertEquals(expected, results);
+
+    }
+
+    /**
+     * Test of getProperty method, of class DataStoreMetaInfo.
+     */
+    @Test
+    public void testGetProperty_String_String() {
+        String key = "doesn't exist";
+        String defaultValue = "default";
+        DataStoreMetaInfo instance = new DataStoreMetaInfo();
+        String expResult = "default";
+        String result = instance.getProperty(key, defaultValue);
+        assertEquals(expResult, result);
+    }
+
+    /**
+     * Test of getPropertiesFile method, of class DataStoreMetaInfo.
+     */
+    @Test
+    public void testGetPropertiesFile() {
+        File result = DataStoreMetaInfo.getPropertiesFile();
+        //wow... rigorous!
+        assertNotNull(result);
+    }
+}

dependency-check-core/src/test/java/org/owasp/dependencycheck/data/update/DatabaseUpdaterIntegrationTest.java

@@ -14,16 +14,16 @@
  * You should have received a copy of the GNU General Public License along with
  * dependency-check-core. If not, see http://www.gnu.org/licenses/.
  *
- * Copyright (c) 2012 Jeremy Long. All Rights Reserved.
+ * Copyright (c) 2013 Jeremy Long. All Rights Reserved.
  */
-package org.owasp.dependencycheck.data.nvdcve.xml;
+package org.owasp.dependencycheck.data.update;
 
-import org.owasp.dependencycheck.data.nvdcve.xml.DatabaseUpdater;
 import org.junit.After;
 import org.junit.AfterClass;
 import org.junit.Before;
 import org.junit.BeforeClass;
 import org.junit.Test;
+import static org.junit.Assert.*;
 
 /**
  *
@@ -35,11 +35,11 @@ public class DatabaseUpdaterIntegrationTest {
     }
 
     @BeforeClass
-    public static void setUpClass() throws Exception {
+    public static void setUpClass() {
     }
 
     @AfterClass
-    public static void tearDownClass() throws Exception {
+    public static void tearDownClass() {
     }
 
     @Before
@@ -52,8 +52,6 @@ public class DatabaseUpdaterIntegrationTest {
 
     /**
      * Test of update method, of class DatabaseUpdater.
-     *
-     * @throws Exception
      */
     @Test
     public void testUpdate() throws Exception {

dependency-check-core/src/test/java/org/owasp/dependencycheck/data/update/NvdCveInfoTest.java

@@ -0,0 +1,113 @@
+/*
+ * This file is part of dependency-check-core.
+ *
+ * Dependency-check-core is free software: you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation, either version 3 of the License, or (at your option) any
+ * later version.
+ *
+ * Dependency-check-core is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+ * details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * dependency-check-core. If not, see http://www.gnu.org/licenses/.
+ *
+ * Copyright (c) 2013 Jeremy Long. All Rights Reserved.
+ */
+package org.owasp.dependencycheck.data.update;
+
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import static org.junit.Assert.*;
+
+/**
+ * Rigorous test of setters/getters.
+ *
+ * @author Jeremy Long (jeremy.long@owasp.org)
+ */
+public class NvdCveInfoTest {
+
+    public NvdCveInfoTest() {
+    }
+
+    @BeforeClass
+    public static void setUpClass() {
+    }
+
+    @AfterClass
+    public static void tearDownClass() {
+    }
+
+    @Before
+    public void setUp() {
+    }
+
+    @After
+    public void tearDown() {
+    }
+
+    /**
+     * Test of setId and getId method, of class NvdCveInfo.
+     */
+    @Test
+    public void testSetGetId() {
+        NvdCveInfo instance = new NvdCveInfo();
+        String expResult = "id";
+        instance.setId(expResult);
+        String result = instance.getId();
+        assertEquals(expResult, result);
+    }
+
+    /**
+     * Test of getUrl method, of class NvdCveInfo.
+     */
+    @Test
+    public void testSetGetUrl() {
+        NvdCveInfo instance = new NvdCveInfo();
+        String expResult = "http://www.someurl.com/something";
+        instance.setUrl(expResult);
+        String result = instance.getUrl();
+        assertEquals(expResult, result);
+    }
+
+    /**
+     * Test of getOldSchemaVersionUrl method, of class NvdCveInfo.
+     */
+    @Test
+    public void testSetGetOldSchemaVersionUrl() {
+        NvdCveInfo instance = new NvdCveInfo();
+        String expResult = "http://www.someurl.com/something";
+        instance.setOldSchemaVersionUrl(expResult);
+        String result = instance.getOldSchemaVersionUrl();
+        assertEquals(expResult, result);
+    }
+
+    /**
+     * Test of getTimestamp method, of class NvdCveInfo.
+     */
+    @Test
+    public void testSetGetTimestamp() {
+        NvdCveInfo instance = new NvdCveInfo();
+        long expResult = 1337L;
+        instance.setTimestamp(expResult);
+        long result = instance.getTimestamp();
+        assertEquals(expResult, result);
+    }
+
+    /**
+     * Test of getNeedsUpdate method, of class NvdCveInfo.
+     */
+    @Test
+    public void testSetGetNeedsUpdate() {
+        NvdCveInfo instance = new NvdCveInfo();
+        boolean expResult = true;
+        instance.setNeedsUpdate(expResult);
+        boolean result = instance.getNeedsUpdate();
+        assertEquals(expResult, result);
+    }
+}

dependency-check-core/src/test/java/org/owasp/dependencycheck/data/update/StandardUpdateIntegrationTest.java

@@ -0,0 +1,141 @@
+/*
+ * This file is part of dependency-check-core.
+ *
+ * Dependency-check-core is free software: you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation, either version 3 of the License, or (at your option) any
+ * later version.
+ *
+ * Dependency-check-core is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+ * details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * dependency-check-core. If not, see http://www.gnu.org/licenses/.
+ *
+ * Copyright (c) 2013 Jeremy Long. All Rights Reserved.
+ */
+package org.owasp.dependencycheck.data.update;
+
+import java.io.IOException;
+import java.net.MalformedURLException;
+import java.util.Calendar;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import static org.junit.Assert.*;
+import org.owasp.dependencycheck.data.UpdateException;
+import org.owasp.dependencycheck.utils.DownloadFailedException;
+
+/**
+ *
+ * @author Jeremy Long (jeremy.long@owasp.org)
+ */
+public class StandardUpdateIntegrationTest {
+
+    public StandardUpdateIntegrationTest() {
+    }
+
+    @BeforeClass
+    public static void setUpClass() {
+    }
+
+    @AfterClass
+    public static void tearDownClass() {
+    }
+
+    @Before
+    public void setUp() {
+    }
+
+    @After
+    public void tearDown() {
+    }
+
+    public StandardUpdate getStandardUpdateTask() throws MalformedURLException, DownloadFailedException, UpdateException {
+        StandardUpdate instance = new StandardUpdate();
+        return instance;
+    }
+
+    /**
+     * Test of setDeleteAndRecreate method, of class StandardUpdate.
+     */
+    @Test
+    public void testSetDeleteAndRecreate() throws Exception {
+        boolean deleteAndRecreate = false;
+        boolean expResult = false;
+        StandardUpdate instance = getStandardUpdateTask();
+        instance.setDeleteAndRecreate(deleteAndRecreate);
+        boolean result = instance.shouldDeleteAndRecreate();
+        assertEquals(expResult, result);
+    }
+
+    /**
+     * Test of deleteExistingData method, of class StandardUpdate.
+     */
+    @Test
+    public void testDeleteExistingData() throws Exception {
+        StandardUpdate instance = getStandardUpdateTask();
+        Exception result = null;
+        try {
+            instance.deleteExistingData();
+        } catch (IOException ex) {
+            result = ex;
+        }
+        assertNull(result);
+    }
+
+    /**
+     * Test of openDataStores method, of class StandardUpdate.
+     */
+    @Test
+    public void testOpenDataStores() throws Exception {
+        StandardUpdate instance = getStandardUpdateTask();
+        instance.openDataStores();
+        instance.closeDataStores();
+    }
+
+    /**
+     * Test of withinRange method, of class StandardUpdate.
+     */
+    @Test
+    public void testWithinRange() throws Exception {
+        Calendar c = Calendar.getInstance();
+
+        long current = c.getTimeInMillis();
+        long lastRun = c.getTimeInMillis() - (3 * (1000 * 60 * 60 * 24));
+        int range = 7; // 7 days
+        StandardUpdate instance = getStandardUpdateTask();
+        boolean expResult = true;
+        boolean result = instance.withinRange(lastRun, current, range);
+        assertEquals(expResult, result);
+
+        lastRun = c.getTimeInMillis() - (8 * (1000 * 60 * 60 * 24));
+        expResult = false;
+        result = instance.withinRange(lastRun, current, range);
+        assertEquals(expResult, result);
+    }
+
+    /**
+     * Test of update method, of class StandardUpdate.
+     */
+    @Test
+    public void testUpdate() throws Exception {
+        StandardUpdate instance = getStandardUpdateTask();
+        instance.update();
+        //TODO make this an actual test
+    }
+
+    /**
+     * Test of updatesNeeded method, of class StandardUpdate.
+     */
+    @Test
+    public void testUpdatesNeeded() throws Exception {
+        StandardUpdate instance = getStandardUpdateTask();
+        Updateable result = instance.updatesNeeded();
+        assertNotNull(result);
+    }
+}

dependency-check-core/src/test/java/org/owasp/dependencycheck/data/update/UpdateableTest.java

@@ -0,0 +1,160 @@
+/*
+ * This file is part of dependency-check-core.
+ *
+ * Dependency-check-core is free software: you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation, either version 3 of the License, or (at your option) any
+ * later version.
+ *
+ * Dependency-check-core is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+ * details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * dependency-check-core. If not, see http://www.gnu.org/licenses/.
+ *
+ * Copyright (c) 2013 Jeremy Long. All Rights Reserved.
+ */
+package org.owasp.dependencycheck.data.update;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.MalformedURLException;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import static org.junit.Assert.*;
+import org.owasp.dependencycheck.utils.DownloadFailedException;
+
+/**
+ *
+ * @author Jeremy Long (jeremy.long@owasp.org)
+ */
+public class UpdateableTest {
+
+    public UpdateableTest() {
+    }
+
+    @BeforeClass
+    public static void setUpClass() {
+    }
+
+    @AfterClass
+    public static void tearDownClass() {
+    }
+
+    @Before
+    public void setUp() {
+    }
+
+    @After
+    public void tearDown() {
+    }
+
+    /**
+     * Test of isUpdateNeeded method, of class Updateable.
+     */
+    @Test
+    public void testIsUpdateNeeded() throws MalformedURLException, DownloadFailedException, IOException {
+        String id = "key";
+        //use a local file as this test will load the result and check the timestamp
+        File f = new File("target/test-classes/nvdcve-2.0-2012.xml");
+        String url = "file:///" + f.getCanonicalPath();
+        Updateable instance = new Updateable();
+        instance.add(id, url, url, false);
+
+        boolean expResult = false;
+        boolean result = instance.isUpdateNeeded();
+        assertEquals(expResult, result);
+
+        instance.add("nextId", url, url, true);
+
+        expResult = true;
+        result = instance.isUpdateNeeded();
+        assertEquals(expResult, result);
+    }
+
+    /**
+     * Test of add method, of class Updateable.
+     */
+    @Test
+    public void testAdd_3args() throws Exception {
+        String id = "key";
+        File f = new File("target/test-classes/nvdcve-2.0-2012.xml");
+        //use a local file as this test will load the result and check the timestamp
+        String url = "file:///" + f.getCanonicalPath();
+        Updateable instance = new Updateable();
+        instance.add(id, url, url);
+        NvdCveInfo results = instance.get(id);
+        assertEquals(id, results.getId());
+        assertEquals(url, results.getUrl());
+        assertEquals(url, results.getOldSchemaVersionUrl());
+    }
+
+    /**
+     * Test of add method, of class Updateable.
+     */
+    @Test
+    public void testAdd_4args() throws Exception {
+        String id = "key";
+        //use a local file as this test will load the result and check the timestamp
+        File f = new File("target/test-classes/nvdcve-2.0-2012.xml");
+        String url = "file:///" + f.getCanonicalPath();
+        Updateable instance = new Updateable();
+        instance.add(id, url, url, false);
+
+        boolean expResult = false;
+        boolean result = instance.isUpdateNeeded();
+        assertEquals(expResult, result);
+
+        instance.add("nextId", url, url, false);
+        NvdCveInfo results = instance.get(id);
+
+        assertEquals(id, results.getId());
+        assertEquals(url, results.getUrl());
+        assertEquals(url, results.getOldSchemaVersionUrl());
+
+    }
+
+    /**
+     * Test of clear method, of class Updateable.
+     */
+    @Test
+    public void testClear() throws MalformedURLException, DownloadFailedException, IOException {
+        String id = "key";
+        //use a local file as this test will load the result and check the timestamp
+        File f = new File("target/test-classes/nvdcve-2.0-2012.xml");
+        String url = "file:///" + f.getCanonicalPath();
+        Updateable instance = new Updateable();
+        instance.add(id, url, url, false);
+        assertFalse(instance.getCollection().isEmpty());
+        instance.clear();
+        assertTrue(instance.getCollection().isEmpty());
+    }
+
+    /**
+     * Test of iterator method, of class Updateable.
+     */
+    @Test
+    public void testIterator() throws IOException {
+        //use a local file as this test will load the result and check the timestamp
+        File f = new File("target/test-classes/nvdcve-2.0-2012.xml");
+        String url = "file:///" + f.getCanonicalPath();
+        Updateable instance = new Updateable();
+        instance.add("one", url, url, false);
+        instance.add("two", url, url, false);
+        instance.add("three", url, url, false);
+        int itemsProcessed = 0;
+        for (NvdCveInfo item : instance) {
+            if ("one".equals(item.getId())) {
+                instance.remove();
+            }
+            itemsProcessed += 1;
+        }
+        assertEquals(3, itemsProcessed);
+        assertEquals(2, instance.getCollection().size());
+    }
+}

dependency-check-core/src/test/java/org/owasp/dependencycheck/reporting/ReportGeneratorTest.java

@@ -18,6 +18,7 @@
  */
 package org.owasp.dependencycheck.reporting;
 
+import org.owasp.dependencycheck.Engine;
 import org.owasp.dependencycheck.data.cpe.BaseIndexTestCase;
 import org.junit.After;
 import org.junit.AfterClass;
@@ -25,6 +26,14 @@ import org.junit.Before;
 import org.junit.BeforeClass;
 import org.junit.Test;
 
+import javax.xml.XMLConstants;
+import javax.xml.transform.stream.StreamSource;
+import javax.xml.validation.Schema;
+import javax.xml.validation.SchemaFactory;
+import javax.xml.validation.Validator;
+import java.io.File;
+import java.io.InputStream;
+
 /**
  *
  * @author Jeremy Long (jeremy.long@owasp.org)
@@ -107,4 +116,41 @@ public class ReportGeneratorTest {
 //        instance.generateReport(templateName, writeTo, properties);
         //assertTrue("need to add a real check here", false);
     }
+
+    /**
+     * Generates an XML report containing known vulnerabilities and realistic
+     * data and validates the generated XML document against the XSD.
+     * @throws Exception
+     */
+    @Test
+    public void testGenerateXMLReport() throws Exception {
+        String templateName = "XmlReport";
+
+        File f = new File("target/test-reports");
+        if (!f.exists()) {
+            f.mkdir();
+        }
+        String writeTo = "target/test-reports/Report.xml";
+
+        File struts = new File(this.getClass().getClassLoader().getResource("struts2-core-2.1.2.jar").getPath());
+        File axis = new File(this.getClass().getClassLoader().getResource("axis2-adb-1.4.1.jar").getPath());
+        File jetty = new File(this.getClass().getClassLoader().getResource("org.mortbay.jetty.jar").getPath());
+
+        Engine engine = new Engine();
+        engine.scan(struts);
+        engine.scan(axis);
+        engine.scan(jetty);
+        engine.analyzeDependencies();
+
+        ReportGenerator generator = new ReportGenerator("Test Report", engine.getDependencies(), engine.getAnalyzers());
+        generator.generateReport(templateName, writeTo);
+
+        InputStream xsdStream = ReportGenerator.class.getClassLoader().getResourceAsStream("schema/DependencyCheck.xsd");
+        StreamSource xsdSource = new StreamSource(xsdStream);
+        StreamSource xmlSource =  new StreamSource(new File(writeTo));
+        SchemaFactory sf = SchemaFactory.newInstance(XMLConstants.W3C_XML_SCHEMA_NS_URI);
+        Schema schema = sf.newSchema(xsdSource);
+        Validator validator = schema.newValidator();
+        validator.validate(xmlSource);
+    }
 }

dependency-check-core/src/test/java/org/owasp/dependencycheck/suppression/PropertyTypeTest.java

@@ -0,0 +1,108 @@
+/*
+ * This file is part of dependency-check-core.
+ *
+ * Dependency-check-core is free software: you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation, either version 3 of the License, or (at your option) any
+ * later version.
+ *
+ * Dependency-check-core is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+ * details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * dependency-check-core. If not, see http://www.gnu.org/licenses/.
+ *
+ * Copyright (c) 2013 Jeremy Long. All Rights Reserved.
+ */
+package org.owasp.dependencycheck.suppression;
+
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import static org.junit.Assert.*;
+
+/**
+ *
+ * @author Jeremy Long (jeremy.long@owasp.org)
+ */
+public class PropertyTypeTest {
+
+    public PropertyTypeTest() {
+    }
+
+    @BeforeClass
+    public static void setUpClass() {
+    }
+
+    @AfterClass
+    public static void tearDownClass() {
+    }
+
+    @Before
+    public void setUp() {
+    }
+
+    @After
+    public void tearDown() {
+    }
+
+    /**
+     * Test of set and getValue method, of class PropertyType.
+     */
+    @Test
+    public void testSetGetValue() {
+
+        PropertyType instance = new PropertyType();
+        String expResult = "test";
+        instance.setValue(expResult);
+        String result = instance.getValue();
+        assertEquals(expResult, result);
+    }
+
+    /**
+     * Test of isRegex method, of class PropertyType.
+     */
+    @Test
+    public void testIsRegex() {
+        PropertyType instance = new PropertyType();
+        boolean result = instance.isRegex();
+        assertFalse(instance.isRegex());
+        instance.setRegex(true);
+        assertTrue(instance.isRegex());
+    }
+
+    /**
+     * Test of isCaseSensitive method, of class PropertyType.
+     */
+    @Test
+    public void testIsCaseSensitive() {
+        PropertyType instance = new PropertyType();
+        assertFalse(instance.isCaseSensitive());
+        instance.setCaseSensitive(true);
+        assertTrue(instance.isCaseSensitive());
+    }
+
+    /**
+     * Test of matches method, of class PropertyType.
+     */
+    @Test
+    public void testMatches() {
+        String text = "Simple";
+
+        PropertyType instance = new PropertyType();
+        instance.setValue("simple");
+        assertTrue(instance.matches(text));
+        instance.setCaseSensitive(true);
+        assertFalse(instance.matches(text));
+
+        instance.setValue("s.*le");
+        instance.setRegex(true);
+        assertFalse(instance.matches(text));
+        instance.setCaseSensitive(false);
+        assertTrue(instance.matches(text));
+    }
+}