version 1.0.5

Former-commit-id: 3315c121f8adeeb5e4dc9fff9d2753bc5faf78fc

README.md

@@ -5,8 +5,70 @@ Dependency-Check is a utility that attempts to detect publicly disclosed vulnera
 
 Documentation and links to production binary releases can be found on the [github pages](http://jeremylong.github.io/DependencyCheck/). Additionally, more information about the architecture and ways to extend dependency-check can be found on the [wiki].
 
-Initial Usage
+Current Releases
 -------------
+### Jenkins Plugin
+
+For instructions on the use of the Jenkins plugin please see the [Jenkins dependency-check page](http://wiki.jenkins-ci.org/x/CwDgAQ).
+
+### Command Line
+
+More detailed instructions can be found on the [dependency-check github pages](http://jeremylong.github.io/DependencyCheck/dependency-check-cli/installation.html).
+The latest CLI can be downloaded from bintray's [dependency-check page](https://bintray.com/jeremy-long/owasp/dependency-check). 
+
+On *nix
+```
+$ ./bin/dependency-check.sh -h
+$ ./bin/dependency-check.sh --app Testing --out . --scan [path to jar files to be scanned]
+```
+On Windows
+```
+> bin/dependency-check.bat -h
+> bin/dependency-check.bat --app Testing --out . --scan [path to jar files to be scanned]
+```
+
+### Maven Plugin
+
+More detailed instructions can be found on the [dependency-check-maven github pages](http://jeremylong.github.io/DependencyCheck/dependency-check-maven/installation.html).
+The plugin can be configured using the following:
+
+```xml
+<project>
+    <build>
+        <plugins>
+            ...
+            <plugin>
+              <groupId>org.owasp</groupId>
+              <artifactId>dependency-check-maven</artifactId>
+              <version>1.0.2</version>
+              <executions>
+                  <execution>
+                      <goals>
+                          <goal>check</goal>
+                      </goals>
+                  </execution>
+              </executions>
+            </plugin>
+            ...
+        </plugins>
+        ...
+    </build>
+    ...
+</project>
+```
+
+### Ant Task
+
+For instructions on the use of the Ant Task, please see the [dependency-check-ant github page](http://jeremylong.github.io/DependencyCheck/dependency-check-maven/installation.html).
+
+Development Usage
+-------------
+The following instructions outline how to compile and use the current snapshot. While every intention is to maintain a stable snapshot it is recommended
+that the release versions listed above be used.
+
+Note, currently the install goal may take a long time to execute the integration tests. However, if this takes more then 30 minutes it is likely that the
+download of data from the NVD is having an issue. This issue is still being researched and a solution should be published soon.
+
 On *nix
 ```
 $ mvn install
@@ -20,7 +82,7 @@ On Windows
 > dependency-check-cli/target/release/bin/dependency-check.bat --app Testing --out . --scan ./src/test/resources
 ```
 
-Then load the resulting 'DependencyCheck-Report.html' into your favorite browser.
+Then load the resulting 'DependencyCheck-Report.html' into your favourite browser.
 
 Mailing List
 ------------
@@ -29,6 +91,8 @@ Subscribe: [dependency-check+subscribe@googlegroups.com] [subscribe]
 
 Post: [dependency-check@googlegroups.com] [post]
 
+Archive: [google group](https://groups.google.com/forum/#!forum/dependency-check)
+
 Copyright & License
 -
 

dependency-check-ant/README.md

@@ -1,29 +1,25 @@
-Dependency-Check-Ant
-===================
+Dependency-Check Ant Task
+=========
 
-Dependency-Check is a utility that attempts to detect publicly disclosed vulnerabilities contained within project dependencies. It does this by determining if there is a Common Platform Enumeration (CPE) identifier for a given dependency. If found, it will generate a report linking to the associated CVE entries.
+Dependency-Check Ant Task can be used to check the project dependencies for published security vulnerabilities. The checks
+performed are a "best effort" and as such, there could be false positives as well as false negatives. However,
+vulnerabilities in 3rd party components is a well-known problem and is currently documented in the 2013 OWASP
+Top 10 as [A9 - Using Components with Known Vulnerabilities](https://www.owasp.org/index.php/Top_10_2013-A9-Using_Components_with_Known_Vulnerabilities).
 
 Documentation and links to production binary releases can be found on the [github pages](http://jeremylong.github.io/DependencyCheck/dependency-check-ant/installation.html).
 
 Mailing List
 ------------
 
-Subscribe: [dependency-check+subscribe@googlegroups.com] [subscribe]
+Subscribe: [dependency-check+subscribe@googlegroups.com](mailto:dependency-check+subscribe@googlegroups.com)
 
-Post: [dependency-check@googlegroups.com] [post]
+Post: [dependency-check@googlegroups.com](mailto:dependency-check@googlegroups.com)
 
 Copyright & License
--
+-------------------
 
 Dependency-Check is Copyright (c) 2012-2013 Jeremy Long. All Rights Reserved.
 
-Permission to modify and redistribute is granted under the terms of the GPLv3 license. See the [LICENSE.txt] [GPLv3] file for the full license.
+Permission to modify and redistribute is granted under the terms of the GPLv3 license. See the [LICENSE.txt](https://github.com/jeremylong/DependencyCheck/dependency-check-ant/blob/master/LICENSE.txt) file for the full license.
 
-Dependency-Check makes use of several other open source libraries. Please see the [NOTICE.txt] [notices] file for more information.
-
-
-  [wiki]: https://github.com/jeremylong/DependencyCheck/wiki
-  [subscribe]: mailto:dependency-check+subscribe@googlegroups.com
-  [post]: mailto:dependency-check@googlegroups.com
-  [GPLv3]: https://github.com/jeremylong/DependencyCheck/blob/master/LICENSE.txt
-  [notices]: https://github.com/jeremylong/DependencyCheck/blob/master/NOTICES.txt
+Dependency-Check-Ant makes use of other open source libraries. Please see the [NOTICE.txt](https://github.com/jeremylong/DependencyCheck/dependency-check-ant/blob/master/NOTICES.txt) file for more information.

dependency-check-ant/pom.xml

@@ -22,13 +22,13 @@ Copyright (c) 2013 - Jeremy Long. All Rights Reserved.
     <parent>
         <groupId>org.owasp</groupId>
         <artifactId>dependency-check-parent</artifactId>
-        <version>1.0.1</version>
+        <version>1.0.5</version>
     </parent>
 
     <artifactId>dependency-check-ant</artifactId>
     <packaging>jar</packaging>
 
-    <name>dependency-check-ant</name>
+    <name>Dependency-Check Ant Task</name>
     <description>Dependency-check is a utility that attempts to detect publicly disclosed vulnerabilities contained within project dependencies. It does this by determining if there is a Common Platform Enumeration (CPE) identifier for a given dependency. If found, it will generate a report linking to the associated CVE entries.</description>
     <!-- begin copy from http://minds.coremedia.com/2012/09/11/problem-solved-deploy-multi-module-maven-project-site-as-github-pages/ -->
     <distributionManagement>
@@ -76,6 +76,25 @@ Copyright (c) 2013 - Jeremy Long. All Rights Reserved.
                 </configuration>
                 <executions>
                     <!-- the following executions are solely to setup the test environment -->
+                    <execution>
+                        <id>copy-test-data.zip</id>
+                        <phase>validate</phase>
+                        <goals>
+                            <goal>copy-resources</goal>
+                        </goals>
+                        <configuration>
+                            <outputDirectory>${project.build.directory}/test-classes</outputDirectory>
+                            <resources>
+                                <resource>
+                                    <directory>${basedir}/../src/test/resources</directory>
+                                    <filtering>false</filtering>
+                                    <includes>
+                                        <include>data.zip</include>
+                                    </includes>
+                                </resource>
+                            </resources>
+                        </configuration>
+                    </execution>
                     <execution>
                         <id>copy-test-resources-1</id>
                         <phase>validate</phase>
@@ -89,7 +108,7 @@ Copyright (c) 2013 - Jeremy Long. All Rights Reserved.
                                     <directory>${basedir}/../src/test/resources</directory>
                                     <filtering>false</filtering>
                                     <includes>
-                                        <include>axis2*.jar</include>
+                                        <include>org.mortbay.*.jar</include>
                                     </includes>
                                 </resource>
                             </resources>
@@ -127,7 +146,7 @@ Copyright (c) 2013 - Jeremy Long. All Rights Reserved.
                                     <directory>${basedir}/../src/test/resources</directory>
                                     <filtering>false</filtering>
                                     <includes>
-                                        <include>*.war</include>
+                                        <include>struts.jar</include>
                                     </includes>
                                 </resource>
                             </resources>
@@ -146,13 +165,32 @@ Copyright (c) 2013 - Jeremy Long. All Rights Reserved.
                                     <directory>${basedir}/../src/test/resources</directory>
                                     <filtering>false</filtering>
                                     <includes>
-                                        <include>jetty-6.1.0.jar</include>
                                         <include>org.mortbay.jetty.jar</include>
                                     </includes>
                                 </resource>
                             </resources>
                         </configuration>
                     </execution>
+                    <execution>
+                        <id>copy-data</id>
+                        <phase>validate</phase>
+                        <goals>
+                            <goal>copy-resources</goal>
+                        </goals>
+                        <configuration>
+                            <outputDirectory>${project.build.directory}/test-classes</outputDirectory>
+                            <resources>
+                                <resource>
+                                    <directory>${basedir}/../src/test/resources</directory>
+                                    <filtering>false</filtering>
+                                    <includes>
+                                        <include>db.cve.zip</include>
+                                        <include>index.cpe.zip</include>
+                                    </includes>
+                                </resource>
+                            </resources>
+                        </configuration>
+                    </execution>
                 </executions>
             </plugin>
             <plugin>
@@ -238,6 +276,10 @@ Copyright (c) 2013 - Jeremy Long. All Rights Reserved.
                             <value>${project.build.directory}/cobertura/cobertura.ser</value>
                             <workingDirectory>target</workingDirectory>
                         </property>
+                        <property>
+                            <name>data.directory</name>
+                            <value>${project.build.directory}/dependency-check-data</value>
+                        </property>
                     </systemProperties>
                 </configuration>
             </plugin>
@@ -389,6 +431,13 @@ Copyright (c) 2013 - Jeremy Long. All Rights Reserved.
             <artifactId>dependency-check-core</artifactId>
             <version>${project.parent.version}</version>
         </dependency>
+        <dependency>
+            <groupId>org.owasp</groupId>
+            <artifactId>dependency-check-core</artifactId>
+            <version>${project.parent.version}</version>
+            <type>test-jar</type>
+            <scope>test</scope>
+        </dependency>
         <dependency>
             <groupId>org.apache.ant</groupId>
             <artifactId>ant</artifactId>

dependency-check-ant/src/main/java/org/owasp/dependencycheck/taskdefs/DependencyCheckTask.java

@@ -23,7 +23,6 @@ import java.io.IOException;
 import java.io.InputStream;
 import java.util.List;
 import java.util.logging.Level;
-import java.util.logging.LogManager;
 import java.util.logging.Logger;
 import org.apache.tools.ant.BuildException;
 import org.apache.tools.ant.Task;
@@ -38,6 +37,7 @@ import org.owasp.dependencycheck.dependency.Dependency;
 import org.owasp.dependencycheck.dependency.Vulnerability;
 import org.owasp.dependencycheck.reporting.ReportGenerator;
 import org.owasp.dependencycheck.reporting.ReportGenerator.Format;
+import org.owasp.dependencycheck.utils.LogUtils;
 import org.owasp.dependencycheck.utils.Settings;
 
 /**
@@ -323,6 +323,53 @@ public class DependencyCheckTask extends Task {
     public void setProxyPort(String proxyPort) {
         this.proxyPort = proxyPort;
     }
+
+    /**
+     * The Proxy username.
+     */
+    private String proxyUsername;
+
+    /**
+     * Get the value of proxyUsername.
+     *
+     * @return the value of proxyUsername
+     */
+    public String getProxyUsername() {
+        return proxyUsername;
+    }
+
+    /**
+     * Set the value of proxyUsername.
+     *
+     * @param proxyUsername new value of proxyUsername
+     */
+    public void setProxyUsername(String proxyUsername) {
+        this.proxyUsername = proxyUsername;
+    }
+
+    /**
+     * The Proxy password.
+     */
+    private String proxyPassword;
+
+    /**
+     * Get the value of proxyPassword.
+     *
+     * @return the value of proxyPassword
+     */
+    public String getProxyPassword() {
+        return proxyPassword;
+    }
+
+    /**
+     * Set the value of proxyPassword.
+     *
+     * @param proxyPassword new value of proxyPassword
+     */
+    public void setProxyPassword(String proxyPassword) {
+        this.proxyPassword = proxyPassword;
+    }
+
     /**
      * The Connection Timeout.
      */
@@ -345,41 +392,33 @@ public class DependencyCheckTask extends Task {
     public void setConnectionTimeout(String connectionTimeout) {
         this.connectionTimeout = connectionTimeout;
     }
+    /**
+     * The file path used for verbose logging.
+     */
+    private String logFile = null;
 
     /**
-     * Configures the logger for use by the application.
+     * Get the value of logFile.
+     *
+     * @return the value of logFile
      */
-    private static void prepareLogger() {
-        InputStream in = null;
-        try {
-            in = DependencyCheckTask.class.getClassLoader().getResourceAsStream(LOG_PROPERTIES_FILE);
-            LogManager.getLogManager().reset();
-            LogManager.getLogManager().readConfiguration(in);
-            //TODO add code to disable fine grained log file.
-//            Logger logger = LogManager.getLogManager().getLogger("");
-//            for (Handler h : logger.getHandlers()) {
-//                if (h.getFormatter(). h.toString());
-//            }
-        } catch (IOException ex) {
-            System.err.println(ex.toString());
-            Logger.getLogger(DependencyCheckTask.class.getName()).log(Level.SEVERE, null, ex);
-        } catch (SecurityException ex) {
-            Logger.getLogger(DependencyCheckTask.class.getName()).log(Level.SEVERE, null, ex);
-        } finally {
-            if (in != null) {
-                try {
-                    in.close();
-                } catch (Exception ex) {
-                    //noinspection UnusedAssignment
-                    in = null;
-                }
-            }
-        }
+    public String getLogFile() {
+        return logFile;
+    }
+
+    /**
+     * Set the value of logFile.
+     *
+     * @param logFile new value of logFile
+     */
+    public void setLogFile(String logFile) {
+        this.logFile = logFile;
     }
 
     @Override
     public void execute() throws BuildException {
-        prepareLogger();
+        final InputStream in = DependencyCheckTask.class.getClassLoader().getResourceAsStream(LOG_PROPERTIES_FILE);
+        LogUtils.prepareLogger(in, logFile);
 
         dealWithReferences();
         validateConfiguration();
@@ -467,6 +506,12 @@ public class DependencyCheckTask extends Task {
         if (proxyPort != null && !proxyPort.isEmpty()) {
             Settings.setString(Settings.KEYS.PROXY_PORT, proxyPort);
         }
+        if (proxyUsername != null && !proxyUsername.isEmpty()) {
+            Settings.setString(Settings.KEYS.PROXY_USERNAME, proxyUsername);
+        }
+        if (proxyPassword != null && !proxyPassword.isEmpty()) {
+            Settings.setString(Settings.KEYS.PROXY_PASSWORD, proxyPassword);
+        }
         if (connectionTimeout != null && !connectionTimeout.isEmpty()) {
             Settings.setString(Settings.KEYS.CONNECTION_TIMEOUT, connectionTimeout);
         }
@@ -512,6 +557,7 @@ public class DependencyCheckTask extends Task {
          *
          * @return the list of values for the report format
          */
+        @Override
         public String[] getValues() {
             int i = 0;
             final Format[] formats = Format.values();

dependency-check-ant/src/main/resources/log.properties

@@ -4,7 +4,7 @@ handlers=java.util.logging.ConsoleHandler, java.util.logging.FileHandler
 # FINEST, FINER, FINE, CONFIG, INFO, WARNING and SEVERE.
 
 # Configure the ConsoleHandler.
-java.util.logging.ConsoleHandler.level=WARNING
+java.util.logging.ConsoleHandler.level=INFO
 
 #org.owasp.dependencycheck.data.nvdcve.xml
 

dependency-check-ant/src/site/markdown/configuration.md

@@ -28,8 +28,11 @@ ReportOutputDirectory | The directory where dependency-check will store data use
 FailBuildOn           | If set and a CVE is found that is greater then the specified value the build will fail. The default value is 11 which means that the build will not fail. Valid values are 0-11. | Optional
 AutoUpdate            | If set to false the NVD CVE data is not automatically updated. Setting this to false could result in false negatives. However, this may be required in some environments. The default value is true. | Optional
 DataDirectory         | The directory where dependency-check will store data used for analysis. Defaults to a folder called, called 'dependency-check-data', that is in the same directory as the dependency-check-ant jar file was installed in. *It is not recommended to change this.* | Optional
+LogFile               | The file path to write verbose logging information. | Optional
 ProxyUrl              | Defines the proxy used to connect to the Internet. | Optional
 ProxyPort             | Defines the port for the proxy. | Optional
+ProxyUsername         | Defines the proxy user name. | Optional
+ProxyPassword         | Defines the proxy password. | Optional
 ConnectionTimeout     | The connection timeout used when downloading data files from the Internet. | Optional
 
 

dependency-check-ant/src/site/markdown/installation.md.vm

@@ -1,6 +1,6 @@
 Installation
 ====================
-Download dependency-check-ant from [bintray here](http://dl.bintray.com/jeremy-long/owasp/dependency-check-ant-1.0.0.jar).
+Download dependency-check-ant from [bintray here](http://dl.bintray.com/jeremy-long/owasp/dependency-check-ant-${project.version}.jar).
 To install dependency-check-ant place the dependency-check-ant-${project.version}.jar into
 the lib directory of your Ant instalation directory. Once installed you can add
 the taskdef to you build.xml and add the task to a new or existing target.

dependency-check-ant/src/test/java/org/owasp/dependencycheck/taskdefs/DependencyCheckTaskTest.java

@@ -26,6 +26,7 @@ import org.junit.Before;
 import org.junit.BeforeClass;
 import org.junit.Test;
 import org.apache.tools.ant.BuildFileTest;
+import org.owasp.dependencycheck.data.nvdcve.BaseDBTestCase;
 
 /**
  *
@@ -46,7 +47,8 @@ public class DependencyCheckTaskTest extends BuildFileTest {
 
     @Before
     @Override
-    public void setUp() {
+    public void setUp() throws Exception {
+        BaseDBTestCase.ensureDBExists();
         final String buildFile = this.getClass().getClassLoader().getResource("build.xml").getPath();
         configureProject(buildFile);
     }

dependency-check-cli/README.md

@@ -1,29 +1,24 @@
-Dependency-Check
+Dependency-Check Command Line
 ================
-
-Dependency-Check is a utility that attempts to detect publicly disclosed vulnerabilities contained within project dependencies. It does this by determining if there is a Common Platform Enumeration (CPE) identifier for a given dependency. If found, it will generate a report linking to the associated CVE entries.
+Dependency-Check Command Line can be used to check project dependencies for published security vulnerabilities. The checks
+performed are a "best effort" and as such, there could be false positives as well as false negatives. However,
+vulnerabilities in 3rd party components is a well-known problem and is currently documented in the 2013 OWASP
+Top 10 as [A9 - Using Components with Known Vulnerabilities](https://www.owasp.org/index.php/Top_10_2013-A9-Using_Components_with_Known_Vulnerabilities).
 
 Documentation and links to production binary releases can be found on the [github pages](http://jeremylong.github.io/DependencyCheck/dependency-check-cli/installation.html).
 
 Mailing List
 ------------
 
-Subscribe: [dependency-check+subscribe@googlegroups.com] [subscribe]
+Subscribe: [dependency-check+subscribe@googlegroups.com](mailto:dependency-check+subscribe@googlegroups.com)
 
-Post: [dependency-check@googlegroups.com] [post]
+Post: [dependency-check@googlegroups.com](mailto:dependency-check@googlegroups.com)
 
 Copyright & License
 ------------
 
 Dependency-Check is Copyright (c) 2012-2013 Jeremy Long. All Rights Reserved.
 
-Permission to modify and redistribute is granted under the terms of the GPLv3 license. See the [LICENSE.txt] [GPLv3] file for the full license.
+Permission to modify and redistribute is granted under the terms of the GPLv3 license. See the [LICENSE.txt](https://github.com/jeremylong/DependencyCheck/dependency-check-cli/blob/master/LICENSE.txt) file for the full license.
 
-Dependency-Check makes use of several other open source libraries. Please see the [NOTICE.txt] [notices] file for more information.
-
-
-  [wiki]: https://github.com/jeremylong/DependencyCheck/wiki
-  [subscribe]: mailto:dependency-check+subscribe@googlegroups.com
-  [post]: mailto:dependency-check@googlegroups.com
-  [GPLv3]: https://github.com/jeremylong/DependencyCheck/blob/master/LICENSE.txt
-  [notices]: https://github.com/jeremylong/DependencyCheck/blob/master/NOTICES.txt
+Dependency-Check Command Line makes use of other open source libraries. Please see the [NOTICE.txt](https://github.com/jeremylong/DependencyCheck/dependency-check-cli/blob/master/NOTICES.txt) file for more information.

dependency-check-cli/pom.xml

@@ -22,13 +22,13 @@ Copyright (c) 2012 - Jeremy Long. All Rights Reserved.
     <parent>
         <groupId>org.owasp</groupId>
         <artifactId>dependency-check-parent</artifactId>
-        <version>1.0.1</version>
+        <version>1.0.5</version>
     </parent>
 
     <artifactId>dependency-check-cli</artifactId>
     <packaging>jar</packaging>
 
-    <name>dependency-check-cli</name>
+    <name>Dependency-Check Command Line</name>
     <description>Dependency-Check-Maven is a Maven Plugin that attempts to detect publicly disclosed vulnerabilities contained within project dependencies. It does this by determining if there is a Common Platform Enumeration (CPE) identifier for a given dependency. If found, it will generate a report linking to the associated CVE entries.</description>
     <!-- begin copy from http://minds.coremedia.com/2012/09/11/problem-solved-deploy-multi-module-maven-project-site-as-github-pages/ -->
     <distributionManagement>

dependency-check-cli/src/main/java/org/owasp/dependencycheck/App.java

@@ -24,12 +24,12 @@ import java.io.IOException;
 import java.io.InputStream;
 import java.util.List;
 import java.util.logging.Level;
-import java.util.logging.LogManager;
 import java.util.logging.Logger;
 import org.apache.commons.cli.ParseException;
 import org.owasp.dependencycheck.reporting.ReportGenerator;
 import org.owasp.dependencycheck.dependency.Dependency;
 import org.owasp.dependencycheck.cli.CliParser;
+import org.owasp.dependencycheck.utils.LogUtils;
 import org.owasp.dependencycheck.utils.Settings;
 
 /*
@@ -67,35 +67,10 @@ public class App {
      * @param args the command line arguments
      */
     public static void main(String[] args) {
-        prepareLogger();
         final App app = new App();
         app.run(args);
     }
 
-    /**
-     * Configures the logger for use by the application.
-     */
-    private static void prepareLogger() {
-        InputStream in = null;
-        try {
-            in = App.class.getClassLoader().getResourceAsStream(LOG_PROPERTIES_FILE);
-            LogManager.getLogManager().reset();
-            LogManager.getLogManager().readConfiguration(in);
-        } catch (IOException ex) {
-            Logger.getLogger(App.class.getName()).log(Level.FINE, "IO Error preparing the logger", ex);
-        } catch (SecurityException ex) {
-            Logger.getLogger(App.class.getName()).log(Level.FINE, "Error preparing the logger", ex);
-        } finally {
-            if (in != null) {
-                try {
-                    in.close();
-                } catch (Exception ex) {
-                    Logger.getLogger(App.class.getName()).log(Level.FINEST, "Error closing resource stream", ex);
-                }
-            }
-        }
-    }
-
     /**
      * Main CLI entry-point into the application.
      *
@@ -116,10 +91,15 @@ public class App {
             return;
         }
 
+        final InputStream in = App.class.getClassLoader().getResourceAsStream(LOG_PROPERTIES_FILE);
+        LogUtils.prepareLogger(in, cli.getVerboseLog());
+
         if (cli.isGetVersion()) {
             cli.printVersionInfo();
         } else if (cli.isRunScan()) {
-            updateSettings(cli.isAutoUpdate(), cli.getConnectionTimeout(), cli.getProxyUrl(), cli.getProxyPort(), cli.getDataDirectory());
+            updateSettings(cli.isAutoUpdate(), cli.getConnectionTimeout(), cli.getProxyUrl(),
+                    cli.getProxyPort(), cli.getProxyUsername(), cli.getProxyPassword(),
+                    cli.getDataDirectory(), cli.getPropertiesFile());
             runScan(cli.getReportDirectory(), cli.getReportFormat(), cli.getApplicationName(), cli.getScanFiles());
         } else {
             cli.printHelp();
@@ -168,8 +148,24 @@ public class App {
      * @param proxyPort the proxy port (null or blank means no port will be
      * used)
      * @param dataDirectory the directory to store/retrieve persistent data from
+     * @param propertiesFile the properties file to utilize
      */
-    private void updateSettings(boolean autoUpdate, String connectionTimeout, String proxyUrl, String proxyPort, String dataDirectory) {
+    private void updateSettings(boolean autoUpdate, String connectionTimeout, String proxyUrl, String proxyPort,
+                                String proxyUser, String proxyPass, String dataDirectory, File propertiesFile) {
+
+        if (propertiesFile != null) {
+            try {
+                Settings.mergeProperties(propertiesFile);
+            } catch (FileNotFoundException ex) {
+                final String msg = String.format("Unable to load properties file '%s'", propertiesFile.getPath());
+                Logger.getLogger(App.class.getName()).log(Level.SEVERE, msg);
+                Logger.getLogger(App.class.getName()).log(Level.FINE, null, ex);
+            } catch (IOException ex) {
+                final String msg = String.format("Unable to find properties file '%s'", propertiesFile.getPath());
+                Logger.getLogger(App.class.getName()).log(Level.SEVERE, msg);
+                Logger.getLogger(App.class.getName()).log(Level.FINE, null, ex);
+            }
+        }
         if (dataDirectory != null) {
             Settings.setString(Settings.KEYS.DATA_DIRECTORY, dataDirectory);
         } else if (System.getProperty("basedir") != null) {
@@ -182,8 +178,6 @@ public class App {
             final File dataDir = new File(base, sub);
             Settings.setString(Settings.KEYS.DATA_DIRECTORY, dataDir.getAbsolutePath());
         }
-
-
         Settings.setBoolean(Settings.KEYS.AUTO_UPDATE, autoUpdate);
         if (proxyUrl != null && !proxyUrl.isEmpty()) {
             Settings.setString(Settings.KEYS.PROXY_URL, proxyUrl);
@@ -191,6 +185,12 @@ public class App {
         if (proxyPort != null && !proxyPort.isEmpty()) {
             Settings.setString(Settings.KEYS.PROXY_PORT, proxyPort);
         }
+        if (proxyUser != null && !proxyUser.isEmpty()) {
+            Settings.setString(Settings.KEYS.PROXY_USERNAME, proxyUser);
+        }
+        if (proxyPass != null && !proxyPass.isEmpty()) {
+            Settings.setString(Settings.KEYS.PROXY_PASSWORD, proxyPass);
+        }
         if (connectionTimeout != null && !connectionTimeout.isEmpty()) {
             Settings.setString(Settings.KEYS.CONNECTION_TIMEOUT, connectionTimeout);
         }

dependency-check-cli/src/main/java/org/owasp/dependencycheck/cli/CliParser.java

@@ -175,6 +175,14 @@ public final class CliParser {
                 .withDescription("The proxy port to use when downloading resources.")
                 .create(ArgumentName.PROXY_PORT_SHORT);
 
+        final Option proxyUsername = OptionBuilder.withArgName("user").hasArg().withLongOpt(ArgumentName.PROXY_USERNAME)
+                .withDescription("The proxy username to use when downloading resources.")
+                .create(ArgumentName.PROXY_USERNAME_SHORT);
+
+        final Option proxyPassword = OptionBuilder.withArgName("pass").hasArg().withLongOpt(ArgumentName.PROXY_PASSWORD)
+                .withDescription("The proxy password to use when downloading resources.")
+                .create(ArgumentName.PROXY_PASSWORD_SHORT);
+
         final Option path = OptionBuilder.withArgName("path").hasArg().withLongOpt(ArgumentName.SCAN)
                 .withDescription("The path to scan - this option can be specified multiple times.")
                 .create(ArgumentName.SCAN_SHORT);
@@ -195,6 +203,10 @@ public final class CliParser {
                 .withDescription("The output format to write to (XML, HTML, VULN, ALL). The default is HTML.")
                 .create(ArgumentName.OUTPUT_FORMAT_SHORT);
 
+        final Option verboseLog = OptionBuilder.withArgName("file").hasArg().withLongOpt(ArgumentName.VERBOSE_LOG)
+                .withDescription("The file path to write verbose logging information.")
+                .create(ArgumentName.VERBOSE_LOG_SHORT);
+
         final OptionGroup og = new OptionGroup();
         og.addOption(path);
 
@@ -208,8 +220,11 @@ public final class CliParser {
         opts.addOption(noUpdate);
         opts.addOption(props);
         opts.addOption(data);
+        opts.addOption(verboseLog);
         opts.addOption(proxyPort);
         opts.addOption(proxyUrl);
+        opts.addOption(proxyUsername);
+        opts.addOption(proxyPassword);
         opts.addOption(connectionTimeout);
 
         return opts;
@@ -325,6 +340,24 @@ public final class CliParser {
         return line.getOptionValue(ArgumentName.PROXY_PORT);
     }
 
+    /**
+     * Returns the proxy username.
+     *
+     * @return the proxy username
+     */
+    public String getProxyUsername() {
+        return line.getOptionValue(ArgumentName.PROXY_USERNAME);
+    }
+
+    /**
+     * Returns the proxy password.
+     *
+     * @return the proxy password
+     */
+    public String getProxyPassword() {
+        return line.getOptionValue(ArgumentName.PROXY_PASSWORD);
+    }
+
     /**
      * Get the value of dataDirectory.
      *
@@ -334,6 +367,28 @@ public final class CliParser {
         return line.getOptionValue(ArgumentName.DATA_DIRECTORY);
     }
 
+    /**
+     * Returns the properties file specified on the command line.
+     *
+     * @return the properties file specified on the command line
+     */
+    public File getPropertiesFile() {
+        final String path = line.getOptionValue(ArgumentName.PROP);
+        if (path != null) {
+            return new File(path);
+        }
+        return null;
+    }
+
+    /**
+     * Returns the path to the verbose log file.
+     *
+     * @return the path to the verbose log file
+     */
+    public String getVerboseLog() {
+        return line.getOptionValue(ArgumentName.VERBOSE_LOG);
+    }
+
     /**
      * <p>Prints the manifest information to standard output.</p>
      * <ul><li>Implementation-Title: ${pom.name}</li>
@@ -443,11 +498,27 @@ public final class CliParser {
          */
         public static final String PROXY_URL = "proxyurl";
         /**
-         * The short CLI argument name indicating the proxy url.
+         * The short CLI argument name indicating the proxy username.
+         */
+        public static final String PROXY_USERNAME_SHORT = "pu";
+        /**
+         * The CLI argument name indicating the proxy username.
+         */
+        public static final String PROXY_USERNAME = "proxyuser";
+        /**
+         * The short CLI argument name indicating the proxy password.
+         */
+        public static final String PROXY_PASSWORD_SHORT = "pp";
+        /**
+         * The CLI argument name indicating the proxy password.
+         */
+        public static final String PROXY_PASSWORD = "proxypass";
+        /**
+         * The short CLI argument name indicating the connection timeout.
          */
         public static final String CONNECTION_TIMEOUT_SHORT = "c";
         /**
-         * The CLI argument name indicating the proxy url.
+         * The CLI argument name indicating the connection timeout.
          */
         public static final String CONNECTION_TIMEOUT = "connectiontimeout";
         /**
@@ -469,5 +540,14 @@ public final class CliParser {
          * directory.
          */
         public static final String DATA_DIRECTORY_SHORT = "d";
+        /**
+         * The CLI argument name for setting the location of the data directory.
+         */
+        public static final String VERBOSE_LOG = "log";
+        /**
+         * The short CLI argument name for setting the location of the data
+         * directory.
+         */
+        public static final String VERBOSE_LOG_SHORT = "l";
     }
 }

dependency-check-cli/src/main/resources/log.properties

@@ -7,8 +7,6 @@ handlers=java.util.logging.ConsoleHandler
 # Configure the ConsoleHandler.
 java.util.logging.ConsoleHandler.level=INFO
 
-org.owasp.dependencycheck.data.nvdcve.xml
-
 # Configure the FileHandler.
 java.util.logging.FileHandler.formatter=java.util.logging.SimpleFormatter
 java.util.logging.FileHandler.level=FINE
@@ -21,4 +19,4 @@ java.util.logging.FileHandler.level=FINE
 #   %g - generation number for rotating logs
 #   %u - unique number to avoid conflicts
 # FileHandler writes to %h/demo0.log by default.
-java.util.logging.FileHandler.pattern=./logs/DependencyCheck.log
+java.util.logging.FileHandler.pattern=./dependency-check.log

dependency-check-cli/src/site/markdown/installation.md.vm

@@ -1,6 +1,6 @@
 Installation & Usage
 --------------------
-Downlod the dependency-check command line tool [here](http://dl.bintray.com/jeremy-long/owasp/dependency-check-1.0.0-release.zip).
+Downlod the dependency-check command line tool [here](http://dl.bintray.com/jeremy-long/owasp/dependency-check-${project.version}-release.zip).
 Extract the zip file to a location on your computer and put the 'bin' directory into the
 path environment variable. On \*nix systems you will likely need to make the shell
 script executable:

dependency-check-core/pom.xml

@@ -22,13 +22,13 @@ along with Dependency-Check.  If not, see <http://www.gnu.org/licenses />.
     <parent>
         <groupId>org.owasp</groupId>
         <artifactId>dependency-check-parent</artifactId>
-        <version>1.0.1</version>
+        <version>1.0.5</version>
     </parent>
 
     <artifactId>dependency-check-core</artifactId>
     <packaging>jar</packaging>
 
-    <name>dependency-check-core</name>
+    <name>Dependency-Check Core</name>
     <!-- begin copy from http://minds.coremedia.com/2012/09/11/problem-solved-deploy-multi-module-maven-project-site-as-github-pages/ -->
     <distributionManagement>
         <site>
@@ -44,6 +44,7 @@ along with Dependency-Check.  If not, see <http://www.gnu.org/licenses />.
                 <directory>src/main/resources</directory>
                 <includes>
                     <include>**/*.properties</include>
+                    <include>**/schema/*.xsd</include>
                 </includes>
                 <filtering>true</filtering>
             </resource>
@@ -83,16 +84,16 @@ along with Dependency-Check.  If not, see <http://www.gnu.org/licenses />.
             <plugin>
                 <groupId>org.apache.maven.plugins</groupId>
                 <artifactId>maven-dependency-plugin</artifactId>
-                <version>2.7</version>
+                <version>2.8</version>
                 <executions>
                     <execution>
-                        <phase>package</phase>
+                        <phase>generate-resources</phase>
                         <goals>
                             <goal>copy-dependencies</goal>
                         </goals>
                         <configuration>
-                            <outputDirectory>${project.build.directory}/lib</outputDirectory>
-                            <excludeScope>provided</excludeScope>
+                            <outputDirectory>${project.build.directory}/test-classes</outputDirectory>
+                            <includeScope>provided</includeScope>
                         </configuration>
                     </execution>
                 </executions>
@@ -101,6 +102,22 @@ along with Dependency-Check.  If not, see <http://www.gnu.org/licenses />.
                 <groupId>org.apache.maven.plugins</groupId>
                 <artifactId>maven-jar-plugin</artifactId>
                 <version>2.4</version>
+                <executions>
+                    <execution>
+                        <id>jar</id>
+                        <phase>package</phase>
+                        <goals>
+                            <goal>jar</goal>
+                        </goals>
+                    </execution>
+                    <execution>
+                        <id>test-jar</id>
+                        <phase>package</phase>
+                        <goals>
+                            <goal>test-jar</goal>
+                        </goals>
+                    </execution>
+                </executions>
                 <configuration>
                     <archive>
                         <manifest>
@@ -177,6 +194,10 @@ along with Dependency-Check.  If not, see <http://www.gnu.org/licenses />.
                             <name>data.directory</name>
                             <value>${project.build.directory}/data</value>
                         </property>
+                        <property>
+                            <name>temp.directory</name>
+                            <value>${project.build.directory}/temp</value>
+                        </property>
                     </systemProperties>
                     <excludes>
                         <exclude>**/*IntegrationTest.java</exclude>
@@ -369,7 +390,7 @@ along with Dependency-Check.  If not, see <http://www.gnu.org/licenses />.
             <groupId>com.google.code.findbugs</groupId>
             <artifactId>annotations</artifactId>
             <version>2.0.1</version>
-            <scope>provided</scope><!-- don't include this in the libs-->
+            <optional>true</optional>
         </dependency>
         <dependency>
             <groupId>commons-cli</groupId>
@@ -461,12 +482,91 @@ along with Dependency-Check.  If not, see <http://www.gnu.org/licenses />.
             <version>1.7.2</version>
             <type>jar</type>
         </dependency>
-        <!-- The following dependencies are only scanned during integration testing -->
-        <!--<dependency>
+        <dependency>
+            <groupId>org.apache.commons</groupId>
+            <artifactId>commons-compress</artifactId>
+            <version>1.5</version>
+        </dependency>
+        <!-- The following dependencies are only used during testing -->
+        <dependency>
+            <groupId>org.apache.maven.scm</groupId>
+            <artifactId>maven-scm-provider-cvsexe</artifactId>
+            <version>1.8.1</version>
+            <scope>provided</scope>
+            <optional>true</optional>
+        </dependency>
+        <dependency>
             <groupId>org.springframework</groupId>
             <artifactId>spring-webmvc</artifactId>
             <version>2.5.5</version>
-            <scope>test</scope>
-        </dependency>-->
+            <scope>provided</scope>
+            <optional>true</optional>
+        </dependency>
+        <dependency>
+            <groupId>com.hazelcast</groupId>
+            <artifactId>hazelcast</artifactId>
+            <version>2.5</version>
+            <scope>provided</scope>
+            <optional>true</optional>
+        </dependency>
+        <dependency>
+            <groupId>net.sf.ehcache</groupId>
+            <artifactId>ehcache-core</artifactId>
+            <version>2.2.0</version>
+            <scope>provided</scope>
+            <optional>true</optional>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.struts</groupId>
+            <artifactId>struts2-core</artifactId>
+            <version>2.1.2</version>
+            <scope>provided</scope>
+            <optional>true</optional>
+        </dependency>
+        <dependency>
+            <groupId>org.mortbay.jetty</groupId>
+            <artifactId>jetty</artifactId>
+            <version>6.1.0</version>
+            <scope>provided</scope>
+            <optional>true</optional>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.axis2</groupId>
+            <artifactId>axis2-spring</artifactId>
+            <version>1.4.1</version>
+            <scope>provided</scope>
+            <optional>true</optional>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.axis2</groupId>
+            <artifactId>axis2-adb</artifactId>
+            <version>1.4.1</version>
+            <scope>provided</scope>
+            <optional>true</optional>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.geronimo.daytrader</groupId>
+            <artifactId>daytrader-ear</artifactId>
+            <version>2.1.7</version>
+            <type>ear</type>
+            <scope>provided</scope>
+            <optional>true</optional>
+        </dependency>
+        <dependency>
+            <groupId>org.glassfish.main.admingui</groupId>
+            <artifactId>war</artifactId>
+            <version>4.0</version>
+            <type>war</type>
+            <scope>provided</scope>
+            <optional>true</optional>
+        </dependency>
+        <dependency>
+            <groupId>org.dojotoolkit</groupId>
+            <artifactId>dojo-war</artifactId>
+            <version>1.3.0</version>
+            <type>war</type>
+            <scope>provided</scope>
+            <optional>true</optional>
+        </dependency>
     </dependencies>
 </project>

dependency-check-core/src/main/java/org/owasp/dependencycheck/Engine.java

@@ -20,6 +20,7 @@ package org.owasp.dependencycheck;
 
 import java.util.EnumMap;
 import java.io.File;
+import java.io.IOException;
 import java.util.ArrayList;
 import java.util.HashSet;
 import java.util.Iterator;
@@ -32,8 +33,10 @@ import org.owasp.dependencycheck.analyzer.AnalysisPhase;
 import org.owasp.dependencycheck.analyzer.Analyzer;
 import org.owasp.dependencycheck.analyzer.AnalyzerService;
 import org.owasp.dependencycheck.data.CachedWebDataSource;
+import org.owasp.dependencycheck.data.NoDataException;
 import org.owasp.dependencycheck.data.UpdateException;
 import org.owasp.dependencycheck.data.UpdateService;
+import org.owasp.dependencycheck.data.cpe.CpeIndexReader;
 import org.owasp.dependencycheck.dependency.Dependency;
 import org.owasp.dependencycheck.utils.FileUtils;
 import org.owasp.dependencycheck.utils.InvalidSettingException;
@@ -277,16 +280,28 @@ public class Engine {
      * Runs the analyzers against all of the dependencies.
      */
     public void analyzeDependencies() {
+        //need to ensure that data exists
+        try {
+            ensureDataExists();
+        } catch (NoDataException ex) {
+            final String msg = String.format("%n%n%s%n%nUnable to continue dependency-check analysis.", ex.getMessage());
+            Logger.getLogger(Engine.class.getName()).log(Level.SEVERE, msg);
+            Logger.getLogger(Engine.class.getName()).log(Level.FINE, null, ex);
+            return;
+        }
+
         //phase one initialize
         for (AnalysisPhase phase : AnalysisPhase.values()) {
             final List<Analyzer> analyzerList = analyzers.get(phase);
             for (Analyzer a : analyzerList) {
                 try {
+                    final String msg = String.format("Initializing %s", a.getName());
+                    Logger.getLogger(Engine.class.getName()).log(Level.FINE, msg);
                     a.initialize();
                 } catch (Exception ex) {
-                    final String msg = String.format("\"Exception occurred initializing \"%s\".\"", a.getName());
+                    final String msg = String.format("Exception occurred initializing %s.", a.getName());
                     Logger.getLogger(Engine.class.getName()).log(Level.SEVERE, msg);
-                    Logger.getLogger(Engine.class.getName()).log(Level.INFO, msg, ex);
+                    Logger.getLogger(Engine.class.getName()).log(Level.INFO, null, ex);
                     try {
                         a.close();
                     } catch (Exception ex1) {
@@ -305,9 +320,13 @@ public class Engine {
                  * analyzers may modify it. This prevents ConcurrentModificationExceptions.
                  * This is okay for adds/deletes because it happens per analyzer.
                  */
+                final String msg = String.format("Begin Analyzer '%s'", a.getName());
+                Logger.getLogger(Engine.class.getName()).log(Level.FINE, msg);
                 final Set<Dependency> dependencySet = new HashSet<Dependency>();
                 dependencySet.addAll(dependencies);
                 for (Dependency d : dependencySet) {
+                    final String msgFile = String.format("Begin Analysis of '%s'", d.getActualFilePath());
+                    Logger.getLogger(Engine.class.getName()).log(Level.FINE, msgFile);
                     if (a.supportsExtension(d.getFileExtension())) {
                         try {
                             a.analyze(d, this);
@@ -323,6 +342,8 @@ public class Engine {
         for (AnalysisPhase phase : AnalysisPhase.values()) {
             final List<Analyzer> analyzerList = analyzers.get(phase);
             for (Analyzer a : analyzerList) {
+                final String msg = String.format("Closing Analyzer '%s'", a.getName());
+                Logger.getLogger(Engine.class.getName()).log(Level.FINE, msg);
                 try {
                     a.close();
                 } catch (Exception ex) {
@@ -388,4 +409,37 @@ public class Engine {
         }
         return false;
     }
+
+    /**
+     * Checks the CPE Index to ensure documents exists. If none exist a
+     * NoDataException is thrown.
+     *
+     * @throws NoDataException thrown if no data exists in the CPE Index
+     */
+    private void ensureDataExists() throws NoDataException {
+        CpeIndexReader cpe = null;
+        boolean noDataExists = false;
+        try {
+            cpe = new CpeIndexReader();
+            cpe.open();
+            if (cpe.numDocs() <= 0) {
+                noDataExists = true;
+            }
+        } catch (IOException ex) {
+            noDataExists = true;
+        } catch (NullPointerException ex) {
+            noDataExists = true;
+        } finally {
+            if (cpe != null) {
+                cpe.close();
+            }
+        }
+        if (noDataExists) {
+            throw new NoDataException("No data exists in the data store. Please check that you are able to connect "
+                    + "to the Internet and re-run dependency-check. If the problem persists determine whether you need "
+                    + "to set a proxy url and port.\\n\\nIf you are unable to solve this problem please contact the mailing "
+                    + "list for help: dependency-check@googlegroups.com");
+
+        }
+    }
 }

dependency-check-core/src/main/java/org/owasp/dependencycheck/analyzer/ArchiveAnalyzer.java

@@ -32,26 +32,21 @@ import java.util.List;
 import java.util.Set;
 import java.util.logging.Level;
 import java.util.logging.Logger;
-import java.util.zip.ZipEntry;
-import java.util.zip.ZipInputStream;
+import org.apache.commons.compress.archivers.ArchiveEntry;
+import org.apache.commons.compress.archivers.ArchiveInputStream;
+import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;
+import org.apache.commons.compress.archivers.zip.ZipArchiveInputStream;
+import org.apache.commons.compress.compressors.CompressorInputStream;
+import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream;
+import org.apache.commons.compress.compressors.gzip.GzipUtils;
 import org.h2.store.fs.FileUtils;
 import org.owasp.dependencycheck.Engine;
 import org.owasp.dependencycheck.dependency.Dependency;
 import org.owasp.dependencycheck.utils.Settings;
 
 /**
- * <p>An analyzer that works on archive files:
- * <ul>
- * <li><b>ZIP</b> - if it is determined to be a JAR, WAR or EAR a copy is made
- * and the copy is given the correct extension so that it will be correctly
- * analyzed.</li>
- * <li><b>WAR</b> - the WAR contents are extracted and added as dependencies to
- * the scan. The displayed path is relative to the WAR.</li>
- * <li><b>EAR</b> - the WAR contents are extracted and added as dependencies to
- * the scan. Any WAR files are also processed so that the contained JAR files
- * are added to the list of dependencies. The displayed path is relative to the
- * EAR.</li>
- * </ul></p>
+ * <p>An analyzer that extracts files from archives and ensures any supported
+ * files contained within the archive are added to the dependency list.</p>
  *
  * @author Jeremy Long (jeremy.long@owasp.org)
  */
@@ -91,7 +86,7 @@ public class ArchiveAnalyzer extends AbstractAnalyzer implements Analyzer {
     /**
      * The set of file extensions supported by this analyzer.
      */
-    private static final Set<String> EXTENSIONS = newHashSet("zip", "ear", "war");
+    private static final Set<String> EXTENSIONS = newHashSet("zip", "ear", "war", "tar", "gz", "tgz");
 
     /**
      * Returns a list of file EXTENSIONS supported by this analyzer.
@@ -140,14 +135,21 @@ public class ArchiveAnalyzer extends AbstractAnalyzer implements Analyzer {
      */
     @Override
     public void initialize() throws Exception {
-        final String tmpDir = Settings.getString(Settings.KEYS.TEMP_DIRECTORY, System.getProperty("java.io.tmpdir"));
-        final File baseDir = new File(tmpDir);
+        final File baseDir = Settings.getTempDirectory();
+        if (!baseDir.exists()) {
+            if (!baseDir.mkdirs()) {
+                final String msg = String.format("Unable to make a temporary folder '%s'", baseDir.getPath());
+                throw new AnalysisException(msg);
+            }
+        }
         tempFileLocation = File.createTempFile("check", "tmp", baseDir);
         if (!tempFileLocation.delete()) {
-            throw new AnalysisException("Unable to delete temporary file '" + tempFileLocation.getAbsolutePath() + "'.");
+            final String msg = String.format("Unable to delete temporary file '%s'.", tempFileLocation.getAbsolutePath());
+            throw new AnalysisException(msg);
         }
         if (!tempFileLocation.mkdirs()) {
-            throw new AnalysisException("Unable to create directory '" + tempFileLocation.getAbsolutePath() + "'.");
+            final String msg = String.format("Unable to create directory '%s'.", tempFileLocation.getAbsolutePath());
+            throw new AnalysisException(msg);
         }
     }
 
@@ -223,8 +225,13 @@ public class ArchiveAnalyzer extends AbstractAnalyzer implements Analyzer {
     private File getNextTempDirectory() throws AnalysisException {
         dirCount += 1;
         final File directory = new File(tempFileLocation, String.valueOf(dirCount));
+        //getting an exception for some directories not being able to be created; might be because the directory already exists?
+        if (directory.exists()) {
+            return getNextTempDirectory();
+        }
         if (!directory.mkdirs()) {
-            throw new AnalysisException("Unable to create temp directory '" + directory.getAbsolutePath() + "'.");
+            final String msg = String.format("Unable to create temp directory '%s'.", directory.getAbsolutePath());
+            throw new AnalysisException(msg);
         }
         return directory;
     }
@@ -233,35 +240,75 @@ public class ArchiveAnalyzer extends AbstractAnalyzer implements Analyzer {
      * Extracts the contents of an archive into the specified directory.
      *
      * @param archive an archive file such as a WAR or EAR
-     * @param extractTo a directory to extract the contents to
+     * @param destination a directory to extract the contents to
      * @param engine the scanning engine
      * @throws AnalysisException thrown if the archive is not found
      */
-    private void extractFiles(File archive, File extractTo, Engine engine) throws AnalysisException {
-        if (archive == null || extractTo == null) {
+    private void extractFiles(File archive, File destination, Engine engine) throws AnalysisException {
+        if (archive == null || destination == null) {
             return;
         }
 
         FileInputStream fis = null;
-        ZipInputStream zis = null;
-
         try {
             fis = new FileInputStream(archive);
         } catch (FileNotFoundException ex) {
             Logger.getLogger(ArchiveAnalyzer.class.getName()).log(Level.INFO, null, ex);
             throw new AnalysisException("Archive file was not found.", ex);
         }
-        zis = new ZipInputStream(new BufferedInputStream(fis));
-        ZipEntry entry;
+        final String archiveExt = org.owasp.dependencycheck.utils.FileUtils.getFileExtension(archive.getName()).toLowerCase();
         try {
-            while ((entry = zis.getNextEntry()) != null) {
+            if ("zip".equals(archiveExt) || "war".equals(archiveExt) || "ear".equals(archiveExt)) {
+                extractArchive(new ZipArchiveInputStream(new BufferedInputStream(fis)), destination, engine);
+            } else if ("tar".equals(archiveExt)) {
+                extractArchive(new TarArchiveInputStream(new BufferedInputStream(fis)), destination, engine);
+            } else if ("gz".equals(archiveExt) || "tgz".equals(archiveExt)) {
+                final String uncompressedName = GzipUtils.getUncompressedFilename(archive.getName());
+                final String uncompressedExt = org.owasp.dependencycheck.utils.FileUtils.getFileExtension(uncompressedName).toLowerCase();
+                if (engine.supportsExtension(uncompressedExt)) {
+                    decompressFile(new GzipCompressorInputStream(new BufferedInputStream(fis)), new File(destination, uncompressedName));
+                }
+            }
+        } catch (ArchiveExtractionException ex) {
+            final String msg = String.format("Exception extracting archive '%s'.", archive.getName());
+            Logger.getLogger(ArchiveAnalyzer.class.getName()).log(Level.WARNING, msg);
+            Logger.getLogger(ArchiveAnalyzer.class.getName()).log(Level.FINE, null, ex);
+        } catch (IOException ex) {
+            final String msg = String.format("Exception reading archive '%s'.", archive.getName());
+            Logger.getLogger(ArchiveAnalyzer.class.getName()).log(Level.WARNING, msg);
+            Logger.getLogger(ArchiveAnalyzer.class.getName()).log(Level.FINE, null, ex);
+        } finally {
+            try {
+                fis.close();
+            } catch (IOException ex) {
+                Logger.getLogger(ArchiveAnalyzer.class.getName()).log(Level.FINEST, null, ex);
+            }
+        }
+    }
+
+    /**
+     * Extracts files from an archive.
+     *
+     * @param input the archive to extract files from
+     * @param destination the location to write the files too
+     * @param engine the dependency-check engine
+     * @throws ArchiveExtractionException thrown if there is an exception
+     * extracting files from the archive
+     */
+    private void extractArchive(ArchiveInputStream input, File destination, Engine engine) throws ArchiveExtractionException {
+        ArchiveEntry entry;
+        try {
+            while ((entry = input.getNextEntry()) != null) {
                 if (entry.isDirectory()) {
-                    final File d = new File(extractTo, entry.getName());
-                    if (!d.mkdirs()) {
-                        throw new AnalysisException("Unable to create '" + d.getAbsolutePath() + "'.");
+                    final File d = new File(destination, entry.getName());
+                    if (!d.exists()) {
+                        if (!d.mkdirs()) {
+                            final String msg = String.format("Unable to create '%s'.", d.getAbsolutePath());
+                            throw new AnalysisException(msg);
+                        }
                     }
                 } else {
-                    final File file = new File(extractTo, entry.getName());
+                    final File file = new File(destination, entry.getName());
                     final String ext = org.owasp.dependencycheck.utils.FileUtils.getFileExtension(file.getName());
                     if (engine.supportsExtension(ext)) {
                         BufferedOutputStream bos = null;
@@ -271,22 +318,27 @@ public class ArchiveAnalyzer extends AbstractAnalyzer implements Analyzer {
                             bos = new BufferedOutputStream(fos, BUFFER_SIZE);
                             int count;
                             final byte data[] = new byte[BUFFER_SIZE];
-                            while ((count = zis.read(data, 0, BUFFER_SIZE)) != -1) {
+                            while ((count = input.read(data, 0, BUFFER_SIZE)) != -1) {
                                 bos.write(data, 0, count);
                             }
                             bos.flush();
                         } catch (FileNotFoundException ex) {
-                            Logger.getLogger(ArchiveAnalyzer.class.getName()).log(Level.FINE, null, ex);
-                            throw new AnalysisException("Unable to find file '" + file.getName() + "'.", ex);
+                            Logger.getLogger(ArchiveAnalyzer.class
+                                    .getName()).log(Level.FINE, null, ex);
+                            final String msg = String.format("Unable to find file '%s'.", file.getName());
+                            throw new AnalysisException(msg, ex);
                         } catch (IOException ex) {
-                            Logger.getLogger(ArchiveAnalyzer.class.getName()).log(Level.FINE, null, ex);
-                            throw new AnalysisException("IO Exception while parsing file '" + file.getName() + "'.", ex);
+                            Logger.getLogger(ArchiveAnalyzer.class
+                                    .getName()).log(Level.FINE, null, ex);
+                            final String msg = String.format("IO Exception while parsing file '%s'.", file.getName());
+                            throw new AnalysisException(msg, ex);
                         } finally {
                             if (bos != null) {
                                 try {
                                     bos.close();
                                 } catch (IOException ex) {
-                                    Logger.getLogger(ArchiveAnalyzer.class.getName()).log(Level.FINEST, null, ex);
+                                    Logger.getLogger(ArchiveAnalyzer.class
+                                            .getName()).log(Level.FINEST, null, ex);
                                 }
                             }
                         }
@@ -294,14 +346,50 @@ public class ArchiveAnalyzer extends AbstractAnalyzer implements Analyzer {
                 }
             }
         } catch (IOException ex) {
-            final String msg = String.format("Exception reading archive '%s'.", archive.getName());
-            Logger.getLogger(ArchiveAnalyzer.class.getName()).log(Level.FINE, msg, ex);
-            throw new AnalysisException(msg, ex);
+            throw new ArchiveExtractionException(ex);
+        } catch (Throwable ex) {
+            throw new ArchiveExtractionException(ex);
         } finally {
-            try {
-                zis.close();
-            } catch (IOException ex) {
-                Logger.getLogger(ArchiveAnalyzer.class.getName()).log(Level.FINEST, null, ex);
+            if (input != null) {
+                try {
+                    input.close();
+                } catch (IOException ex) {
+                    Logger.getLogger(ArchiveAnalyzer.class.getName()).log(Level.FINEST, null, ex);
+                }
+            }
+        }
+    }
+
+    /**
+     * Decompresses a file.
+     *
+     * @param inputStream the compressed file
+     * @param outputFile the location to write the decompressed file
+     * @throws ArchiveExtractionException thrown if there is an exception
+     * decompressing the file
+     */
+    private void decompressFile(CompressorInputStream inputStream, File outputFile) throws ArchiveExtractionException {
+        FileOutputStream out = null;
+        try {
+            out = new FileOutputStream(outputFile);
+            final byte[] buffer = new byte[BUFFER_SIZE];
+            int n = 0;
+            while (-1 != (n = inputStream.read(buffer))) {
+                out.write(buffer, 0, n);
+            }
+        } catch (FileNotFoundException ex) {
+            Logger.getLogger(ArchiveAnalyzer.class.getName()).log(Level.FINE, null, ex);
+            throw new ArchiveExtractionException(ex);
+        } catch (IOException ex) {
+            Logger.getLogger(ArchiveAnalyzer.class.getName()).log(Level.FINE, null, ex);
+            throw new ArchiveExtractionException(ex);
+        } finally {
+            if (out != null) {
+                try {
+                    out.close();
+                } catch (IOException ex) {
+                    Logger.getLogger(ArchiveAnalyzer.class.getName()).log(Level.FINEST, null, ex);
+                }
             }
         }
     }

dependency-check-core/src/main/java/org/owasp/dependencycheck/analyzer/ArchiveExtractionException.java

@@ -0,0 +1,67 @@
+/*
+ * This file is part of dependency-check-core.
+ *
+ * Dependency-check-core is free software: you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation, either version 3 of the License, or (at your option) any
+ * later version.
+ *
+ * Dependency-check-core is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+ * details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * dependency-check-core. If not, see http://www.gnu.org/licenses/.
+ *
+ * Copyright (c) 2013 Jeremy Long. All Rights Reserved.
+ */
+package org.owasp.dependencycheck.analyzer;
+
+/**
+ * An exception thrown when the analysis of a dependency fails.
+ *
+ * @author Jeremy Long (jeremy.long@owasp.org)
+ */
+public class ArchiveExtractionException extends Exception {
+
+    /**
+     * The serial version UID for serialization.
+     */
+    private static final long serialVersionUID = 1L;
+
+    /**
+     * Creates a new AnalysisException.
+     */
+    public ArchiveExtractionException() {
+        super();
+    }
+
+    /**
+     * Creates a new AnalysisException.
+     *
+     * @param msg a message for the exception.
+     */
+    public ArchiveExtractionException(String msg) {
+        super(msg);
+    }
+
+    /**
+     * Creates a new AnalysisException.
+     *
+     * @param ex the cause of the failure.
+     */
+    public ArchiveExtractionException(Throwable ex) {
+        super(ex);
+    }
+
+    /**
+     * Creates a new DownloadFailedException.
+     *
+     * @param msg a message for the exception.
+     * @param ex the cause of the failure.
+     */
+    public ArchiveExtractionException(String msg, Throwable ex) {
+        super(msg, ex);
+    }
+}

dependency-check-core/src/main/java/org/owasp/dependencycheck/analyzer/CPEAnalyzer.java

@@ -16,7 +16,7 @@
  *
  * Copyright (c) 2012 Jeremy Long. All Rights Reserved.
  */
-package org.owasp.dependencycheck.data.cpe;
+package org.owasp.dependencycheck.analyzer;
 
 import java.io.IOException;
 import java.io.UnsupportedEncodingException;
@@ -35,14 +35,14 @@ import org.apache.lucene.queryparser.classic.ParseException;
 import org.apache.lucene.search.ScoreDoc;
 import org.apache.lucene.search.TopDocs;
 import org.owasp.dependencycheck.Engine;
-import org.owasp.dependencycheck.analyzer.AnalysisException;
-import org.owasp.dependencycheck.analyzer.AnalysisPhase;
 import org.owasp.dependencycheck.data.lucene.LuceneUtils;
 import org.owasp.dependencycheck.dependency.Dependency;
 import org.owasp.dependencycheck.dependency.Evidence;
 import org.owasp.dependencycheck.dependency.Evidence.Confidence;
 import org.owasp.dependencycheck.dependency.EvidenceCollection;
-import org.owasp.dependencycheck.analyzer.Analyzer;
+import org.owasp.dependencycheck.data.cpe.CpeIndexReader;
+import org.owasp.dependencycheck.data.cpe.Fields;
+import org.owasp.dependencycheck.data.cpe.IndexEntry;
 import org.owasp.dependencycheck.data.nvdcve.CveDB;
 import org.owasp.dependencycheck.data.nvdcve.DatabaseException;
 import org.owasp.dependencycheck.dependency.Identifier;
@@ -83,9 +83,9 @@ public class CPEAnalyzer implements Analyzer {
      */
     static final int STRING_BUILDER_BUFFER = 20;
     /**
-     * The CPE Index.
+     * The CPE Index Reader.
      */
-    private Index cpe;
+    private CpeIndexReader cpe;
     /**
      * The CVE Database.
      */
@@ -100,7 +100,7 @@ public class CPEAnalyzer implements Analyzer {
      * usually occurs when the database is in use by another process.
      */
     public void open() throws IOException, DatabaseException {
-        cpe = new Index();
+        cpe = new CpeIndexReader();
         cpe.open();
         cve = new CveDB();
         try {
@@ -119,8 +119,12 @@ public class CPEAnalyzer implements Analyzer {
      */
     @Override
     public void close() {
-        cpe.close();
-        cve.close();
+        if (cpe != null) {
+            cpe.close();
+        }
+        if (cve != null) {
+            cve.close();
+        }
     }
 
     /**
@@ -162,7 +166,6 @@ public class CPEAnalyzer implements Analyzer {
         String vendors = addEvidenceWithoutDuplicateTerms("", dependency.getVendorEvidence(), vendorConf);
         String products = addEvidenceWithoutDuplicateTerms("", dependency.getProductEvidence(), productConf);
 
-        //boolean found = false;
         int ctr = 0;
         do {
             if (!vendors.isEmpty() && !products.isEmpty()) {
@@ -171,27 +174,20 @@ public class CPEAnalyzer implements Analyzer {
 
                 for (IndexEntry e : entries) {
                     if (verifyEntry(e, dependency)) {
-                        //found = true; // we found a vendor/product pair. Now find version from the cve db.
                         final String vendor = e.getVendor();
                         final String product = e.getProduct();
-                        // cve.getVersions(vendor, product);
                         determineIdentifiers(dependency, vendor, product);
                     }
                 }
             }
-            //if (!found) {
             vendorConf = reduceConfidence(vendorConf);
             if (dependency.getVendorEvidence().contains(vendorConf)) {
-                //vendors += " " + dependency.getVendorEvidence().toString(vendorConf);
                 vendors = addEvidenceWithoutDuplicateTerms(vendors, dependency.getVendorEvidence(), vendorConf);
             }
             productConf = reduceConfidence(productConf);
             if (dependency.getProductEvidence().contains(productConf)) {
-                //products += " " + dependency.getProductEvidence().toString(productConf);
                 products = addEvidenceWithoutDuplicateTerms(products, dependency.getProductEvidence(), productConf);
             }
-            //}
-            //} while (!found && (++ctr) < 4);
         } while ((++ctr) < 4);
     }
 
@@ -275,13 +271,22 @@ public class CPEAnalyzer implements Analyzer {
 
         final TopDocs docs = cpe.search(searchString, MAX_QUERY_RESULTS);
         for (ScoreDoc d : docs.scoreDocs) {
-            final Document doc = cpe.getDocument(d.doc);
-            final IndexEntry entry = new IndexEntry();
-            entry.setVendor(doc.get(Fields.VENDOR));
-            entry.setProduct(doc.get(Fields.PRODUCT));
-            entry.setSearchScore(d.score);
-            if (!ret.contains(entry)) {
-                ret.add(entry);
+            if (d.score >= 0.08) {
+                final Document doc = cpe.getDocument(d.doc);
+                final IndexEntry entry = new IndexEntry();
+                entry.setVendor(doc.get(Fields.VENDOR));
+                entry.setProduct(doc.get(Fields.PRODUCT));
+//                if (d.score < 0.08) {
+//                    System.out.print(entry.getVendor());
+//                    System.out.print(":");
+//                    System.out.print(entry.getProduct());
+//                    System.out.print(":");
+//                    System.out.println(d.score);
+//                }
+                entry.setSearchScore(d.score);
+                if (!ret.contains(entry)) {
+                    ret.add(entry);
+                }
             }
         }
         return ret;
@@ -439,6 +444,9 @@ public class CPEAnalyzer implements Analyzer {
         //</editor-fold>
 
         //TODO - likely need to change the split... not sure if this will work for CPE with special chars
+        if (text == null) {
+            return false;
+        }
         final String[] words = text.split("[\\s_-]");
         final List<String> list = new ArrayList<String>();
         String tempWord = null;
@@ -454,9 +462,10 @@ public class CPEAnalyzer implements Analyzer {
                 list.add(word);
             }
         }
-//        if (tempWord != null) {
-//            //for now ignore any last single letter words...
-//        }
+        if (tempWord != null && !list.isEmpty()) {
+            final String tmp = list.get(list.size() - 1) + tempWord;
+            list.add(tmp);
+        }
         boolean contains = true;
         for (String word : list) {
             contains &= ec.containsUsedString(word);

dependency-check-core/src/main/java/org/owasp/dependencycheck/analyzer/DependencyBundlingAnalyzer.java

@@ -23,12 +23,15 @@ import java.util.HashSet;
 import java.util.Iterator;
 import java.util.ListIterator;
 import java.util.Set;
+import java.util.logging.Level;
+import java.util.logging.Logger;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 import org.owasp.dependencycheck.Engine;
 import org.owasp.dependencycheck.dependency.Dependency;
 import org.owasp.dependencycheck.utils.DependencyVersion;
 import org.owasp.dependencycheck.utils.DependencyVersionUtil;
+import org.owasp.dependencycheck.utils.LogUtils;
 
 /**
  * <p>This analyzer ensures dependencies that should be grouped together, to
@@ -144,16 +147,14 @@ public class DependencyBundlingAnalyzer extends AbstractAnalyzer implements Anal
                                 }
                                 dependenciesToRemove.add(nextDependency);
                             } else {
-                                if (isCore(nextDependency, dependency)) {
-                                    nextDependency.addRelatedDependency(dependency);
-                                    //move any "related dependencies" to the new "parent" dependency
-                                    final Iterator<Dependency> i = dependency.getRelatedDependencies().iterator();
-                                    while (i.hasNext()) {
-                                        nextDependency.addRelatedDependency(i.next());
-                                        i.remove();
-                                    }
-                                    dependenciesToRemove.add(dependency);
+                                nextDependency.addRelatedDependency(dependency);
+                                //move any "related dependencies" to the new "parent" dependency
+                                final Iterator<Dependency> i = dependency.getRelatedDependencies().iterator();
+                                while (i.hasNext()) {
+                                    nextDependency.addRelatedDependency(i.next());
+                                    i.remove();
                                 }
+                                dependenciesToRemove.add(dependency);
                             }
                         }
                     }
@@ -260,8 +261,13 @@ public class DependencyBundlingAnalyzer extends AbstractAnalyzer implements Anal
                 || dependency2 == null || dependency2.getIdentifiers() == null) {
             return false;
         }
-        return dependency1.getIdentifiers().size() > 0
+        final boolean matches = dependency1.getIdentifiers().size() > 0
                 && dependency2.getIdentifiers().equals(dependency1.getIdentifiers());
+        if (LogUtils.isVerboseLoggingEnabled()) {
+            final String msg = String.format("IdentifiersMatch=%s (%s, %s)", matches, dependency1.getFileName(), dependency2.getFileName());
+            Logger.getLogger(DependencyBundlingAnalyzer.class.getName()).log(Level.FINE, msg);
+        }
+        return matches;
     }
 
     /**
@@ -299,10 +305,6 @@ public class DependencyBundlingAnalyzer extends AbstractAnalyzer implements Anal
      * This is likely a very broken attempt at determining if the 'left'
      * dependency is the 'core' library in comparison to the 'right' library.
      *
-     * TODO - consider splitting on /\._-\s/ and checking if all of one side is
-     * fully contained in the other With the exception of the word "core". This
-     * might work even on groups when we don't have a CVE.
-     *
      * @param left the dependency to test
      * @param right the dependency to test against
      * @return a boolean indicating whether or not the left dependency should be
@@ -311,18 +313,31 @@ public class DependencyBundlingAnalyzer extends AbstractAnalyzer implements Anal
     private boolean isCore(Dependency left, Dependency right) {
         final String leftName = left.getFileName().toLowerCase();
         final String rightName = right.getFileName().toLowerCase();
-
+        final boolean returnVal;
         if (rightName.contains("core") && !leftName.contains("core")) {
-            return false;
+            returnVal = false;
         } else if (!rightName.contains("core") && leftName.contains("core")) {
-            return true;
+            returnVal = true;
         } else {
-            //TODO should we be splitting the name on [-_(.\d)+] and seeing if the
-            //  parts are contained in the other side?
+            /*
+             * considered splitting the names up and comparing the components,
+             * but decided that the file name length should be sufficient as the
+             * "core" component, if this follows a normal namming protocol should
+             * be shorter:
+             * axis2-saaj-1.4.1.jar
+             * axis2-1.4.1.jar       <-----
+             * axis2-kernal-1.4.1.jar
+             */
             if (leftName.length() > rightName.length()) {
-                return false;
+                returnVal = false;
+            } else {
+                returnVal = true;
             }
-            return true;
         }
+        if (LogUtils.isVerboseLoggingEnabled()) {
+            final String msg = String.format("IsCore=%s (%s, %s)", returnVal, left.getFileName(), right.getFileName());
+            Logger.getLogger(DependencyBundlingAnalyzer.class.getName()).log(Level.FINE, msg);
+        }
+        return returnVal;
     }
 }

dependency-check-core/src/main/java/org/owasp/dependencycheck/analyzer/FalsePositiveAnalyzer.java

@@ -109,6 +109,7 @@ public class FalsePositiveAnalyzer extends AbstractAnalyzer {
     public void analyze(Dependency dependency, Engine engine) throws AnalysisException {
         removeJreEntries(dependency);
         removeBadMatches(dependency);
+        removeWrongVersionMatches(dependency);
         removeSpuriousCPE(dependency);
         addFalseNegativeCPEs(dependency);
     }
@@ -129,6 +130,7 @@ public class FalsePositiveAnalyzer extends AbstractAnalyzer {
      *
      * @param dependency the dependency being analyzed
      */
+    @SuppressWarnings("null")
     private void removeSpuriousCPE(Dependency dependency) {
         final List<Identifier> ids = new ArrayList<Identifier>();
         ids.addAll(dependency.getIdentifiers());
@@ -155,6 +157,8 @@ public class FalsePositiveAnalyzer extends AbstractAnalyzer {
                         final String nextVersion = nextCpe.getVersion();
                         if (currentVersion == null && nextVersion == null) {
                             //how did we get here?
+                            Logger.getLogger(FalsePositiveAnalyzer.class
+                                    .getName()).log(Level.FINE, "currentVersion and nextVersion are both null?");
                         } else if (currentVersion == null && nextVersion != null) {
                             dependency.getIdentifiers().remove(currentId);
                         } else if (nextVersion == null && currentVersion != null) {
@@ -273,18 +277,56 @@ public class FalsePositiveAnalyzer extends AbstractAnalyzer {
                 if ((i.getValue().matches(".*c\\+\\+.*")
                         || i.getValue().startsWith("cpe:/a:jquery:jquery")
                         || i.getValue().startsWith("cpe:/a:prototypejs:prototype")
-                        || i.getValue().startsWith("cpe:/a:yahoo:yui"))
+                        || i.getValue().startsWith("cpe:/a:yahoo:yui")
+                        || i.getValue().startsWith("cpe:/a:file:file")
+                        || i.getValue().startsWith("cpe:/a:mozilla:mozilla")
+                        || i.getValue().startsWith("cpe:/a:cvs:cvs")
+                        || i.getValue().startsWith("cpe:/a:ftp:ftp")
+                        || i.getValue().startsWith("cpe:/a:ssh:ssh"))
                         && dependency.getFileName().toLowerCase().endsWith(".jar")) {
                     itr.remove();
-                } else if (i.getValue().startsWith("cpe:/a:file:file")
-                        || i.getValue().startsWith("cpe:/a:mozilla:mozilla")
-                        || i.getValue().startsWith("cpe:/a:ssh:ssh")) {
+                } else if (i.getValue().startsWith("cpe:/a:apache:maven")
+                        && !dependency.getFileName().toLowerCase().matches("maven-core-[\\d\\.]+\\.jar")) {
                     itr.remove();
                 }
             }
         }
     }
 
+    /**
+     * Removes CPE matches for the wrong version of a dependency. Currently,
+     * this only covers Axis 1 & 2.
+     *
+     * @param dependency the dependency to analyze
+     */
+    private void removeWrongVersionMatches(Dependency dependency) {
+        final Set<Identifier> identifiers = dependency.getIdentifiers();
+        final Iterator<Identifier> itr = identifiers.iterator();
+
+        final String fileName = dependency.getFileName();
+        if (fileName != null && fileName.contains("axis2")) {
+            while (itr.hasNext()) {
+                final Identifier i = itr.next();
+                if ("cpe".equals(i.getType())) {
+                    final String cpe = i.getValue();
+                    if (cpe != null && (cpe.startsWith("cpe:/a:apache:axis:") || "cpe:/a:apache:axis".equals(cpe))) {
+                        itr.remove();
+                    }
+                }
+            }
+        } else if (fileName != null && fileName.contains("axis")) {
+            while (itr.hasNext()) {
+                final Identifier i = itr.next();
+                if ("cpe".equals(i.getType())) {
+                    final String cpe = i.getValue();
+                    if (cpe != null && (cpe.startsWith("cpe:/a:apache:axis2:") || "cpe:/a:apache:axis2".equals(cpe))) {
+                        itr.remove();
+                    }
+                }
+            }
+        }
+    }
+
     /**
      * There are some known CPE entries, specifically regarding sun and oracle
      * products due to the acquisition and changes in product names, that based

dependency-check-core/src/main/java/org/owasp/dependencycheck/analyzer/JarAnalyzer.java

@@ -159,7 +159,7 @@ public class JarAnalyzer extends AbstractAnalyzer implements Analyzer {
     /**
      * The set of file extensions supported by this analyzer.
      */
-    private static final Set<String> EXTENSIONS = newHashSet("jar");
+    private static final Set<String> EXTENSIONS = newHashSet("jar", "war");
 
     /**
      * Returns a list of file EXTENSIONS supported by this analyzer.
@@ -515,7 +515,7 @@ public class JarAnalyzer extends AbstractAnalyzer implements Analyzer {
             if (ratio > 0.5) {
                 //TODO remove weighting
                 vendor.addWeighting(entry.getKey());
-                if (addPackagesAsEvidence) {
+                if (addPackagesAsEvidence && entry.getKey().length() > 1) {
                     vendor.addEvidence("jar", "package", entry.getKey(), Evidence.Confidence.LOW);
                 }
             }
@@ -524,7 +524,7 @@ public class JarAnalyzer extends AbstractAnalyzer implements Analyzer {
             final float ratio = entry.getValue() / (float) classCount;
             if (ratio > 0.5) {
                 product.addWeighting(entry.getKey());
-                if (addPackagesAsEvidence) {
+                if (addPackagesAsEvidence && entry.getKey().length() > 1) {
                     product.addEvidence("jar", "package", entry.getKey(), Evidence.Confidence.LOW);
                 }
             }
@@ -580,38 +580,38 @@ public class JarAnalyzer extends AbstractAnalyzer implements Analyzer {
                 }
                 if (IGNORE_VALUES.contains(value)) {
                     continue;
-                } else if (key.equals(Attributes.Name.IMPLEMENTATION_TITLE.toString())) {
+                } else if (key.equalsIgnoreCase(Attributes.Name.IMPLEMENTATION_TITLE.toString())) {
                     foundSomething = true;
                     productEvidence.addEvidence(source, key, value, Evidence.Confidence.HIGH);
                     addMatchingValues(classInformation, value, productEvidence);
-                } else if (key.equals(Attributes.Name.IMPLEMENTATION_VERSION.toString())) {
+                } else if (key.equalsIgnoreCase(Attributes.Name.IMPLEMENTATION_VERSION.toString())) {
                     foundSomething = true;
                     versionEvidence.addEvidence(source, key, value, Evidence.Confidence.HIGH);
-                } else if (key.equals(Attributes.Name.IMPLEMENTATION_VENDOR.toString())) {
+                } else if (key.equalsIgnoreCase(Attributes.Name.IMPLEMENTATION_VENDOR.toString())) {
                     foundSomething = true;
                     vendorEvidence.addEvidence(source, key, value, Evidence.Confidence.HIGH);
                     addMatchingValues(classInformation, value, vendorEvidence);
-                } else if (key.equals(Attributes.Name.IMPLEMENTATION_VENDOR_ID.toString())) {
+                } else if (key.equalsIgnoreCase(Attributes.Name.IMPLEMENTATION_VENDOR_ID.toString())) {
                     foundSomething = true;
                     vendorEvidence.addEvidence(source, key, value, Evidence.Confidence.MEDIUM);
                     addMatchingValues(classInformation, value, vendorEvidence);
-                } else if (key.equals(BUNDLE_DESCRIPTION)) {
+                } else if (key.equalsIgnoreCase(BUNDLE_DESCRIPTION)) {
                     foundSomething = true;
                     addDescription(dependency, value, "manifest", key);
                     //productEvidence.addEvidence(source, key, value, Evidence.Confidence.MEDIUM);
                     addMatchingValues(classInformation, value, productEvidence);
-                } else if (key.equals(BUNDLE_NAME)) {
+                } else if (key.equalsIgnoreCase(BUNDLE_NAME)) {
                     foundSomething = true;
                     productEvidence.addEvidence(source, key, value, Evidence.Confidence.MEDIUM);
                     addMatchingValues(classInformation, value, productEvidence);
-                } else if (key.equals(BUNDLE_VENDOR)) {
+                } else if (key.equalsIgnoreCase(BUNDLE_VENDOR)) {
                     foundSomething = true;
                     vendorEvidence.addEvidence(source, key, value, Evidence.Confidence.HIGH);
                     addMatchingValues(classInformation, value, vendorEvidence);
-                } else if (key.equals(BUNDLE_VERSION)) {
+                } else if (key.equalsIgnoreCase(BUNDLE_VERSION)) {
                     foundSomething = true;
                     versionEvidence.addEvidence(source, key, value, Evidence.Confidence.HIGH);
-                } else if (key.equals(Attributes.Name.MAIN_CLASS.toString())) {
+                } else if (key.equalsIgnoreCase(Attributes.Name.MAIN_CLASS.toString())) {
                     continue;
                     //skipping main class as if this has important information to add
                     // it will be added during class name analysis...  if other fields
@@ -637,13 +637,22 @@ public class JarAnalyzer extends AbstractAnalyzer implements Analyzer {
 
                         foundSomething = true;
                         if (key.contains("version")) {
-                            versionEvidence.addEvidence(source, key, value, Evidence.Confidence.MEDIUM);
+                            if (key.contains("specification")) {
+                                versionEvidence.addEvidence(source, key, value, Evidence.Confidence.LOW);
+                            } else {
+                                versionEvidence.addEvidence(source, key, value, Evidence.Confidence.MEDIUM);
+                            }
+
                         } else if (key.contains("title")) {
                             productEvidence.addEvidence(source, key, value, Evidence.Confidence.MEDIUM);
                             addMatchingValues(classInformation, value, productEvidence);
                         } else if (key.contains("vendor")) {
-                            vendorEvidence.addEvidence(source, key, value, Evidence.Confidence.MEDIUM);
-                            addMatchingValues(classInformation, value, vendorEvidence);
+                            if (key.contains("specification")) {
+                                vendorEvidence.addEvidence(source, key, value, Evidence.Confidence.LOW);
+                            } else {
+                                vendorEvidence.addEvidence(source, key, value, Evidence.Confidence.MEDIUM);
+                                addMatchingValues(classInformation, value, vendorEvidence);
+                            }
                         } else if (key.contains("name")) {
                             productEvidence.addEvidence(source, key, value, Evidence.Confidence.MEDIUM);
                             vendorEvidence.addEvidence(source, key, value, Evidence.Confidence.MEDIUM);

dependency-check-core/src/main/java/org/owasp/dependencycheck/analyzer/NvdCveAnalyzer.java

@@ -16,19 +16,18 @@
  *
  * Copyright (c) 2012 Jeremy Long. All Rights Reserved.
  */
-package org.owasp.dependencycheck.data.nvdcve;
+package org.owasp.dependencycheck.analyzer;
 
 import java.io.IOException;
 import java.sql.SQLException;
 import java.util.List;
 import java.util.Set;
 import org.owasp.dependencycheck.Engine;
-import org.owasp.dependencycheck.analyzer.AnalysisException;
-import org.owasp.dependencycheck.analyzer.AnalysisPhase;
 import org.owasp.dependencycheck.dependency.Dependency;
 import org.owasp.dependencycheck.dependency.Vulnerability;
 import org.owasp.dependencycheck.dependency.Identifier;
-import org.owasp.dependencycheck.analyzer.Analyzer;
+import org.owasp.dependencycheck.data.nvdcve.CveDB;
+import org.owasp.dependencycheck.data.nvdcve.DatabaseException;
 
 /**
  * NvdCveAnalyzer is a utility class that takes a project dependency and

dependency-check-core/src/main/java/org/owasp/dependencycheck/concurrency/DirectoryLockException.java

@@ -0,0 +1,67 @@
+/*
+ * This file is part of dependency-check-core.
+ *
+ * Dependency-check-core is free software: you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation, either version 3 of the License, or (at your option) any
+ * later version.
+ *
+ * Dependency-check-core is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+ * details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * dependency-check-core. If not, see http://www.gnu.org/licenses/.
+ *
+ * Copyright (c) 2013 Jeremy Long. All Rights Reserved.
+ */
+package org.owasp.dependencycheck.concurrency;
+
+/**
+ * If thrown, indicates that a problem occurred when locking a directory.
+ *
+ * @author Jeremy Long (jeremy.long@owasp.org)
+ */
+public class DirectoryLockException extends Exception {
+
+    /**
+     * Default serial version UID.
+     */
+    private static final long serialVersionUID = 1L;
+
+    /**
+     * Constructs a new Directory Lock Exception.
+     */
+    public DirectoryLockException() {
+        super();
+    }
+
+    /**
+     * Constructs a new Directory Lock Exception.
+     *
+     * @param msg the message describing the exception
+     */
+    public DirectoryLockException(String msg) {
+        super(msg);
+    }
+
+    /**
+     * Constructs a new Directory Lock Exception.
+     *
+     * @param ex the cause of the exception
+     */
+    public DirectoryLockException(Throwable ex) {
+        super(ex);
+    }
+
+    /**
+     * Constructs a new Directory Lock Exception.
+     *
+     * @param msg the message describing the exception
+     * @param ex the cause of the exception
+     */
+    public DirectoryLockException(String msg, Throwable ex) {
+        super(msg, ex);
+    }
+}

dependency-check-core/src/main/java/org/owasp/dependencycheck/concurrency/DirectorySpinLock.java

@@ -0,0 +1,267 @@
+/*
+ * This file is part of dependency-check-core.
+ *
+ * Dependency-check-core is free software: you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation, either version 3 of the License, or (at your option) any
+ * later version.
+ *
+ * Dependency-check-core is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+ * details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * dependency-check-core. If not, see http://www.gnu.org/licenses/.
+ *
+ * Copyright (c) 2013 Jeremy Long. All Rights Reserved.
+ */
+package org.owasp.dependencycheck.concurrency;
+
+import java.io.Closeable;
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.RandomAccessFile;
+import java.nio.channels.AsynchronousCloseException;
+import java.nio.channels.ClosedChannelException;
+import java.nio.channels.FileChannel;
+import java.nio.channels.FileLock;
+import java.nio.channels.FileLockInterruptionException;
+import java.nio.channels.NonWritableChannelException;
+import java.nio.channels.OverlappingFileLockException;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+/**
+ * Implements a spin lock on a given directory. If the lock cannot be obtained,
+ * the process will "spin" waiting for an opportunity to obtain the lock
+ * requested.
+ *
+ * @author Jeremy Long (jeremy.long@owasp.org)
+ */
+public class DirectorySpinLock implements Closeable /*, AutoCloseable*/ {
+
+    /**
+     * The name of the lock file.
+     */
+    public static final String LOCK_NAME = "data.lock";
+    /**
+     * The maximum wait period used when attempting to obtain a lock.
+     */
+    public static final int MAX_SPIN = 100;
+    /**
+     * The file channel used to perform the lock.
+     */
+    private FileChannel channel = null;
+    /**
+     * The file used to perform the lock.
+     */
+    private File lockFile = null;
+    /**
+     * The lock object.
+     */
+    private FileLock lock = null;
+    /**
+     * The maximum number of seconds that the spin lock will wait while trying
+     * to obtain a lock.
+     */
+    private long maxWait = MAX_SPIN;
+
+    /**
+     * Get the maximum wait time, in seconds, that the spin lock will wait while
+     * trying to obtain a lock.
+     *
+     * @return the number of seconds the spin lock will wait
+     */
+    public long getMaxWait() {
+        return maxWait / 2; //sleep is for 500, so / 2
+    }
+
+    /**
+     * Set the maximum wait time, in seconds, that the spin lock will wait while
+     * trying to obtain a lock.
+     *
+     * @param maxWait the number of seconds the spin lock will wait
+     */
+    public void setMaxWait(long maxWait) {
+        this.maxWait = maxWait * 2; //sleep is for 500, so * 2
+    }
+
+    /**
+     * Constructs a new spin lock on the given directory.
+     *
+     * @param directory the directory to monitor/lock
+     * @throws InvalidDirectoryException thrown if there is an issue with the
+     * directory provided
+     * @throws DirectoryLockException thrown there is an issue obtaining a
+     * handle to the lock file
+     */
+    public DirectorySpinLock(File directory) throws InvalidDirectoryException, DirectoryLockException {
+        checkDirectory(directory);
+        lockFile = new File(directory, LOCK_NAME);
+        RandomAccessFile file = null;
+        try {
+            file = new RandomAccessFile(lockFile, "rw");
+        } catch (FileNotFoundException ex) {
+            throw new DirectoryLockException("Lock file not found", ex);
+        }
+        channel = file.getChannel();
+    }
+
+    /**
+     * Attempts to obtain an exclusive lock; an exception is thrown if the lock
+     * could not be obtained. This method may block for a few seconds if a lock
+     * cannot be obtained.
+     *
+     * @throws DirectoryLockException thrown if there is an exception obtaining
+     * the lock
+     */
+    public void obtainSharedLock() throws DirectoryLockException {
+        obtainLock(true);
+    }
+
+    /**
+     * Attempts to obtain an exclusive lock; an exception is thrown if the lock
+     * could not be obtained. This method may block for a few seconds if a lock
+     * cannot be obtained.
+     *
+     * @throws DirectoryLockException thrown if there is an exception obtaining
+     * the lock
+     */
+    public void obtainExclusiveLock() throws DirectoryLockException {
+        obtainLock(false);
+    }
+
+    /**
+     * Attempts to obtain a lock; an exception is thrown if the lock could not
+     * be obtained. This method may block for a few seconds if a lock cannot be
+     * obtained.
+     *
+     * @param shared true if the lock is shared, otherwise false
+     * @param maxWait the maximum time to wait, in seconds, while trying to
+     * obtain the lock
+     * @throws DirectoryLockException thrown if there is an exception obtaining
+     * the lock
+     */
+    protected void obtainLock(boolean shared, long maxWait) throws DirectoryLockException {
+        setMaxWait(maxWait);
+        obtainLock(shared);
+    }
+
+    /**
+     * Attempts to obtain a lock; an exception is thrown if the lock could not
+     * be obtained. This method may block for a few seconds if a lock cannot be
+     * obtained.
+     *
+     * @param shared true if the lock is shared, otherwise false
+     * @throws DirectoryLockException thrown if there is an exception obtaining
+     * the lock
+     */
+    protected void obtainLock(boolean shared) throws DirectoryLockException {
+        if (lock != null) {
+            release();
+        }
+        if (channel == null) {
+            throw new DirectoryLockException("Unable to create lock, no file channel exists");
+        }
+        int count = 0;
+        Exception lastException = null;
+        while (lock == null && count++ < maxWait) {
+            try {
+                lock = channel.lock(0, Long.MAX_VALUE, shared);
+            } catch (AsynchronousCloseException ex) {
+                lastException = ex;
+            } catch (ClosedChannelException ex) {
+                lastException = ex;
+            } catch (FileLockInterruptionException ex) {
+                lastException = ex;
+            } catch (OverlappingFileLockException ex) {
+                lastException = ex;
+            } catch (NonWritableChannelException ex) {
+                lastException = ex;
+            } catch (IOException ex) {
+                lastException = ex;
+            }
+            try {
+                Thread.sleep(500);
+            } catch (InterruptedException ex) {
+                Thread.currentThread().interrupt();
+            }
+        }
+        if (lock == null) {
+            if (lastException == null) {
+                throw new DirectoryLockException("Unable to obtain lock");
+            } else {
+                throw new DirectoryLockException("Unable to obtain lock", lastException);
+            }
+        }
+    }
+
+    /**
+     * Performs a few simple rudimentary checks on the specified directory.
+     * Specifically, does the file exist and is it a directory.
+     *
+     * @param directory the File object to inspect
+     * @throws InvalidDirectoryException thrown if the directory is null or is
+     * not a directory
+     */
+    private void checkDirectory(File directory) throws InvalidDirectoryException {
+        if (directory == null) {
+            throw new InvalidDirectoryException("Unable to obtain lock on a null File");
+        }
+        if (!directory.isDirectory()) {
+            final String msg = String.format("File, '%s', does not exist or is not a directory", directory.getAbsolutePath());
+            throw new InvalidDirectoryException(msg);
+        }
+    }
+
+    /**
+     * Releases any locks and closes the underlying channel.
+     *
+     * @throws IOException if an IO Exception occurs
+     */
+    @Override
+    public void close() throws IOException {
+        release();
+// TODO uncomment this once support for 1.6 is dropped.
+//        if (lock != null) {
+//            try {
+//                lock.close();
+//            } catch (IOException ex) {
+//                Logger.getLogger(DirectorySpinLock.class.getName()).log(Level.FINEST, "Unable to close file lock due to IO Exception", ex);
+//            }
+//        }
+        if (channel != null) {
+            try {
+                channel.close();
+            } catch (IOException ex) {
+                Logger.getLogger(DirectorySpinLock.class.getName()).log(Level.FINEST, "Unable to close the channel for the file lock", ex);
+            }
+        }
+        if (lockFile != null) {
+            if (lockFile.exists()) {
+                /* yes, this delete could fail which is totally fine. The other
+                 * thread holding the lock while delete it.
+                 */
+                lockFile.delete();
+            }
+        }
+    }
+
+    /**
+     * Releases the lock. Any exceptions that are thrown by the underlying lock
+     * during the release are ignored.
+     */
+    public void release() {
+        if (lock != null) {
+            try {
+                lock.release();
+            } catch (ClosedChannelException ex) {
+                Logger.getLogger(DirectorySpinLock.class.getName()).log(Level.FINEST, "Uable to release file lock", ex);
+            } catch (IOException ex) {
+                Logger.getLogger(DirectorySpinLock.class.getName()).log(Level.FINEST, "Unable to release file lock due to IO Exception", ex);
+            }
+        }
+    }
+}

dependency-check-core/src/main/java/org/owasp/dependencycheck/concurrency/InvalidDirectoryException.java

@@ -0,0 +1,67 @@
+/*
+ * This file is part of dependency-check-core.
+ *
+ * Dependency-check-core is free software: you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation, either version 3 of the License, or (at your option) any
+ * later version.
+ *
+ * Dependency-check-core is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+ * details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * dependency-check-core. If not, see http://www.gnu.org/licenses/.
+ *
+ * Copyright (c) 2013 Jeremy Long. All Rights Reserved.
+ */
+package org.owasp.dependencycheck.concurrency;
+
+/**
+ * If thrown, indicates that there is a problem with a directory.
+ *
+ * @author Jeremy Long (jeremy.long@owasp.org)
+ */
+public class InvalidDirectoryException extends Exception {
+
+    /**
+     * Default serial version UID.
+     */
+    private static final long serialVersionUID = 1L;
+
+    /**
+     * Constructs a new Invalid Directory Exception.
+     */
+    public InvalidDirectoryException() {
+        super();
+    }
+
+    /**
+     * Constructs a new Invalid Directory Exception.
+     *
+     * @param msg the message describing the exception
+     */
+    public InvalidDirectoryException(String msg) {
+        super(msg);
+    }
+
+    /**
+     * Constructs a new Invalid Directory Exception.
+     *
+     * @param ex the cause of the exception
+     */
+    public InvalidDirectoryException(Throwable ex) {
+        super(ex);
+    }
+
+    /**
+     * Constructs a new Invalid Directory Exception.
+     *
+     * @param msg the message describing the exception
+     * @param ex the cause of the exception
+     */
+    public InvalidDirectoryException(String msg, Throwable ex) {
+        super(msg, ex);
+    }
+}

dependency-check-core/src/main/java/org/owasp/dependencycheck/concurrency/package-info.java

@@ -0,0 +1,11 @@
+/**
+ * <html>
+ * <head>
+ * <title>org.owasp.dependencycheck.concurrency</title>
+ * </head>
+ * <body>
+ * Contains classes used to create shared and exclusive locks on directories.
+ * </body>
+ * </html>
+ */
+package org.owasp.dependencycheck.concurrency;

dependency-check-core/src/main/java/org/owasp/dependencycheck/data/NoDataException.java

@@ -0,0 +1,69 @@
+/*
+ * This file is part of dependency-check-core.
+ *
+ * Dependency-check-core is free software: you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation, either version 3 of the License, or (at your option) any
+ * later version.
+ *
+ * Dependency-check-core is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+ * details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * dependency-check-core. If not, see http://www.gnu.org/licenses/.
+ *
+ * Copyright (c) 2013 Jeremy Long. All Rights Reserved.
+ */
+package org.owasp.dependencycheck.data;
+
+import java.io.IOException;
+
+/**
+ * An exception used when the data needed does not exist to perform analysis.
+ *
+ * @author Jeremy Long (jeremy.long@owasp.org)
+ */
+public class NoDataException extends IOException {
+
+    /**
+     * The serial version uid.
+     */
+    private static final long serialVersionUID = 1L;
+
+    /**
+     * Creates a new NoDataException.
+     */
+    public NoDataException() {
+        super();
+    }
+
+    /**
+     * Creates a new NoDataException.
+     *
+     * @param msg a message for the exception.
+     */
+    public NoDataException(String msg) {
+        super(msg);
+    }
+
+    /**
+     * Creates a new NoDataException.
+     *
+     * @param ex the cause of the exception.
+     */
+    public NoDataException(Throwable ex) {
+        super(ex);
+    }
+
+    /**
+     * Creates a new NoDataException.
+     *
+     * @param msg a message for the exception.
+     * @param ex the cause of the exception.
+     */
+    public NoDataException(String msg, Throwable ex) {
+        super(msg, ex);
+    }
+}

dependency-check-core/src/main/java/org/owasp/dependencycheck/data/cpe/BaseIndex.java

@@ -0,0 +1,117 @@
+/*
+ * This file is part of dependency-check-core.
+ *
+ * Dependency-check-core is free software: you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation, either version 3 of the License, or (at your option) any
+ * later version.
+ *
+ * Dependency-check-core is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+ * details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * dependency-check-core. If not, see http://www.gnu.org/licenses/.
+ *
+ * Copyright (c) 2013 Jeremy Long. All Rights Reserved.
+ */
+package org.owasp.dependencycheck.data.cpe;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.FSDirectory;
+import org.owasp.dependencycheck.utils.Settings;
+
+/**
+ * The Base Index class used to access the CPE Index.
+ *
+ * @author Jeremy Long (jeremy.long@owasp.org)
+ */
+public abstract class BaseIndex {
+
+    /**
+     * The Lucene directory containing the index.
+     */
+    private Directory directory;
+    /**
+     * Indicates whether or not the Lucene Index is open.
+     */
+    private boolean indexOpen = false;
+
+    /**
+     * Gets the directory.
+     *
+     * @return the directory
+     */
+    public Directory getDirectory() {
+        return directory;
+    }
+
+    /**
+     * Opens the CPE Index.
+     *
+     * @throws IOException is thrown if an IOException occurs opening the index.
+     */
+    public void open() throws IOException {
+        directory = this.openDirectory();
+        indexOpen = true;
+    }
+
+    /**
+     * Closes the CPE Index.
+     */
+    public void close() {
+        try {
+            directory.close();
+        } catch (IOException ex) {
+            final String msg = "Unable to update database due to an IO error.";
+            Logger.getLogger(BaseIndex.class.getName()).log(Level.SEVERE, msg);
+            Logger.getLogger(BaseIndex.class.getName()).log(Level.FINE, null, ex);
+        } finally {
+            directory = null;
+        }
+        indexOpen = false;
+
+    }
+
+    /**
+     * Returns the status of the data source - is the index open.
+     *
+     * @return true or false.
+     */
+    public boolean isOpen() {
+        return indexOpen;
+    }
+
+    /**
+     * Returns the Lucene directory object for the CPE Index.
+     *
+     * @return the Lucene Directory object for the CPE Index.
+     * @throws IOException is thrown if an IOException occurs.
+     */
+    protected Directory openDirectory() throws IOException {
+        final File path = getDataDirectory();
+        return FSDirectory.open(path);
+    }
+
+    /**
+     * Retrieves the directory that the JAR file exists in so that we can ensure
+     * we always use a common data directory.
+     *
+     * @return the data directory for this index.
+     * @throws IOException is thrown if an IOException occurs of course...
+     */
+    public static File getDataDirectory() throws IOException {
+        final File path = Settings.getFile(Settings.KEYS.CPE_DATA_DIRECTORY);
+        if (!path.exists()) {
+            if (!path.mkdirs()) {
+                throw new IOException("Unable to create CPE Data directory");
+            }
+        }
+        return path;
+    }
+}

dependency-check-core/src/main/java/org/owasp/dependencycheck/data/cpe/CpeIndexReader.java

@@ -0,0 +1,209 @@
+/*
+ * This file is part of dependency-check-core.
+ *
+ * Dependency-check-core is free software: you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation, either version 3 of the License, or (at your option) any
+ * later version.
+ *
+ * Dependency-check-core is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+ * details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * dependency-check-core. If not, see http://www.gnu.org/licenses/.
+ *
+ * Copyright (c) 2013 Jeremy Long. All Rights Reserved.
+ */
+package org.owasp.dependencycheck.data.cpe;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.core.KeywordAnalyzer;
+import org.apache.lucene.analysis.miscellaneous.PerFieldAnalyzerWrapper;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.index.CorruptIndexException;
+import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.queryparser.classic.ParseException;
+import org.apache.lucene.queryparser.classic.QueryParser;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.TopDocs;
+import org.apache.lucene.util.Version;
+import org.owasp.dependencycheck.data.lucene.FieldAnalyzer;
+import org.owasp.dependencycheck.data.lucene.SearchFieldAnalyzer;
+
+/**
+ *
+ * @author Jeremy Long (jeremy.long@owasp.org)
+ */
+public class CpeIndexReader extends BaseIndex {
+
+    /**
+     * The Lucene IndexReader.
+     */
+    private IndexReader indexReader;
+    /**
+     * The Lucene IndexSearcher.
+     */
+    private IndexSearcher indexSearcher;
+    /**
+     * The Lucene Analyzer used for Searching.
+     */
+    private Analyzer searchingAnalyzer;
+    /**
+     * The Lucene QueryParser used for Searching.
+     */
+    private QueryParser queryParser;
+    /**
+     * The search field analyzer for the product field.
+     */
+    private SearchFieldAnalyzer productSearchFieldAnalyzer;
+    /**
+     * The search field analyzer for the vendor field.
+     */
+    private SearchFieldAnalyzer vendorSearchFieldAnalyzer;
+
+    /**
+     * Opens the CPE Index.
+     *
+     * @throws IOException is thrown if an IOException occurs opening the index.
+     */
+    @Override
+    public void open() throws IOException {
+        //TODO add spinlock (shared)
+        super.open();
+        indexReader = DirectoryReader.open(getDirectory());
+        indexSearcher = new IndexSearcher(indexReader);
+        searchingAnalyzer = createSearchingAnalyzer();
+        queryParser = new QueryParser(Version.LUCENE_43, Fields.DOCUMENT_KEY, searchingAnalyzer);
+    }
+
+    /**
+     * Closes the CPE Index.
+     */
+    @Override
+    public void close() {
+        //TODO remove spinlock (shared)
+        if (searchingAnalyzer != null) {
+            searchingAnalyzer.close();
+            searchingAnalyzer = null;
+        }
+        if (indexReader != null) {
+            try {
+                indexReader.close();
+            } catch (IOException ex) {
+                Logger.getLogger(CpeIndexReader.class.getName()).log(Level.FINEST, null, ex);
+            }
+            indexReader = null;
+        }
+        queryParser = null;
+        indexSearcher = null;
+        super.close();
+    }
+
+    /**
+     * Searches the index using the given search string.
+     *
+     * @param searchString the query text
+     * @param maxQueryResults the maximum number of documents to return
+     * @return the TopDocs found by the search
+     * @throws ParseException thrown when the searchString is invalid
+     * @throws IOException is thrown if there is an issue with the underlying
+     * Index
+     */
+    public TopDocs search(String searchString, int maxQueryResults) throws ParseException, IOException {
+        if (searchString == null || searchString.trim().isEmpty()) {
+            throw new ParseException("Query is null or empty");
+        }
+        if (queryParser == null) {
+            if (isOpen()) {
+                final String msg = String.format("QueryParser is null for query: '%s'. Attempting to reopen index.",
+                        searchString);
+                Logger.getLogger(CpeIndexReader.class.getName()).log(Level.WARNING, msg);
+                close();
+                open();
+            } else {
+                final String msg = String.format("QueryParser is null, but data source is open, for query: '%s'. Attempting to reopen index.",
+                        searchString);
+                Logger.getLogger(CpeIndexReader.class.getName()).log(Level.WARNING, msg);
+                close();
+                open();
+            }
+        }
+        final Query query = queryParser.parse(searchString);
+        return indexSearcher.search(query, maxQueryResults);
+    }
+
+    /**
+     * Searches the index using the given query.
+     *
+     * @param query the query used to search the index
+     * @param maxQueryResults the max number of results to return
+     * @return the TopDocs found be the query
+     * @throws CorruptIndexException thrown if the Index is corrupt
+     * @throws IOException thrown if there is an IOException
+     */
+    public TopDocs search(Query query, int maxQueryResults) throws CorruptIndexException, IOException {
+        resetSearchingAnalyzer();
+        return indexSearcher.search(query, maxQueryResults);
+    }
+
+    /**
+     * Retrieves a document from the Index.
+     *
+     * @param documentId the id of the document to retrieve
+     * @return the Document
+     * @throws IOException thrown if there is an IOException
+     */
+    public Document getDocument(int documentId) throws IOException {
+        return indexSearcher.doc(documentId);
+    }
+
+    /**
+     * Creates an Analyzer for searching the CPE Index.
+     *
+     * @return the CPE Analyzer.
+     */
+    @SuppressWarnings("unchecked")
+    private Analyzer createSearchingAnalyzer() {
+        final Map fieldAnalyzers = new HashMap();
+        fieldAnalyzers.put(Fields.DOCUMENT_KEY, new KeywordAnalyzer());
+        productSearchFieldAnalyzer = new SearchFieldAnalyzer(Version.LUCENE_43);
+        vendorSearchFieldAnalyzer = new SearchFieldAnalyzer(Version.LUCENE_43);
+        fieldAnalyzers.put(Fields.PRODUCT, productSearchFieldAnalyzer);
+        fieldAnalyzers.put(Fields.VENDOR, vendorSearchFieldAnalyzer);
+
+        return new PerFieldAnalyzerWrapper(new FieldAnalyzer(Version.LUCENE_43), fieldAnalyzers);
+    }
+
+    /**
+     * Resets the searching analyzers
+     */
+    private void resetSearchingAnalyzer() {
+        if (productSearchFieldAnalyzer != null) {
+            productSearchFieldAnalyzer.clear();
+        }
+        if (vendorSearchFieldAnalyzer != null) {
+            vendorSearchFieldAnalyzer.clear();
+        }
+    }
+
+    /**
+     * Returns the number of CPE entries stored in the index.
+     *
+     * @return the number of CPE entries stored in the index
+     */
+    public int numDocs() {
+        if (indexReader == null) {
+            return -1;
+        }
+        return indexReader.numDocs();
+    }
+}

dependency-check-core/src/main/java/org/owasp/dependencycheck/data/cpe/CpeIndexWriter.java

@@ -0,0 +1,149 @@
+/*
+ * This file is part of dependency-check-core.
+ *
+ * Dependency-check-core is free software: you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation, either version 3 of the License, or (at your option) any
+ * later version.
+ *
+ * Dependency-check-core is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+ * details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * dependency-check-core. If not, see http://www.gnu.org/licenses/.
+ *
+ * Copyright (c) 2013 Jeremy Long. All Rights Reserved.
+ */
+package org.owasp.dependencycheck.data.cpe;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.core.KeywordAnalyzer;
+import org.apache.lucene.analysis.miscellaneous.PerFieldAnalyzerWrapper;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.document.StringField;
+import org.apache.lucene.document.TextField;
+import org.apache.lucene.index.CorruptIndexException;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.util.Version;
+import org.owasp.dependencycheck.data.lucene.FieldAnalyzer;
+
+/**
+ *
+ * @author Jeremy Long (jeremy.long@owasp.org)
+ */
+public class CpeIndexWriter extends BaseIndex {
+
+    /**
+     * The IndexWriter for the Lucene index.
+     */
+    private IndexWriter indexWriter;
+    /**
+     * The Lucene Analyzer used for Indexing.
+     */
+    private Analyzer indexingAnalyzer;
+
+    /**
+     * Opens the CPE Index.
+     *
+     * @throws IOException is thrown if an IOException occurs opening the index.
+     */
+    @Override
+    public void open() throws IOException {
+        //TODO add spinlock
+        super.open();
+        indexingAnalyzer = createIndexingAnalyzer();
+        final IndexWriterConfig conf = new IndexWriterConfig(Version.LUCENE_43, indexingAnalyzer);
+        indexWriter = new IndexWriter(getDirectory(), conf);
+    }
+
+    /**
+     * Closes the CPE Index.
+     */
+    @Override
+    public void close() {
+        //TODO remove spinlock
+        if (indexWriter != null) {
+            commit();
+            try {
+                indexWriter.close(true);
+            } catch (CorruptIndexException ex) {
+                final String msg = "Unable to update database, there is a corrupt index.";
+                Logger.getLogger(CpeIndexWriter.class.getName()).log(Level.SEVERE, msg);
+                Logger.getLogger(CpeIndexWriter.class.getName()).log(Level.FINE, null, ex);
+            } catch (IOException ex) {
+                final String msg = "Unable to update database due to an IO error.";
+                Logger.getLogger(CpeIndexWriter.class.getName()).log(Level.SEVERE, msg);
+                Logger.getLogger(CpeIndexWriter.class.getName()).log(Level.FINE, null, ex);
+            } finally {
+                indexWriter = null;
+            }
+        }
+        if (indexingAnalyzer != null) {
+            indexingAnalyzer.close();
+            indexingAnalyzer = null;
+        }
+        super.close();
+    }
+
+    /**
+     * Commits any pending changes.
+     */
+    public void commit() {
+        if (indexWriter != null) {
+            try {
+                indexWriter.forceMerge(1);
+                indexWriter.commit();
+            } catch (CorruptIndexException ex) {
+                final String msg = "Unable to update database, there is a corrupt index.";
+                Logger.getLogger(CpeIndexWriter.class.getName()).log(Level.SEVERE, msg);
+                Logger.getLogger(CpeIndexWriter.class.getName()).log(Level.FINE, null, ex);
+            } catch (IOException ex) {
+                final String msg = "Unable to update database due to an IO error.";
+                Logger.getLogger(CpeIndexWriter.class.getName()).log(Level.SEVERE, msg);
+                Logger.getLogger(CpeIndexWriter.class.getName()).log(Level.FINE, null, ex);
+            }
+        }
+    }
+
+    /**
+     * Creates the indexing analyzer for the CPE Index.
+     *
+     * @return the CPE Analyzer.
+     */
+    @SuppressWarnings("unchecked")
+    private Analyzer createIndexingAnalyzer() {
+        final Map fieldAnalyzers = new HashMap();
+        fieldAnalyzers.put(Fields.DOCUMENT_KEY, new KeywordAnalyzer());
+        return new PerFieldAnalyzerWrapper(new FieldAnalyzer(Version.LUCENE_43), fieldAnalyzers);
+    }
+
+    /**
+     * Saves a CPE IndexEntry into the Lucene index.
+     *
+     * @param entry a CPE entry.
+     * @throws CorruptIndexException is thrown if the index is corrupt.
+     * @throws IOException is thrown if an IOException occurs.
+     */
+    public void saveEntry(IndexEntry entry) throws CorruptIndexException, IOException {
+        final Document doc = new Document();
+        final Field documentKey = new StringField(Fields.DOCUMENT_KEY, entry.getDocumentId(), Field.Store.NO);
+        final Field vendor = new TextField(Fields.VENDOR, entry.getVendor(), Field.Store.YES);
+        final Field product = new TextField(Fields.PRODUCT, entry.getProduct(), Field.Store.YES);
+        doc.add(documentKey);
+        doc.add(vendor);
+        doc.add(product);
+
+        final Term term = new Term(Fields.DOCUMENT_KEY, entry.getDocumentId());
+        indexWriter.updateDocument(term, doc);
+    }
+}

dependency-check-core/src/main/java/org/owasp/dependencycheck/data/cpe/Index.java

@@ -1,170 +0,0 @@
-/*
- * This file is part of dependency-check-core.
- *
- * Dependency-check-core is free software: you can redistribute it and/or modify it
- * under the terms of the GNU General Public License as published by the Free
- * Software Foundation, either version 3 of the License, or (at your option) any
- * later version.
- *
- * Dependency-check-core is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
- * details.
- *
- * You should have received a copy of the GNU General Public License along with
- * dependency-check-core. If not, see http://www.gnu.org/licenses/.
- *
- * Copyright (c) 2012 Jeremy Long. All Rights Reserved.
- */
-package org.owasp.dependencycheck.data.cpe;
-
-import java.io.File;
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.Map;
-import org.apache.lucene.analysis.Analyzer;
-import org.apache.lucene.analysis.core.KeywordAnalyzer;
-import org.apache.lucene.analysis.miscellaneous.PerFieldAnalyzerWrapper;
-import org.apache.lucene.document.Document;
-import org.apache.lucene.document.Field;
-import org.apache.lucene.document.TextField;
-import org.apache.lucene.index.CorruptIndexException;
-import org.apache.lucene.index.Term;
-import org.apache.lucene.queryparser.classic.QueryParser;
-import org.apache.lucene.store.Directory;
-import org.apache.lucene.store.FSDirectory;
-import org.apache.lucene.util.Version;
-import org.owasp.dependencycheck.data.lucene.AbstractIndex;
-import org.owasp.dependencycheck.utils.Settings;
-import org.owasp.dependencycheck.data.lucene.FieldAnalyzer;
-import org.owasp.dependencycheck.data.lucene.SearchFieldAnalyzer;
-
-/**
- * The Index class is used to utilize and maintain the CPE Index.
- *
- * @author Jeremy Long (jeremy.long@owasp.org)
- */
-public class Index extends AbstractIndex {
-
-    /**
-     * Returns the directory that holds the CPE Index.
-     *
-     * @return the Directory containing the CPE Index.
-     * @throws IOException is thrown if an IOException occurs.
-     */
-    @Override
-    public Directory getDirectory() throws IOException {
-        final File path = getDataDirectory();
-        return FSDirectory.open(path);
-    }
-
-    /**
-     * Retrieves the directory that the JAR file exists in so that we can ensure
-     * we always use a common data directory.
-     *
-     * @return the data directory for this index.
-     * @throws IOException is thrown if an IOException occurs of course...
-     */
-    public File getDataDirectory() throws IOException {
-        final File path = Settings.getFile(Settings.KEYS.CPE_DATA_DIRECTORY);
-        if (!path.exists()) {
-            if (!path.mkdirs()) {
-                throw new IOException("Unable to create CPE Data directory");
-            }
-        }
-        return path;
-    }
-
-    /**
-     * Creates an Analyzer for the CPE Index.
-     *
-     * @return the CPE Analyzer.
-     */
-    @SuppressWarnings("unchecked")
-    @Override
-    public Analyzer createIndexingAnalyzer() {
-        final Map fieldAnalyzers = new HashMap();
-        fieldAnalyzers.put(Fields.DOCUMENT_KEY, new KeywordAnalyzer());
-        return new PerFieldAnalyzerWrapper(new FieldAnalyzer(Version.LUCENE_43), fieldAnalyzers);
-    }
-    /**
-     * The search field analyzer for the product field.
-     */
-    private SearchFieldAnalyzer productSearchFieldAnalyzer;
-    /**
-     * The search field analyzer for the vendor field.
-     */
-    private SearchFieldAnalyzer vendorSearchFieldAnalyzer;
-
-    /**
-     * Creates an Analyzer for searching the CPE Index.
-     *
-     * @return the CPE Analyzer.
-     */
-    @SuppressWarnings("unchecked")
-    @Override
-    public Analyzer createSearchingAnalyzer() {
-        final Map fieldAnalyzers = new HashMap();
-
-        fieldAnalyzers.put(Fields.DOCUMENT_KEY, new KeywordAnalyzer());
-        productSearchFieldAnalyzer = new SearchFieldAnalyzer(Version.LUCENE_43);
-        vendorSearchFieldAnalyzer = new SearchFieldAnalyzer(Version.LUCENE_43);
-        fieldAnalyzers.put(Fields.PRODUCT, productSearchFieldAnalyzer);
-        fieldAnalyzers.put(Fields.VENDOR, vendorSearchFieldAnalyzer);
-
-        return new PerFieldAnalyzerWrapper(new FieldAnalyzer(Version.LUCENE_43), fieldAnalyzers);
-    }
-
-    /**
-     * Creates the Lucene QueryParser used when querying the index.
-     *
-     * @return a QueryParser.
-     */
-    @Override
-    public QueryParser createQueryParser() {
-        return new QueryParser(Version.LUCENE_43, Fields.DOCUMENT_KEY, getSearchingAnalyzer());
-    }
-
-    /**
-     * Resets the searching analyzers
-     */
-    @Override
-    protected void resetSearchingAnalyzer() {
-        if (productSearchFieldAnalyzer != null) {
-            productSearchFieldAnalyzer.clear();
-        }
-        if (vendorSearchFieldAnalyzer != null) {
-            vendorSearchFieldAnalyzer.clear();
-        }
-    }
-
-    /**
-     * Saves a CPE IndexEntry into the Lucene index.
-     *
-     * @param entry a CPE entry.
-     * @throws CorruptIndexException is thrown if the index is corrupt.
-     * @throws IOException is thrown if an IOException occurs.
-     */
-    public void saveEntry(IndexEntry entry) throws CorruptIndexException, IOException {
-        final Document doc = convertEntryToDoc(entry);
-        final Term term = new Term(Fields.DOCUMENT_KEY, entry.getDocumentId());
-        getIndexWriter().updateDocument(term, doc);
-    }
-
-    /**
-     * Converts a CPE entry into a Lucene Document.
-     *
-     * @param entry a CPE IndexEntry.
-     * @return a Lucene Document containing a CPE IndexEntry.
-     */
-    protected Document convertEntryToDoc(IndexEntry entry) {
-        final Document doc = new Document();
-
-        final Field vendor = new TextField(Fields.VENDOR, entry.getVendor(), Field.Store.YES);
-        doc.add(vendor);
-
-        final Field product = new TextField(Fields.PRODUCT, entry.getProduct(), Field.Store.YES);
-        doc.add(product);
-        return doc;
-    }
-}

dependency-check-core/src/main/java/org/owasp/dependencycheck/data/lucene/AbstractIndex.java

@@ -1,341 +0,0 @@
-/*
- * This file is part of dependency-check-core.
- *
- * Dependency-check-core is free software: you can redistribute it and/or modify it
- * under the terms of the GNU General Public License as published by the Free
- * Software Foundation, either version 3 of the License, or (at your option) any
- * later version.
- *
- * Dependency-check-core is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
- * details.
- *
- * You should have received a copy of the GNU General Public License along with
- * dependency-check-core. If not, see http://www.gnu.org/licenses/.
- *
- * Copyright (c) 2012 Jeremy Long. All Rights Reserved.
- */
-package org.owasp.dependencycheck.data.lucene;
-
-import java.io.IOException;
-import java.util.logging.Level;
-import java.util.logging.Logger;
-import org.apache.lucene.analysis.Analyzer;
-import org.apache.lucene.document.Document;
-import org.apache.lucene.index.CorruptIndexException;
-import org.apache.lucene.index.DirectoryReader;
-import org.apache.lucene.index.IndexReader;
-import org.apache.lucene.index.IndexWriter;
-import org.apache.lucene.index.IndexWriterConfig;
-import org.apache.lucene.queryparser.classic.ParseException;
-import org.apache.lucene.queryparser.classic.QueryParser;
-import org.apache.lucene.search.IndexSearcher;
-import org.apache.lucene.search.Query;
-import org.apache.lucene.search.TopDocs;
-import org.apache.lucene.store.Directory;
-import org.apache.lucene.store.LockObtainFailedException;
-import org.apache.lucene.util.Version;
-
-/**
- * The base Index for other index objects. Implements the open and close
- * methods.
- *
- * @author Jeremy Long (jeremy.long@owasp.org)
- */
-public abstract class AbstractIndex {
-
-    /**
-     * The Lucene directory containing the index.
-     */
-    private Directory directory;
-    /**
-     * The IndexWriter for the Lucene index.
-     */
-    private IndexWriter indexWriter;
-    /**
-     * The Lucene IndexReader.
-     */
-    private IndexReader indexReader;
-    /**
-     * The Lucene IndexSearcher.
-     */
-    private IndexSearcher indexSearcher;
-    /**
-     * The Lucene Analyzer used for Indexing.
-     */
-    private Analyzer indexingAnalyzer;
-    /**
-     * The Lucene Analyzer used for Searching.
-     */
-    private Analyzer searchingAnalyzer;
-    /**
-     * The Lucene QueryParser used for Searching.
-     */
-    private QueryParser queryParser;
-    /**
-     * Indicates whether or not the Lucene Index is open.
-     */
-    private boolean indexOpen = false;
-
-    /**
-     * Opens the CPE Index.
-     *
-     * @throws IOException is thrown if an IOException occurs opening the index.
-     */
-    public void open() throws IOException {
-        directory = this.getDirectory();
-        indexingAnalyzer = this.getIndexingAnalyzer();
-        searchingAnalyzer = this.getSearchingAnalyzer();
-        indexOpen = true;
-    }
-
-    /**
-     * Commits any pending changes.
-     */
-    public void commit() {
-        if (indexWriter != null) {
-            try {
-                indexWriter.commit();
-            } catch (CorruptIndexException ex) {
-                final String msg = "Unable to update database, there is a corrupt index.";
-                Logger.getLogger(AbstractIndex.class.getName()).log(Level.SEVERE, msg);
-                Logger.getLogger(AbstractIndex.class.getName()).log(Level.FINE, null, ex);
-            } catch (IOException ex) {
-                final String msg = "Unable to update database due to an IO error.";
-                Logger.getLogger(AbstractIndex.class.getName()).log(Level.SEVERE, msg);
-                Logger.getLogger(AbstractIndex.class.getName()).log(Level.FINE, null, ex);
-            }
-        }
-    }
-
-    /**
-     * Closes the CPE Index.
-     */
-    public void close() {
-        if (indexWriter != null) {
-            commit();
-            try {
-                indexWriter.close(true);
-            } catch (CorruptIndexException ex) {
-                final String msg = "Unable to update database, there is a corrupt index.";
-                Logger.getLogger(AbstractIndex.class.getName()).log(Level.SEVERE, msg);
-                Logger.getLogger(AbstractIndex.class.getName()).log(Level.FINE, null, ex);
-            } catch (IOException ex) {
-                final String msg = "Unable to update database due to an IO error.";
-                Logger.getLogger(AbstractIndex.class.getName()).log(Level.SEVERE, msg);
-                Logger.getLogger(AbstractIndex.class.getName()).log(Level.FINE, null, ex);
-            } finally {
-                indexWriter = null;
-            }
-        }
-        if (indexSearcher != null) {
-            indexSearcher = null;
-        }
-
-        if (indexingAnalyzer != null) {
-            indexingAnalyzer.close();
-            indexingAnalyzer = null;
-        }
-
-        if (searchingAnalyzer != null) {
-            searchingAnalyzer.close();
-            searchingAnalyzer = null;
-        }
-
-        try {
-            directory.close();
-        } catch (IOException ex) {
-            final String msg = "Unable to update database due to an IO error.";
-            Logger.getLogger(AbstractIndex.class.getName()).log(Level.SEVERE, msg);
-            Logger.getLogger(AbstractIndex.class.getName()).log(Level.FINE, null, ex);
-        } finally {
-            directory = null;
-        }
-        indexOpen = false;
-    }
-
-    /**
-     * Returns the status of the data source - is the index open.
-     *
-     * @return true or false.
-     */
-    public boolean isOpen() {
-        return indexOpen;
-    }
-
-    /**
-     * Opens the Lucene Index Writer.
-     *
-     * @throws CorruptIndexException is thrown if the Lucene index is corrupt.
-     * @throws IOException is thrown if an IOException occurs opening the index.
-     */
-    public void openIndexWriter() throws CorruptIndexException, IOException {
-        if (!isOpen()) {
-            open();
-        }
-        final IndexWriterConfig conf = new IndexWriterConfig(Version.LUCENE_43, indexingAnalyzer);
-        indexWriter = new IndexWriter(directory, conf);
-    }
-
-    /**
-     * Retrieves the IndexWriter for the Lucene Index.
-     *
-     * @return an IndexWriter.
-     * @throws CorruptIndexException is thrown if the Lucene Index is corrupt.
-     * @throws LockObtainFailedException is thrown if there is an exception
-     * obtaining a lock on the Lucene index.
-     * @throws IOException is thrown if an IOException occurs opening the index.
-     */
-    public IndexWriter getIndexWriter() throws CorruptIndexException, LockObtainFailedException, IOException {
-        if (indexWriter == null) {
-            openIndexWriter();
-        }
-        return indexWriter;
-    }
-
-    /**
-     * Opens the Lucene Index for reading.
-     *
-     * @throws CorruptIndexException is thrown if the index is corrupt.
-     * @throws IOException is thrown if there is an exception reading the index.
-     */
-    public void openIndexReader() throws CorruptIndexException, IOException {
-        if (!isOpen()) {
-            open();
-        }
-        //indexReader = IndexReader.open(directory, true);
-        indexReader = DirectoryReader.open(directory);
-    }
-
-    /**
-     * Returns an IndexSearcher for the Lucene Index.
-     *
-     * @return an IndexSearcher.
-     * @throws CorruptIndexException is thrown if the index is corrupt.
-     * @throws IOException is thrown if there is an exception reading the index.
-     */
-    protected IndexSearcher getIndexSearcher() throws CorruptIndexException, IOException {
-        if (indexReader == null) {
-            openIndexReader();
-        }
-        if (indexSearcher == null) {
-            indexSearcher = new IndexSearcher(indexReader);
-        }
-        return indexSearcher;
-    }
-
-    /**
-     * Returns an Analyzer to be used when indexing.
-     *
-     * @return an Analyzer.
-     */
-    public Analyzer getIndexingAnalyzer() {
-        if (indexingAnalyzer == null) {
-            indexingAnalyzer = createIndexingAnalyzer();
-        }
-        return indexingAnalyzer;
-    }
-
-    /**
-     * Returns an analyzer used for searching the index
-     *
-     * @return a lucene analyzer
-     */
-    protected Analyzer getSearchingAnalyzer() {
-        if (searchingAnalyzer == null) {
-            searchingAnalyzer = createSearchingAnalyzer();
-        }
-        return searchingAnalyzer;
-    }
-
-    /**
-     * Gets a query parser
-     *
-     * @return a query parser
-     */
-    protected QueryParser getQueryParser() {
-        if (queryParser == null) {
-            queryParser = createQueryParser();
-        }
-        return queryParser;
-    }
-
-    /**
-     * Searches the index using the given search string.
-     *
-     * @param searchString the query text
-     * @param maxQueryResults the maximum number of documents to return
-     * @return the TopDocs found by the search
-     * @throws ParseException thrown when the searchString is invalid
-     * @throws IOException is thrown if there is an issue with the underlying
-     * Index
-     */
-    public TopDocs search(String searchString, int maxQueryResults) throws ParseException, IOException {
-        final QueryParser parser = getQueryParser();
-        final Query query = parser.parse(searchString);
-        resetSearchingAnalyzer();
-        final IndexSearcher is = getIndexSearcher();
-        return is.search(query, maxQueryResults);
-    }
-
-    /**
-     * Searches the index using the given query.
-     *
-     * @param query the query used to search the index
-     * @param maxQueryResults the max number of results to return
-     * @return the TopDocs found be the query
-     * @throws CorruptIndexException thrown if the Index is corrupt
-     * @throws IOException thrown if there is an IOException
-     */
-    public TopDocs search(Query query, int maxQueryResults) throws CorruptIndexException, IOException {
-        final IndexSearcher is = getIndexSearcher();
-        return is.search(query, maxQueryResults);
-    }
-
-    /**
-     * Retrieves a document from the Index.
-     *
-     * @param documentId the id of the document to retrieve
-     * @return the Document
-     * @throws IOException thrown if there is an IOException
-     */
-    public Document getDocument(int documentId) throws IOException {
-        final IndexSearcher is = getIndexSearcher();
-        return is.doc(documentId);
-    }
-
-    /**
-     * Gets the directory that contains the Lucene Index.
-     *
-     * @return a Lucene Directory
-     * @throws IOException is thrown when an IOException occurs
-     */
-    public abstract Directory getDirectory() throws IOException;
-
-    /**
-     * Creates the Lucene Analyzer used when indexing.
-     *
-     * @return a Lucene Analyzer
-     */
-    public abstract Analyzer createIndexingAnalyzer();
-
-    /**
-     * Creates the Lucene Analyzer used when querying the index.
-     *
-     * @return a Lucene Analyzer
-     */
-    public abstract Analyzer createSearchingAnalyzer();
-
-    /**
-     * Creates the Lucene QueryParser used when querying the index.
-     *
-     * @return a QueryParser
-     */
-    public abstract QueryParser createQueryParser();
-
-    /**
-     * Resets the searching analyzers
-     */
-    protected abstract void resetSearchingAnalyzer();
-}

dependency-check-core/src/main/java/org/owasp/dependencycheck/data/lucene/TokenPairConcatenatingFilter.java

@@ -47,6 +47,24 @@ public final class TokenPairConcatenatingFilter extends TokenFilter {
      */
     private final LinkedList<String> words;
 
+    /**
+     * Returns the previous word. This is needed in the test cases.
+     *
+     * @return te previous word
+     */
+    protected String getPreviousWord() {
+        return previousWord;
+    }
+
+    /**
+     * Returns the words list. This is needed in the test cases.
+     *
+     * @return the words list
+     */
+    protected LinkedList<String> getWords() {
+        return words;
+    }
+
     /**
      * Constructs a new TokenPairConcatenatingFilter.
      *

dependency-check-core/src/main/java/org/owasp/dependencycheck/data/nvdcve/CveDB.java

@@ -58,7 +58,7 @@ public class CveDB {
     /**
      * The version of the current DB Schema.
      */
-    public static final String DB_SCHEMA_VERSION = "2.5";
+    public static final String DB_SCHEMA_VERSION = "2.6";
     /**
      * Database connection
      */
@@ -162,27 +162,16 @@ public class CveDB {
             value = "DMI_EMPTY_DB_PASSWORD",
             justification = "Yes, I know... Blank password.")
     public void open() throws IOException, SQLException, DatabaseException, ClassNotFoundException {
-        /*
-         * TODO - make it so we can exteralize the database (lucene index is a problem), could I store it as a blob
-         *        and just download it when needed?
-         */
-//        String dbDriver = Settings.getString(Settings.KEYS.DB_DRIVER);
-//        String dbConnStr = Settings.getString(Settings.KEYS.DB_CONNECTION_STRING);
-//        if (dbDriver != null && dbConnStr != null) {
-//            Class.forName(dbDriver);
-//            conn = DriverManager.getConnection(dbConnStr);
-//        } else { //use the embeded version
         final String fileName = CveDB.getDataDirectory().getCanonicalPath();
         final File f = new File(fileName, "cve." + DB_SCHEMA_VERSION);
         final File check = new File(f.getAbsolutePath() + ".h2.db");
         final boolean createTables = !check.exists();
-        final String connStr = "jdbc:h2:file:" + f.getAbsolutePath();
+        final String connStr = String.format("jdbc:h2:file:%s;AUTO_SERVER=TRUE", f.getAbsolutePath());
         Class.forName("org.h2.Driver");
         conn = DriverManager.getConnection(connStr, "sa", "");
         if (createTables) {
             createTables();
         }
-//        }
     }
 
     /**

dependency-check-core/src/main/java/org/owasp/dependencycheck/data/nvdcve/InvalidDataException.java

@@ -16,7 +16,7 @@
  *
  * Copyright (c) 2012 Jeremy Long. All Rights Reserved.
  */
-package org.owasp.dependencycheck.data.nvdcve.xml;
+package org.owasp.dependencycheck.data.nvdcve;
 
 /**
  * An InvalidDataDataException is a generic exception used when trying to load

dependency-check-core/src/main/java/org/owasp/dependencycheck/data/nvdcve/NvdCve12Handler.java

@@ -16,7 +16,7 @@
  *
  * Copyright (c) 2012 Jeremy Long. All Rights Reserved.
  */
-package org.owasp.dependencycheck.data.nvdcve.xml;
+package org.owasp.dependencycheck.data.nvdcve;
 
 import java.util.ArrayList;
 import java.util.HashMap;

dependency-check-core/src/main/java/org/owasp/dependencycheck/data/nvdcve/NvdCve20Handler.java

@@ -16,7 +16,7 @@
  *
  * Copyright (c) 2012 Jeremy Long. All Rights Reserved.
  */
-package org.owasp.dependencycheck.data.nvdcve.xml;
+package org.owasp.dependencycheck.data.nvdcve;
 
 import java.io.IOException;
 import java.util.List;
@@ -24,9 +24,7 @@ import java.util.Map;
 import java.util.logging.Level;
 import java.util.logging.Logger;
 import org.apache.lucene.index.CorruptIndexException;
-import org.owasp.dependencycheck.data.cpe.Index;
-import org.owasp.dependencycheck.data.nvdcve.CveDB;
-import org.owasp.dependencycheck.data.nvdcve.DatabaseException;
+import org.owasp.dependencycheck.data.cpe.CpeIndexWriter;
 import org.owasp.dependencycheck.dependency.Reference;
 import org.owasp.dependencycheck.dependency.Vulnerability;
 import org.owasp.dependencycheck.dependency.VulnerableSoftware;
@@ -243,9 +241,7 @@ public class NvdCve20Handler extends DefaultHandler {
     }
 
     /**
-     * Saves a vulnerability to the CVE Database. This is a callback method
-     * called by the Sax Parser Handler
-     * {@link org.owasp.dependencycheck.data.nvdcve.xml.NvdCve20Handler}.
+     * Saves a vulnerability to the CVE Database.
      *
      * @param vuln the vulnerability to store in the database
      * @throws DatabaseException thrown if there is an error writing to the
@@ -253,7 +249,7 @@ public class NvdCve20Handler extends DefaultHandler {
      * @throws CorruptIndexException is thrown if the CPE Index is corrupt
      * @throws IOException thrown if there is an IOException with the CPE Index
      */
-    public void saveEntry(Vulnerability vuln) throws DatabaseException, CorruptIndexException, IOException {
+    private void saveEntry(Vulnerability vuln) throws DatabaseException, CorruptIndexException, IOException {
         if (cveDB == null) {
             return;
         }
@@ -274,14 +270,14 @@ public class NvdCve20Handler extends DefaultHandler {
     /**
      * the cpe index.
      */
-    private Index cpeIndex;
+    private CpeIndexWriter cpeIndex;
 
     /**
-     * Sets the cpe index.
+     * Sets the cpe index writer.
      *
      * @param index the CPE Lucene Index
      */
-    void setCpeIndex(Index index) {
+    public void setCpeIndex(CpeIndexWriter index) {
         cpeIndex = index;
     }
 

dependency-check-core/src/main/java/org/owasp/dependencycheck/data/nvdcve/xml/DatabaseUpdater.java

@@ -1,671 +0,0 @@
-/*
- * This file is part of dependency-check-core.
- *
- * Dependency-check-core is free software: you can redistribute it and/or modify it
- * under the terms of the GNU General Public License as published by the Free
- * Software Foundation, either version 3 of the License, or (at your option) any
- * later version.
- *
- * Dependency-check-core is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
- * details.
- *
- * You should have received a copy of the GNU General Public License along with
- * dependency-check-core. If not, see http://www.gnu.org/licenses/.
- *
- * Copyright (c) 2012 Jeremy Long. All Rights Reserved.
- */
-package org.owasp.dependencycheck.data.nvdcve.xml;
-
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileNotFoundException;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.io.OutputStreamWriter;
-import javax.xml.parsers.ParserConfigurationException;
-import org.owasp.dependencycheck.data.CachedWebDataSource;
-import java.net.MalformedURLException;
-import java.net.URL;
-import java.sql.SQLException;
-import java.util.Calendar;
-import java.util.Date;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Properties;
-import java.util.logging.Level;
-import java.util.logging.Logger;
-import javax.xml.parsers.SAXParser;
-import javax.xml.parsers.SAXParserFactory;
-import org.owasp.dependencycheck.data.UpdateException;
-import org.owasp.dependencycheck.data.cpe.Index;
-import org.owasp.dependencycheck.data.nvdcve.CveDB;
-import org.owasp.dependencycheck.dependency.VulnerableSoftware;
-import org.owasp.dependencycheck.utils.DownloadFailedException;
-import org.owasp.dependencycheck.utils.Downloader;
-import org.owasp.dependencycheck.utils.FileUtils;
-import org.owasp.dependencycheck.utils.InvalidSettingException;
-import org.owasp.dependencycheck.utils.Settings;
-import org.xml.sax.SAXException;
-import org.owasp.dependencycheck.data.nvdcve.DatabaseException;
-
-/**
- *
- * @author Jeremy Long (jeremy.long@owasp.org)
- */
-public class DatabaseUpdater implements CachedWebDataSource {
-
-    /**
-     * The name of the properties file containing the timestamp of the last
-     * update.
-     */
-    private static final String UPDATE_PROPERTIES_FILE = "lastupdated.prop";
-    /**
-     * The properties file key for the last updated field - used to store the
-     * last updated time of the Modified NVD CVE xml file.
-     */
-    private static final String LAST_UPDATED_MODIFIED = "lastupdated.modified";
-    /**
-     * Stores the last updated time for each of the NVD CVE files. These
-     * timestamps should be updated if we process the modified file within 7
-     * days of the last update.
-     */
-    private static final String LAST_UPDATED_BASE = "lastupdated.";
-    /**
-     * Modified key word.
-     */
-    public static final String MODIFIED = "modified";
-    /**
-     * Reference to the Cve Database.
-     */
-    private CveDB cveDB = null;
-    /**
-     * Reference to the Cpe Index.
-     */
-    private Index cpeIndex = null;
-
-    /**
-     * <p>Downloads the latest NVD CVE XML file from the web and imports it into
-     * the current CVE Database.</p>
-     *
-     * @throws UpdateException is thrown if there is an error updating the
-     * database
-     */
-    public void update() throws UpdateException {
-        try {
-            final Map<String, NvdCveUrl> update = updateNeeded();
-            int maxUpdates = 0;
-            for (NvdCveUrl cve : update.values()) {
-                if (cve.getNeedsUpdate()) {
-                    maxUpdates += 1;
-                }
-            }
-            if (maxUpdates > 3) {
-                Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.INFO,
-                        "NVD CVE requires several updates; this could take a couple of minutes.");
-            }
-            if (maxUpdates > 0) {
-                openDataStores();
-            }
-            int count = 0;
-
-            for (NvdCveUrl cve : update.values()) {
-                if (cve.getNeedsUpdate()) {
-                    count += 1;
-                    Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.INFO,
-                            "Updating NVD CVE ({0} of {1})", new Object[]{count, maxUpdates});
-                    URL url = new URL(cve.getUrl());
-                    File outputPath = null;
-                    File outputPath12 = null;
-                    try {
-                        Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.INFO,
-                                "Downloading {0}", cve.getUrl());
-
-                        outputPath = File.createTempFile("cve" + cve.getId() + "_", ".xml");
-                        Downloader.fetchFile(url, outputPath, false);
-
-                        url = new URL(cve.getOldSchemaVersionUrl());
-                        outputPath12 = File.createTempFile("cve_1_2_" + cve.getId() + "_", ".xml");
-                        Downloader.fetchFile(url, outputPath12, false);
-
-                        Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.INFO,
-                                "Processing {0}", cve.getUrl());
-
-                        importXML(outputPath, outputPath12);
-
-                        cveDB.commit();
-                        cpeIndex.commit();
-
-                        writeLastUpdatedPropertyFile(cve);
-
-                        Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.INFO,
-                                "Completed update {0} of {1}", new Object[]{count, maxUpdates});
-                    } catch (FileNotFoundException ex) {
-                        throw new UpdateException(ex);
-                    } catch (ParserConfigurationException ex) {
-                        throw new UpdateException(ex);
-                    } catch (SAXException ex) {
-                        throw new UpdateException(ex);
-                    } catch (IOException ex) {
-                        throw new UpdateException(ex);
-                    } catch (SQLException ex) {
-                        throw new UpdateException(ex);
-                    } catch (DatabaseException ex) {
-                        throw new UpdateException(ex);
-                    } catch (ClassNotFoundException ex) {
-                        throw new UpdateException(ex);
-                    } finally {
-                        boolean deleted = false;
-                        try {
-                            if (outputPath != null && outputPath.exists()) {
-                                deleted = outputPath.delete();
-                            }
-                        } finally {
-                            if (outputPath != null && (outputPath.exists() || !deleted)) {
-                                outputPath.deleteOnExit();
-                            }
-                        }
-                        try {
-                            deleted = false;
-                            if (outputPath12 != null && outputPath12.exists()) {
-                                deleted = outputPath12.delete();
-                            }
-                        } finally {
-                            if (outputPath12 != null && (outputPath12.exists() || !deleted)) {
-                                outputPath12.deleteOnExit();
-                            }
-                        }
-                    }
-                }
-            }
-            if (maxUpdates >= 1) {
-                ensureModifiedIsInLastUpdatedProperties(update);
-                cveDB.cleanupDatabase();
-            }
-        } catch (MalformedURLException ex) {
-            throw new UpdateException(ex);
-        } catch (DownloadFailedException ex) {
-            throw new UpdateException(ex);
-        } finally {
-            closeDataStores();
-        }
-    }
-
-    /**
-     * Imports the NVD CVE XML File into the Lucene Index.
-     *
-     * @param file the file containing the NVD CVE XML
-     * @param oldVersion contains the file containing the NVD CVE XML 1.2
-     * @throws ParserConfigurationException is thrown if there is a parser
-     * configuration exception
-     * @throws SAXException is thrown if there is a SAXException
-     * @throws IOException is thrown if there is a ioexception
-     * @throws SQLException is thrown if there is a sql exception
-     * @throws DatabaseException is thrown if there is a database exception
-     * @throws ClassNotFoundException thrown if the h2 database driver cannot be
-     * loaded
-     */
-    private void importXML(File file, File oldVersion)
-            throws ParserConfigurationException, SAXException, IOException, SQLException, DatabaseException, ClassNotFoundException {
-
-        final SAXParserFactory factory = SAXParserFactory.newInstance();
-        final SAXParser saxParser = factory.newSAXParser();
-
-        final NvdCve12Handler cve12Handler = new NvdCve12Handler();
-        saxParser.parse(oldVersion, cve12Handler);
-        final Map<String, List<VulnerableSoftware>> prevVersionVulnMap = cve12Handler.getVulnerabilities();
-
-        final NvdCve20Handler cve20Handler = new NvdCve20Handler();
-        cve20Handler.setCveDB(cveDB);
-        cve20Handler.setPrevVersionVulnMap(prevVersionVulnMap);
-        cve20Handler.setCpeIndex(cpeIndex);
-        saxParser.parse(file, cve20Handler);
-    }
-
-    /**
-     * Closes the CVE and CPE data stores.
-     */
-    private void closeDataStores() {
-        if (cveDB != null) {
-            try {
-                cveDB.close();
-            } catch (Exception ignore) {
-                Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.FINEST, "Error closing the cveDB", ignore);
-            }
-        }
-        if (cpeIndex != null) {
-            try {
-                cpeIndex.close();
-            } catch (Exception ignore) {
-                Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.FINEST, "Error closing the cpeIndex", ignore);
-            }
-        }
-    }
-
-    /**
-     * Opens the CVE and CPE data stores.
-     *
-     * @throws UpdateException thrown if a data store cannot be opened
-     */
-    private void openDataStores() throws UpdateException {
-        //open the cve and cpe data stores
-        try {
-            cveDB = new CveDB();
-            cveDB.open();
-            cpeIndex = new Index();
-            cpeIndex.openIndexWriter();
-        } catch (IOException ex) {
-            closeDataStores();
-            Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.FINE, "IO Error opening databases", ex);
-            throw new UpdateException("Error updating the CPE/CVE data, please see the log file for more details.");
-        } catch (SQLException ex) {
-            closeDataStores();
-            Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.FINE, "SQL Exception opening databases", ex);
-            throw new UpdateException("Error updating the CPE/CVE data, please see the log file for more details.");
-        } catch (DatabaseException ex) {
-            closeDataStores();
-            Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.FINE, "Database Exception opening databases", ex);
-            throw new UpdateException("Error updating the CPE/CVE data, please see the log file for more details.");
-        } catch (ClassNotFoundException ex) {
-            closeDataStores();
-            Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.FINE, "Class not found exception opening databases", ex);
-            throw new UpdateException("Error updating the CPE/CVE data, please see the log file for more details.");
-        }
-    }
-
-    //<editor-fold defaultstate="collapsed" desc="Code to read/write properties files regarding the last update dates">
-    /**
-     * Writes a properties file containing the last updated date to the
-     * VULNERABLE_CPE directory.
-     *
-     * @param updatedValue the updated nvdcve entry
-     * @throws UpdateException is thrown if there is an update exception
-     */
-    private void writeLastUpdatedPropertyFile(NvdCveUrl updatedValue) throws UpdateException {
-        if (updatedValue == null) {
-            return;
-        }
-        String dir;
-        try {
-            dir = CveDB.getDataDirectory().getCanonicalPath();
-        } catch (IOException ex) {
-            Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.FINE, "Error updating the databases propterty file.", ex);
-            throw new UpdateException("Unable to locate last updated properties file.", ex);
-        }
-        final File cveProp = new File(dir, UPDATE_PROPERTIES_FILE);
-        final Properties prop = new Properties();
-        if (cveProp.exists()) {
-            FileInputStream in = null;
-            try {
-                in = new FileInputStream(cveProp);
-                prop.load(in);
-            } catch (Exception ignoreMe) {
-                Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.FINEST, null, ignoreMe);
-            } finally {
-                if (in != null) {
-                    try {
-                        in.close();
-                    } catch (Exception ignoreMeToo) {
-                        Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.FINEST, null, ignoreMeToo);
-                    }
-                }
-            }
-
-        }
-        prop.put("version", CveDB.DB_SCHEMA_VERSION);
-        prop.put(LAST_UPDATED_BASE + updatedValue.getId(), String.valueOf(updatedValue.getTimestamp()));
-
-        OutputStream os = null;
-        OutputStreamWriter out = null;
-        try {
-            os = new FileOutputStream(cveProp);
-            out = new OutputStreamWriter(os, "UTF-8");
-            prop.store(out, dir);
-        } catch (FileNotFoundException ex) {
-            Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.FINE, null, ex);
-            throw new UpdateException("Unable to find last updated properties file.", ex);
-        } catch (IOException ex) {
-            Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.FINE, null, ex);
-            throw new UpdateException("Unable to update last updated properties file.", ex);
-        } finally {
-            if (out != null) {
-                try {
-                    out.close();
-                } catch (IOException ex) {
-                    Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.FINEST, null, ex);
-                }
-            }
-            if (os != null) {
-                try {
-                    os.close();
-                } catch (IOException ex) {
-                    Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.FINEST, null, ex);
-                }
-            }
-        }
-    }
-
-    /**
-     * Determines if the index needs to be updated. This is done by fetching the
-     * nvd cve meta data and checking the last update date. If the data needs to
-     * be refreshed this method will return the NvdCveUrl for the files that
-     * need to be updated.
-     *
-     * @return the NvdCveUrl of the files that need to be updated.
-     * @throws MalformedURLException is thrown if the URL for the NVD CVE Meta
-     * data is incorrect.
-     * @throws DownloadFailedException is thrown if there is an error.
-     * downloading the nvd cve download data file.
-     * @throws UpdateException Is thrown if there is an issue with the last
-     * updated properties file.
-     */
-    public Map<String, NvdCveUrl> updateNeeded() throws MalformedURLException, DownloadFailedException, UpdateException {
-
-        Map<String, NvdCveUrl> currentlyPublished;
-        try {
-            currentlyPublished = retrieveCurrentTimestampsFromWeb();
-        } catch (InvalidDataException ex) {
-            final String msg = "Unable to retrieve valid timestamp from nvd cve downloads page";
-            Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.FINE, msg, ex);
-            throw new DownloadFailedException(msg, ex);
-
-        } catch (InvalidSettingException ex) {
-            Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.FINE, "Invalid setting found when retrieving timestamps", ex);
-            throw new DownloadFailedException("Invalid settings", ex);
-        }
-
-        if (currentlyPublished == null) {
-            throw new DownloadFailedException("Unable to retrieve valid timestamp from nvd cve downloads page");
-        }
-        String dir;
-        try {
-            dir = CveDB.getDataDirectory().getCanonicalPath();
-        } catch (IOException ex) {
-            Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.FINE, "CveDB data directory doesn't exist?", ex);
-            throw new UpdateException("Unable to locate last updated properties file.", ex);
-        }
-
-        final File f = new File(dir);
-        if (f.exists()) {
-            final File cveProp = new File(dir, UPDATE_PROPERTIES_FILE);
-            if (cveProp.exists()) {
-                final Properties prop = new Properties();
-                InputStream is = null;
-                try {
-                    is = new FileInputStream(cveProp);
-                    prop.load(is);
-
-                    boolean deleteAndRecreate = false;
-                    float version;
-
-                    if (prop.getProperty("version") == null) {
-                        deleteAndRecreate = true;
-                    } else {
-                        try {
-                            version = Float.parseFloat(prop.getProperty("version"));
-                            final float currentVersion = Float.parseFloat(CveDB.DB_SCHEMA_VERSION);
-                            if (currentVersion > version) {
-                                deleteAndRecreate = true;
-                            }
-                        } catch (NumberFormatException ex) {
-                            deleteAndRecreate = true;
-                        }
-                    }
-                    if (deleteAndRecreate) {
-                        Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.INFO, "The database version is old. Rebuilding the database.");
-                        is.close();
-                        //this is an old version of the lucene index - just delete it
-                        FileUtils.delete(f);
-
-                        //this importer also updates the CPE index and it is also using an old version
-                        final Index cpeId = new Index();
-                        final File cpeDir = cpeId.getDataDirectory();
-                        FileUtils.delete(cpeDir);
-                        return currentlyPublished;
-                    }
-
-                    final long lastUpdated = Long.parseLong(prop.getProperty(LAST_UPDATED_MODIFIED, "0"));
-                    final Date now = new Date();
-                    final int days = Settings.getInt(Settings.KEYS.CVE_MODIFIED_VALID_FOR_DAYS, 7);
-                    final int start = Settings.getInt(Settings.KEYS.CVE_START_YEAR, 2002);
-                    final int end = Calendar.getInstance().get(Calendar.YEAR);
-                    if (lastUpdated == currentlyPublished.get(MODIFIED).timestamp) {
-                        currentlyPublished.clear(); //we don't need to update anything.
-                    } else if (withinRange(lastUpdated, now.getTime(), days)) {
-                        currentlyPublished.get(MODIFIED).setNeedsUpdate(true);
-                        for (int i = start; i <= end; i++) {
-                            currentlyPublished.get(String.valueOf(i)).setNeedsUpdate(false);
-                        }
-                    } else { //we figure out which of the several XML files need to be downloaded.
-                        currentlyPublished.get(MODIFIED).setNeedsUpdate(false);
-                        for (int i = start; i <= end; i++) {
-                            final NvdCveUrl cve = currentlyPublished.get(String.valueOf(i));
-                            long currentTimestamp = 0;
-                            try {
-                                currentTimestamp = Long.parseLong(prop.getProperty(LAST_UPDATED_BASE + String.valueOf(i), "0"));
-                            } catch (NumberFormatException ex) {
-                                final String msg = String.format("Error parsing '%s' '%s' from nvdcve.lastupdated",
-                                        LAST_UPDATED_BASE, String.valueOf(i));
-                                Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.FINE, msg, ex);
-                            }
-                            if (currentTimestamp == cve.getTimestamp()) {
-                                cve.setNeedsUpdate(false); //they default to true.
-                            }
-                        }
-                    }
-                } catch (FileNotFoundException ex) {
-                    Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.FINEST, null, ex);
-                } catch (IOException ex) {
-                    Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.FINEST, null, ex);
-                } catch (NumberFormatException ex) {
-                    Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.FINEST, null, ex);
-                } finally {
-                    if (is != null) {
-                        try {
-                            is.close();
-                        } catch (IOException ex) {
-                            Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.FINEST, null, ex);
-                        }
-                    }
-                }
-            }
-        }
-        return currentlyPublished;
-    }
-
-    /**
-     * Determines if the epoch date is within the range specified of the
-     * compareTo epoch time. This takes the (compareTo-date)/1000/60/60/24 to
-     * get the number of days. If the calculated days is less then the range the
-     * date is considered valid.
-     *
-     * @param date the date to be checked.
-     * @param compareTo the date to compare to.
-     * @param range the range in days to be considered valid.
-     * @return whether or not the date is within the range.
-     */
-    private boolean withinRange(long date, long compareTo, int range) {
-        final double differenceInDays = (compareTo - date) / 1000.0 / 60.0 / 60.0 / 24.0;
-        return differenceInDays < range;
-    }
-
-    /**
-     * Retrieves the timestamps from the NVD CVE meta data file.
-     *
-     * @return the timestamp from the currently published nvdcve downloads page
-     * @throws MalformedURLException thrown if the URL for the NVD CCE Meta data
-     * is incorrect.
-     * @throws DownloadFailedException thrown if there is an error downloading
-     * the nvd cve meta data file
-     * @throws InvalidDataException thrown if there is an exception parsing the
-     * timestamps
-     * @throws InvalidSettingException thrown if the settings are invalid
-     */
-    protected Map<String, NvdCveUrl> retrieveCurrentTimestampsFromWeb()
-            throws MalformedURLException, DownloadFailedException, InvalidDataException, InvalidSettingException {
-
-        final Map<String, NvdCveUrl> map = new HashMap<String, NvdCveUrl>();
-        String retrieveUrl = Settings.getString(Settings.KEYS.CVE_MODIFIED_20_URL);
-
-        NvdCveUrl item = new NvdCveUrl();
-        item.setNeedsUpdate(false); //the others default to true, to make life easier later this should default to false.
-        item.setId(MODIFIED);
-        item.setUrl(retrieveUrl);
-        item.setOldSchemaVersionUrl(Settings.getString(Settings.KEYS.CVE_MODIFIED_12_URL));
-
-        item.timestamp = Downloader.getLastModified(new URL(retrieveUrl));
-        map.put(MODIFIED, item);
-
-        final int start = Settings.getInt(Settings.KEYS.CVE_START_YEAR);
-        final int end = Calendar.getInstance().get(Calendar.YEAR);
-        final String baseUrl20 = Settings.getString(Settings.KEYS.CVE_SCHEMA_2_0);
-        final String baseUrl12 = Settings.getString(Settings.KEYS.CVE_SCHEMA_1_2);
-        for (int i = start; i <= end; i++) {
-            retrieveUrl = String.format(baseUrl20, i);
-            item = new NvdCveUrl();
-            item.setId(Integer.toString(i));
-            item.setUrl(retrieveUrl);
-            item.setOldSchemaVersionUrl(String.format(baseUrl12, i));
-            item.setTimestamp(Downloader.getLastModified(new URL(retrieveUrl)));
-            map.put(item.id, item);
-        }
-        return map;
-    }
-
-    /**
-     * Method to double check that the "modified" nvdcve file is listed and has
-     * a timestamp in the last updated properties file.
-     *
-     * @param update a set of updated NvdCveUrl objects
-     */
-    private void ensureModifiedIsInLastUpdatedProperties(Map<String, NvdCveUrl> update) {
-        try {
-            writeLastUpdatedPropertyFile(update.get(MODIFIED));
-        } catch (UpdateException ex) {
-            Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.FINE, null, ex);
-        }
-    }
-
-    /**
-     * A pojo that contains the Url and timestamp of the current NvdCve XML
-     * files.
-     */
-    protected static class NvdCveUrl {
-
-        /**
-         * an id.
-         */
-        private String id;
-
-        /**
-         * Get the value of id.
-         *
-         * @return the value of id
-         */
-        public String getId() {
-            return id;
-        }
-
-        /**
-         * Set the value of id.
-         *
-         * @param id new value of id
-         */
-        public void setId(String id) {
-            this.id = id;
-        }
-        /**
-         * a url.
-         */
-        private String url;
-
-        /**
-         * Get the value of url.
-         *
-         * @return the value of url
-         */
-        public String getUrl() {
-            return url;
-        }
-
-        /**
-         * Set the value of url.
-         *
-         * @param url new value of url
-         */
-        public void setUrl(String url) {
-            this.url = url;
-        }
-        /**
-         * The 1.2 schema URL.
-         */
-        private String oldSchemaVersionUrl;
-
-        /**
-         * Get the value of oldSchemaVersionUrl.
-         *
-         * @return the value of oldSchemaVersionUrl
-         */
-        public String getOldSchemaVersionUrl() {
-            return oldSchemaVersionUrl;
-        }
-
-        /**
-         * Set the value of oldSchemaVersionUrl.
-         *
-         * @param oldSchemaVersionUrl new value of oldSchemaVersionUrl
-         */
-        public void setOldSchemaVersionUrl(String oldSchemaVersionUrl) {
-            this.oldSchemaVersionUrl = oldSchemaVersionUrl;
-        }
-        /**
-         * a timestamp - epoch time.
-         */
-        private long timestamp;
-
-        /**
-         * Get the value of timestamp - epoch time.
-         *
-         * @return the value of timestamp - epoch time
-         */
-        public long getTimestamp() {
-            return timestamp;
-        }
-
-        /**
-         * Set the value of timestamp - epoch time.
-         *
-         * @param timestamp new value of timestamp - epoch time
-         */
-        public void setTimestamp(long timestamp) {
-            this.timestamp = timestamp;
-        }
-        /**
-         * indicates whether or not this item should be updated.
-         */
-        private boolean needsUpdate = true;
-
-        /**
-         * Get the value of needsUpdate.
-         *
-         * @return the value of needsUpdate
-         */
-        public boolean getNeedsUpdate() {
-            return needsUpdate;
-        }
-
-        /**
-         * Set the value of needsUpdate.
-         *
-         * @param needsUpdate new value of needsUpdate
-         */
-        public void setNeedsUpdate(boolean needsUpdate) {
-            this.needsUpdate = needsUpdate;
-        }
-    }
-    //</editor-fold>
-}

dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/AbstractUpdateTask.java

@@ -0,0 +1,288 @@
+/*
+ * This file is part of dependency-check-core.
+ *
+ * Dependency-check-core is free software: you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation, either version 3 of the License, or (at your option) any
+ * later version.
+ *
+ * Dependency-check-core is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+ * details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * dependency-check-core. If not, see http://www.gnu.org/licenses/.
+ *
+ * Copyright (c) 2012 Jeremy Long. All Rights Reserved.
+ */
+package org.owasp.dependencycheck.data.update;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.MalformedURLException;
+import java.sql.SQLException;
+import java.util.List;
+import java.util.Map;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+import javax.xml.parsers.ParserConfigurationException;
+import javax.xml.parsers.SAXParser;
+import javax.xml.parsers.SAXParserFactory;
+import org.owasp.dependencycheck.data.UpdateException;
+import org.owasp.dependencycheck.data.cpe.CpeIndexWriter;
+import org.owasp.dependencycheck.data.nvdcve.CveDB;
+import org.owasp.dependencycheck.utils.FileUtils;
+import org.owasp.dependencycheck.utils.Settings;
+import org.owasp.dependencycheck.data.nvdcve.DatabaseException;
+import org.owasp.dependencycheck.data.nvdcve.NvdCve12Handler;
+import org.owasp.dependencycheck.data.nvdcve.NvdCve20Handler;
+import org.owasp.dependencycheck.dependency.VulnerableSoftware;
+import org.owasp.dependencycheck.utils.DownloadFailedException;
+import org.xml.sax.SAXException;
+
+/**
+ * Class responsible for updating the CPE and NVDCVE data stores.
+ *
+ * @author Jeremy Long (jeremy.long@owasp.org)
+ */
+public abstract class AbstractUpdateTask implements UpdateTask {
+
+    /**
+     * Initializes the AbstractUpdateTask.
+     *
+     * @param properties information about the data store
+     * @throws MalformedURLException thrown if the configuration contains a
+     * malformed url
+     * @throws DownloadFailedException thrown if the timestamp on a file cannot
+     * be checked
+     * @throws UpdateException thrown if the update fails
+     */
+    public AbstractUpdateTask(DataStoreMetaInfo properties) throws MalformedURLException, DownloadFailedException, UpdateException {
+        this.properties = properties;
+        this.updateable = updatesNeeded();
+    }
+    /**
+     * A collection of updateable NVD CVE items.
+     */
+    private Updateable updateable;
+    /**
+     * Utility to read and write meta-data about the data.
+     */
+    private DataStoreMetaInfo properties = null;
+
+    /**
+     * Returns the data store properties.
+     *
+     * @return the data store properties
+     */
+    protected DataStoreMetaInfo getProperties() {
+        return properties;
+    }
+    /**
+     * Reference to the Cve Database.
+     */
+    private CveDB cveDB = null;
+
+    /**
+     * Returns the CveDB.
+     *
+     * @return the CveDB
+     */
+    protected CveDB getCveDB() {
+        return cveDB;
+    }
+    /**
+     * Reference to the Cpe Index.
+     */
+    private CpeIndexWriter cpeIndex = null;
+
+    /**
+     * Returns the CpeIndex.
+     *
+     * @return the CpeIndex
+     */
+    protected CpeIndexWriter getCpeIndex() {
+        return cpeIndex;
+    }
+
+    /**
+     * Gets whether or not an update is needed.
+     *
+     * @return true or false depending on whether an update is needed
+     */
+    public boolean isUpdateNeeded() {
+        return updateable.isUpdateNeeded();
+    }
+
+    /**
+     * Gets the updateable NVD CVE Entries.
+     *
+     * @return an Updateable object containing the NVD CVE entries
+     */
+    protected Updateable getUpdateable() {
+        return updateable;
+    }
+
+    /**
+     * Determines if the index needs to be updated.
+     *
+     * @return a collection of updateable resources.
+     * @throws MalformedURLException is thrown if the URL for the NVD CVE Meta
+     * data is incorrect.
+     * @throws DownloadFailedException is thrown if there is an error.
+     * downloading the NVD CVE download data file.
+     * @throws UpdateException Is thrown if there is an issue with the last
+     * updated properties file.
+     */
+    protected abstract Updateable updatesNeeded() throws MalformedURLException, DownloadFailedException, UpdateException;
+
+    /**
+     * <p>Updates the data store to the latest version.</p>
+     *
+     * @throws UpdateException is thrown if there is an error updating the
+     * database
+     */
+    public abstract void update() throws UpdateException;
+    /**
+     * A flag indicating whether or not the current data store should be
+     * deleted.
+     */
+    private boolean deleteAndRecreate = false;
+
+    /**
+     * Get the value of deleteAndRecreate.
+     *
+     * @return the value of deleteAndRecreate
+     */
+    public boolean shouldDeleteAndRecreate() {
+        return deleteAndRecreate;
+    }
+
+    /**
+     * Set the value of deleteAndRecreate.
+     *
+     * @param deleteAndRecreate new value of deleteAndRecreate
+     */
+    protected void setDeleteAndRecreate(boolean deleteAndRecreate) {
+        this.deleteAndRecreate = deleteAndRecreate;
+    }
+
+    /**
+     * Deletes the existing data directories.
+     *
+     * @throws IOException thrown if the directory cannot be deleted
+     */
+    protected void deleteExistingData() throws IOException {
+        File data = Settings.getFile(Settings.KEYS.CVE_DATA_DIRECTORY);
+        if (data.exists()) {
+            FileUtils.delete(data);
+        }
+        data = Settings.getFile(Settings.KEYS.CPE_DATA_DIRECTORY);
+        if (data.exists()) {
+            FileUtils.delete(data);
+        }
+        data = DataStoreMetaInfo.getPropertiesFile();
+        if (data.exists()) {
+            FileUtils.delete(data);
+        }
+    }
+
+    /**
+     * Closes the CVE and CPE data stores.
+     */
+    protected void closeDataStores() {
+        if (cveDB != null) {
+            try {
+                cveDB.close();
+            } catch (Exception ignore) {
+                Logger.getLogger(AbstractUpdateTask.class.getName()).log(Level.FINEST, "Error closing the cveDB", ignore);
+            }
+        }
+        if (cpeIndex != null) {
+            try {
+                cpeIndex.close();
+            } catch (Exception ignore) {
+                Logger.getLogger(AbstractUpdateTask.class.getName()).log(Level.FINEST, "Error closing the cpeIndex", ignore);
+            }
+        }
+    }
+
+    /**
+     * Opens the CVE and CPE data stores.
+     *
+     * @throws UpdateException thrown if a data store cannot be opened
+     */
+    protected void openDataStores() throws UpdateException {
+        //open the cve and cpe data stores
+        try {
+            cveDB = new CveDB();
+            cveDB.open();
+            cpeIndex = new CpeIndexWriter();
+            cpeIndex.open();
+        } catch (IOException ex) {
+            closeDataStores();
+            Logger.getLogger(AbstractUpdateTask.class.getName()).log(Level.FINE, "IO Error opening databases", ex);
+            throw new UpdateException("Error updating the CPE/CVE data, please see the log file for more details.");
+        } catch (SQLException ex) {
+            closeDataStores();
+            Logger.getLogger(AbstractUpdateTask.class.getName()).log(Level.FINE, "SQL Exception opening databases", ex);
+            throw new UpdateException("Error updating the CPE/CVE data, please see the log file for more details.");
+        } catch (DatabaseException ex) {
+            closeDataStores();
+            Logger.getLogger(AbstractUpdateTask.class.getName()).log(Level.FINE, "Database Exception opening databases", ex);
+            throw new UpdateException("Error updating the CPE/CVE data, please see the log file for more details.");
+        } catch (ClassNotFoundException ex) {
+            closeDataStores();
+            Logger.getLogger(AbstractUpdateTask.class.getName()).log(Level.FINE, "Class not found exception opening databases", ex);
+            throw new UpdateException("Error updating the CPE/CVE data, please see the log file for more details.");
+        }
+    }
+
+    /**
+     * Determines if the epoch date is within the range specified of the
+     * compareTo epoch time. This takes the (compareTo-date)/1000/60/60/24 to
+     * get the number of days. If the calculated days is less then the range the
+     * date is considered valid.
+     *
+     * @param date the date to be checked.
+     * @param compareTo the date to compare to.
+     * @param range the range in days to be considered valid.
+     * @return whether or not the date is within the range.
+     */
+    protected boolean withinRange(long date, long compareTo, int range) {
+        final double differenceInDays = (compareTo - date) / 1000.0 / 60.0 / 60.0 / 24.0;
+        return differenceInDays < range;
+    }
+
+    /**
+     * Imports the NVD CVE XML File into the Lucene Index.
+     *
+     * @param file the file containing the NVD CVE XML
+     * @param oldVersion contains the file containing the NVD CVE XML 1.2
+     * @throws ParserConfigurationException is thrown if there is a parser
+     * configuration exception
+     * @throws SAXException is thrown if there is a SAXException
+     * @throws IOException is thrown if there is a IO Exception
+     * @throws SQLException is thrown if there is a SQL exception
+     * @throws DatabaseException is thrown if there is a database exception
+     * @throws ClassNotFoundException thrown if the h2 database driver cannot be
+     * loaded
+     */
+    protected void importXML(File file, File oldVersion)
+            throws ParserConfigurationException, SAXException, IOException, SQLException, DatabaseException, ClassNotFoundException {
+
+        final SAXParserFactory factory = SAXParserFactory.newInstance();
+        final SAXParser saxParser = factory.newSAXParser();
+
+        final NvdCve12Handler cve12Handler = new NvdCve12Handler();
+        saxParser.parse(oldVersion, cve12Handler);
+        final Map<String, List<VulnerableSoftware>> prevVersionVulnMap = cve12Handler.getVulnerabilities();
+
+        final NvdCve20Handler cve20Handler = new NvdCve20Handler();
+        cve20Handler.setCveDB(cveDB);
+        cve20Handler.setPrevVersionVulnMap(prevVersionVulnMap);
+        cve20Handler.setCpeIndex(cpeIndex);
+        saxParser.parse(file, cve20Handler);
+    }
+}

dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/BatchUpdateTask.java

@@ -0,0 +1,270 @@
+/*
+ * This file is part of dependency-check-core.
+ *
+ * Dependency-check-core is free software: you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation, either version 3 of the License, or (at your option) any
+ * later version.
+ *
+ * Dependency-check-core is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+ * details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * dependency-check-core. If not, see http://www.gnu.org/licenses/.
+ *
+ * Copyright (c) 2012 Jeremy Long. All Rights Reserved.
+ */
+package org.owasp.dependencycheck.data.update;
+
+import org.owasp.dependencycheck.data.nvdcve.InvalidDataException;
+import java.io.File;
+import java.io.IOException;
+import java.net.MalformedURLException;
+import java.net.URISyntaxException;
+import java.net.URL;
+import java.util.Calendar;
+import java.util.Date;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+import org.owasp.dependencycheck.data.UpdateException;
+import org.owasp.dependencycheck.data.nvdcve.CveDB;
+import org.owasp.dependencycheck.utils.DownloadFailedException;
+import org.owasp.dependencycheck.utils.Downloader;
+import org.owasp.dependencycheck.utils.FileUtils;
+import org.owasp.dependencycheck.utils.Settings;
+import org.owasp.dependencycheck.utils.InvalidSettingException;
+import static org.owasp.dependencycheck.data.update.DataStoreMetaInfo.BATCH;
+import static org.owasp.dependencycheck.data.update.DataStoreMetaInfo.MODIFIED;
+
+/**
+ * Class responsible for updating the CPE and NVDCVE data stores.
+ *
+ * @author Jeremy Long (jeremy.long@owasp.org)
+ */
+public class BatchUpdateTask extends AbstractUpdateTask {
+
+    /**
+     * Constructs a new BatchUpdateTask.
+     *
+     * @param properties information about the data store
+     * @throws MalformedURLException thrown if a configured URL is malformed
+     * @throws DownloadFailedException thrown if a timestamp cannot be checked
+     * on a configured URL
+     * @throws UpdateException thrown if there is an exception generating the
+     * update task
+     */
+    public BatchUpdateTask(DataStoreMetaInfo properties) throws MalformedURLException, DownloadFailedException, UpdateException {
+        super(properties);
+    }
+    /**
+     * A flag indicating whether or not the batch update should be performed.
+     */
+    private boolean doBatchUpdate;
+
+    /**
+     * Get the value of doBatchUpdate
+     *
+     * @return the value of doBatchUpdate
+     */
+    protected boolean isDoBatchUpdate() {
+        return doBatchUpdate;
+    }
+
+    /**
+     * Set the value of doBatchUpdate
+     *
+     * @param doBatchUpdate new value of doBatchUpdate
+     */
+    protected void setDoBatchUpdate(boolean doBatchUpdate) {
+        this.doBatchUpdate = doBatchUpdate;
+    }
+
+    /**
+     * <p>Downloads the latest NVD CVE XML file from the web and imports it into
+     * the current CVE Database.</p>
+     *
+     * @throws UpdateException is thrown if there is an error updating the
+     * database
+     */
+    @Override
+    public void update() throws UpdateException {
+        if (getProperties().isBatchUpdateMode() && doBatchUpdate) {
+            final String batchSrc = Settings.getString(Settings.KEYS.BATCH_UPDATE_URL);
+            File tmp = null;
+            try {
+                deleteExistingData();
+                final File dataDirectory = CveDB.getDataDirectory().getParentFile();
+                final URL batchUrl = new URL(batchSrc);
+                if ("file".equals(batchUrl.getProtocol())) {
+                    try {
+                        tmp = new File(batchUrl.toURI());
+                    } catch (URISyntaxException ex) {
+                        final String msg = String.format("Invalid batch update URI: %s", batchSrc);
+                        throw new UpdateException(msg, ex);
+                    }
+                } else if ("http".equals(batchUrl.getProtocol())
+                        || "https".equals(batchUrl.getProtocol())) {
+                    tmp = File.createTempFile("batch_", ".zip");
+                    Downloader.fetchFile(batchUrl, tmp);
+                }
+                //TODO add FTP?
+                FileUtils.extractFiles(tmp, dataDirectory);
+
+            } catch (IOException ex) {
+                final String msg = String.format("IO Exception Occured performing batch update using: %s", batchSrc);
+                throw new UpdateException(msg, ex);
+            } finally {
+                if (tmp != null && !tmp.delete()) {
+                    tmp.deleteOnExit();
+                }
+            }
+        }
+    }
+
+    /**
+     * Determines if the index needs to be updated. This is done by fetching the
+     * NVD CVE meta data and checking the last update date. If the data needs to
+     * be refreshed this method will return the NvdCveUrl for the files that
+     * need to be updated.
+     *
+     * @return the collection of files that need to be updated
+     * @throws MalformedURLException is thrown if the URL for the NVD CVE Meta
+     * data is incorrect
+     * @throws DownloadFailedException is thrown if there is an error.
+     * downloading the NVD CVE download data file
+     * @throws UpdateException Is thrown if there is an issue with the last
+     * updated properties file
+     */
+    @Override
+    public Updateable updatesNeeded() throws MalformedURLException, DownloadFailedException, UpdateException {
+        Updateable updates = null;
+        try {
+            updates = retrieveCurrentTimestampsFromWeb();
+        } catch (InvalidDataException ex) {
+            final String msg = "Unable to retrieve valid timestamp from nvd cve downloads page";
+            Logger.getLogger(BatchUpdateTask.class.getName()).log(Level.FINE, msg, ex);
+            throw new DownloadFailedException(msg, ex);
+        } catch (InvalidSettingException ex) {
+            Logger.getLogger(BatchUpdateTask.class.getName()).log(Level.FINE, "Invalid setting found when retrieving timestamps", ex);
+            throw new DownloadFailedException("Invalid settings", ex);
+        }
+
+        if (updates == null) {
+            throw new DownloadFailedException("Unable to retrieve the timestamps of the currently published NVD CVE data");
+        }
+        final DataStoreMetaInfo properties = getProperties();
+        if (!properties.isEmpty()) {
+            try {
+                boolean deleteAndRecreate = false;
+                float version;
+
+                if (properties.getProperty("version") == null) {
+                    deleteAndRecreate = true;
+                } else {
+                    try {
+                        version = Float.parseFloat(properties.getProperty("version"));
+                        final float currentVersion = Float.parseFloat(CveDB.DB_SCHEMA_VERSION);
+                        if (currentVersion > version) {
+                            deleteAndRecreate = true;
+                        }
+                    } catch (NumberFormatException ex) {
+                        deleteAndRecreate = true;
+                    }
+                }
+
+                final NvdCveInfo batchInfo = updates.get(BATCH);
+                if (properties.isBatchUpdateMode() && batchInfo != null) {
+                    final long lastUpdated = Long.parseLong(properties.getProperty(DataStoreMetaInfo.BATCH, "0"));
+                    if (lastUpdated != batchInfo.getTimestamp()) {
+                        deleteAndRecreate = true;
+                    }
+                }
+
+                if (deleteAndRecreate) {
+                    setDoBatchUpdate(properties.isBatchUpdateMode());
+                    try {
+                        deleteExistingData();
+                    } catch (IOException ex) {
+                        final String msg = "Unable to delete existing data";
+                        Logger.getLogger(BatchUpdateTask.class.getName()).log(Level.WARNING, msg);
+                        Logger.getLogger(BatchUpdateTask.class.getName()).log(Level.FINE, null, ex);
+                    }
+                    return updates;
+                }
+
+                final long lastUpdated = Long.parseLong(properties.getProperty(DataStoreMetaInfo.LAST_UPDATED, "0"));
+                final Date now = new Date();
+                final int days = Settings.getInt(Settings.KEYS.CVE_MODIFIED_VALID_FOR_DAYS, 7);
+                final int start = Settings.getInt(Settings.KEYS.CVE_START_YEAR, 2002);
+                final int end = Calendar.getInstance().get(Calendar.YEAR);
+                if (lastUpdated == updates.get(MODIFIED).getTimestamp()) {
+                    updates.clear(); //we don't need to update anything.
+                    setDoBatchUpdate(properties.isBatchUpdateMode());
+                } else if (withinRange(lastUpdated, now.getTime(), days)) {
+                    updates.get(MODIFIED).setNeedsUpdate(true);
+                    if (properties.isBatchUpdateMode()) {
+                        setDoBatchUpdate(false);
+                    } else {
+                        for (int i = start; i <= end; i++) {
+                            updates.get(String.valueOf(i)).setNeedsUpdate(false);
+                        }
+                    }
+                } else if (properties.isBatchUpdateMode()) {
+                    updates.get(MODIFIED).setNeedsUpdate(true);
+                    setDoBatchUpdate(true);
+                } else { //we figure out which of the several XML files need to be downloaded.
+                    updates.get(MODIFIED).setNeedsUpdate(false);
+                    for (int i = start; i <= end; i++) {
+                        final NvdCveInfo cve = updates.get(String.valueOf(i));
+                        long currentTimestamp = 0;
+                        try {
+                            currentTimestamp = Long.parseLong(properties.getProperty(DataStoreMetaInfo.LAST_UPDATED_BASE + String.valueOf(i), "0"));
+                        } catch (NumberFormatException ex) {
+                            final String msg = String.format("Error parsing '%s' '%s' from nvdcve.lastupdated",
+                                    DataStoreMetaInfo.LAST_UPDATED_BASE, String.valueOf(i));
+                            Logger.getLogger(BatchUpdateTask.class.getName()).log(Level.FINE, msg, ex);
+                        }
+                        if (currentTimestamp == cve.getTimestamp()) {
+                            cve.setNeedsUpdate(false); //they default to true.
+                        }
+                    }
+                }
+            } catch (NumberFormatException ex) {
+                final String msg = "An invalid schema version or timestamp exists in the data.properties file.";
+                Logger.getLogger(BatchUpdateTask.class.getName()).log(Level.WARNING, msg);
+                Logger.getLogger(BatchUpdateTask.class.getName()).log(Level.FINE, null, ex);
+                setDoBatchUpdate(properties.isBatchUpdateMode());
+            }
+        }
+        return updates;
+    }
+
+    /**
+     * Retrieves the timestamps from the NVD CVE meta data file.
+     *
+     * @return the timestamp from the currently published nvdcve downloads page
+     * @throws MalformedURLException thrown if the URL for the NVD CCE Meta data
+     * is incorrect.
+     * @throws DownloadFailedException thrown if there is an error downloading
+     * the nvd cve meta data file
+     * @throws InvalidDataException thrown if there is an exception parsing the
+     * timestamps
+     * @throws InvalidSettingException thrown if the settings are invalid
+     */
+    private Updateable retrieveCurrentTimestampsFromWeb()
+            throws MalformedURLException, DownloadFailedException, InvalidDataException, InvalidSettingException {
+        final Updateable updates = new Updateable();
+        updates.add(BATCH, Settings.getString(Settings.KEYS.BATCH_UPDATE_URL),
+                null, false);
+
+        final String url = Settings.getString(Settings.KEYS.CVE_MODIFIED_20_URL, "");
+        if (!url.isEmpty()) {
+            updates.add(MODIFIED, url,
+                    Settings.getString(Settings.KEYS.CVE_MODIFIED_12_URL),
+                    false);
+        }
+        return updates;
+    }
+}

dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/DataStoreMetaInfo.java

@@ -0,0 +1,241 @@
+/*
+ * This file is part of dependency-check-core.
+ *
+ * Dependency-check-core is free software: you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation, either version 3 of the License, or (at your option) any
+ * later version.
+ *
+ * Dependency-check-core is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+ * details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * dependency-check-core. If not, see http://www.gnu.org/licenses/.
+ *
+ * Copyright (c) 2013 Jeremy Long. All Rights Reserved.
+ */
+package org.owasp.dependencycheck.data.update;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.io.OutputStreamWriter;
+import java.util.Properties;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+import org.owasp.dependencycheck.data.UpdateException;
+import org.owasp.dependencycheck.data.nvdcve.CveDB;
+import org.owasp.dependencycheck.utils.Settings;
+
+/**
+ *
+ * @author Jeremy Long (jeremy.long@owasp.org)
+ */
+public class DataStoreMetaInfo {
+
+    /**
+     * Batch key word, used as key to store information about batch mode.
+     */
+    public static final String BATCH = "batch";
+    /**
+     * Modified key word, used as a key to store information about the modified
+     * file (i.e. the containing the last 8 days of updates)..
+     */
+    public static final String MODIFIED = "modified";
+    /**
+     * The name of the properties file containing the timestamp of the last
+     * update.
+     */
+    private static final String UPDATE_PROPERTIES_FILE = "data.properties";
+    /**
+     * The properties file key for the last updated field - used to store the
+     * last updated time of the Modified NVD CVE xml file.
+     */
+    public static final String LAST_UPDATED = "lastupdated.modified";
+    /**
+     * Stores the last updated time for each of the NVD CVE files. These
+     * timestamps should be updated if we process the modified file within 7
+     * days of the last update.
+     */
+    public static final String LAST_UPDATED_BASE = "lastupdated.";
+    /**
+     * A collection of properties about the data.
+     */
+    private Properties properties = new Properties();
+    /**
+     * Indicates whether or not the updates are using a batch update mode or
+     * not.
+     */
+    private boolean batchUpdateMode;
+
+    /**
+     * Get the value of batchUpdateMode.
+     *
+     * @return the value of batchUpdateMode
+     */
+    protected boolean isBatchUpdateMode() {
+        return batchUpdateMode;
+    }
+
+    /**
+     * Set the value of batchUpdateMode.
+     *
+     * @param batchUpdateMode new value of batchUpdateMode
+     */
+    protected void setBatchUpdateMode(boolean batchUpdateMode) {
+        this.batchUpdateMode = batchUpdateMode;
+    }
+
+    /**
+     * Constructs a new data properties object.
+     */
+    public DataStoreMetaInfo() {
+        batchUpdateMode = !Settings.getString(Settings.KEYS.BATCH_UPDATE_URL, "").isEmpty();
+        loadProperties();
+    }
+
+    /**
+     * Loads the data's meta properties.
+     */
+    private void loadProperties() {
+        final File file = getPropertiesFile();
+        if (file.exists()) {
+            InputStream is = null;
+            try {
+                is = new FileInputStream(file);
+            } catch (FileNotFoundException ignore) {
+                //we will never get here as we check for existence above.
+                Logger.getLogger(DataStoreMetaInfo.class.getName()).log(Level.FINEST, null, ignore);
+            }
+            try {
+                properties.load(is);
+            } catch (IOException ex) {
+                final String msg = String.format("Unable to load properties file '%s'", file.getPath());
+                Logger.getLogger(DataStoreMetaInfo.class.getName()).log(Level.WARNING, msg);
+                Logger.getLogger(DataStoreMetaInfo.class.getName()).log(Level.FINE, null, ex);
+            } finally {
+                if (is != null) {
+                    try {
+                        is.close();
+                    } catch (IOException ex) {
+                        final String msg = String.format("Unable to close properties file '%s'", file.getPath());
+                        Logger.getLogger(DataStoreMetaInfo.class.getName()).log(Level.WARNING, msg);
+                        Logger.getLogger(DataStoreMetaInfo.class.getName()).log(Level.FINE, null, ex);
+                    }
+                }
+            }
+        }
+    }
+
+    /**
+     * Returns whether or not any properties are set.
+     *
+     * @return whether or not any properties are set
+     */
+    public boolean isEmpty() {
+        return properties.isEmpty();
+    }
+
+    /**
+     * Writes a properties file containing the last updated date to the
+     * VULNERABLE_CPE directory.
+     *
+     * @param updatedValue the updated nvdcve entry
+     * @throws UpdateException is thrown if there is an update exception
+     */
+    public void save(NvdCveInfo updatedValue) throws UpdateException {
+        if (updatedValue == null) {
+            return;
+        }
+        final File cveProp = getPropertiesFile();
+        final Properties prop = new Properties();
+        if (cveProp.exists()) {
+            FileInputStream in = null;
+            try {
+                in = new FileInputStream(cveProp);
+                prop.load(in);
+            } catch (Exception ignoreMe) {
+                Logger.getLogger(DataStoreMetaInfo.class.getName()).log(Level.FINEST, null, ignoreMe);
+            } finally {
+                if (in != null) {
+                    try {
+                        in.close();
+                    } catch (Exception ignoreMeToo) {
+                        Logger.getLogger(DataStoreMetaInfo.class.getName()).log(Level.FINEST, null, ignoreMeToo);
+                    }
+                }
+            }
+        }
+        prop.put("version", CveDB.DB_SCHEMA_VERSION);
+        prop.put(LAST_UPDATED_BASE + updatedValue.getId(), String.valueOf(updatedValue.getTimestamp()));
+
+        OutputStream os = null;
+        OutputStreamWriter out = null;
+        try {
+            os = new FileOutputStream(cveProp);
+            out = new OutputStreamWriter(os, "UTF-8");
+            prop.store(out, "Meta data about data and data sources used by dependency-check");
+        } catch (FileNotFoundException ex) {
+            Logger.getLogger(DataStoreMetaInfo.class.getName()).log(Level.FINE, null, ex);
+            throw new UpdateException("Unable to find last updated properties file.", ex);
+        } catch (IOException ex) {
+            Logger.getLogger(DataStoreMetaInfo.class.getName()).log(Level.FINE, null, ex);
+            throw new UpdateException("Unable to update last updated properties file.", ex);
+        } finally {
+            if (out != null) {
+                try {
+                    out.close();
+                } catch (IOException ex) {
+                    Logger.getLogger(DataStoreMetaInfo.class.getName()).log(Level.FINEST, null, ex);
+                }
+            }
+            if (os != null) {
+                try {
+                    os.close();
+                } catch (IOException ex) {
+                    Logger.getLogger(DataStoreMetaInfo.class.getName()).log(Level.FINEST, null, ex);
+                }
+            }
+        }
+    }
+
+    /**
+     * Returns the property value for the given key. If the key is not contained
+     * in the underlying properties null is returned.
+     *
+     * @param key the property key
+     * @return the value of the property
+     */
+    public String getProperty(String key) {
+        return properties.getProperty(key);
+    }
+
+    /**
+     * Returns the property value for the given key. If the key is not contained
+     * in the underlying properties the default value is returned.
+     *
+     * @param key the property key
+     * @param defaultValue the default value
+     * @return the value of the property
+     */
+    public String getProperty(String key, String defaultValue) {
+        return properties.getProperty(key, defaultValue);
+    }
+
+    /**
+     * Retrieves the properties file.
+     *
+     * @return the properties file
+     */
+    public static File getPropertiesFile() {
+        final File dataDirectory = Settings.getFile(Settings.KEYS.DATA_DIRECTORY);
+        final File file = new File(dataDirectory, UPDATE_PROPERTIES_FILE);
+        return file;
+    }
+}

dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/DatabaseUpdater.java

@@ -0,0 +1,117 @@
+/*
+ * This file is part of dependency-check-core.
+ *
+ * Dependency-check-core is free software: you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation, either version 3 of the License, or (at your option) any
+ * later version.
+ *
+ * Dependency-check-core is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+ * details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * dependency-check-core. If not, see http://www.gnu.org/licenses/.
+ *
+ * Copyright (c) 2012 Jeremy Long. All Rights Reserved.
+ */
+package org.owasp.dependencycheck.data.update;
+
+import java.io.File;
+import java.io.IOException;
+import org.owasp.dependencycheck.data.CachedWebDataSource;
+import java.net.MalformedURLException;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+import org.owasp.dependencycheck.concurrency.DirectoryLockException;
+import org.owasp.dependencycheck.concurrency.DirectorySpinLock;
+import org.owasp.dependencycheck.concurrency.InvalidDirectoryException;
+import org.owasp.dependencycheck.data.UpdateException;
+import org.owasp.dependencycheck.utils.DownloadFailedException;
+import org.owasp.dependencycheck.utils.FileUtils;
+import org.owasp.dependencycheck.utils.Settings;
+
+/**
+ * Class responsible for updating the CPE and NVDCVE data stores.
+ *
+ * @author Jeremy Long (jeremy.long@owasp.org)
+ */
+public class DatabaseUpdater implements CachedWebDataSource {
+
+    /**
+     * <p>Downloads the latest NVD CVE XML file from the web and imports it into
+     * the current CVE Database.</p>
+     *
+     * @throws UpdateException is thrown if there is an error updating the
+     * database
+     */
+    @Override
+    public void update() throws UpdateException {
+        final File dataDir = Settings.getFile(Settings.KEYS.DATA_DIRECTORY);
+        DirectorySpinLock lock = null;
+        try {
+            lock = new DirectorySpinLock(dataDir);
+        } catch (InvalidDirectoryException ex) {
+            throw new UpdateException("Unable to obtain lock on the data directory", ex);
+        } catch (DirectoryLockException ex) {
+            throw new UpdateException("Unable to obtain exclusive lock on the data directory", ex);
+        }
+
+        try {
+            lock.obtainSharedLock();
+            final UpdateTask task = UpdateTaskFactory.getUpdateTask();
+
+
+            if (task.isUpdateNeeded()) {
+                lock.release();
+                lock.obtainExclusiveLock();
+                if (task.shouldDeleteAndRecreate()) {
+                    try {
+                        deleteExistingData();
+                    } catch (IOException ex) {
+                        Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.WARNING, "Unable to delete the existing data directory");
+                        Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.FINE, null, ex);
+                    }
+                }
+                task.update();
+            }
+        } catch (DirectoryLockException ex) {
+            Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.WARNING,
+                    "Unable to obtain lock on data directory, unable to update the data to use the most current data.");
+            Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.FINE, null, ex);
+        } catch (MalformedURLException ex) {
+            Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.WARNING,
+                    "NVD CVE properties files contain an invalid URL, unable to update the data to use the most current data.");
+            Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.FINE, null, ex);
+        } catch (DownloadFailedException ex) {
+            Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.WARNING,
+                    "Unable to download the NVD CVE data, unable to update the data to use the most current data.");
+            Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.FINE, null, ex);
+        } finally {
+            if (lock != null) {
+                lock.release();
+            }
+        }
+    }
+
+    /**
+     * Deletes the existing data directories.
+     *
+     * @throws IOException thrown if the directory cannot be deleted
+     */
+    protected void deleteExistingData() throws IOException {
+        File data = Settings.getFile(Settings.KEYS.CVE_DATA_DIRECTORY);
+        if (data.exists()) {
+            FileUtils.delete(data);
+        }
+        data = Settings.getFile(Settings.KEYS.CPE_DATA_DIRECTORY);
+        if (data.exists()) {
+            FileUtils.delete(data);
+        }
+        data = DataStoreMetaInfo.getPropertiesFile();
+        if (data.exists()) {
+            FileUtils.delete(data);
+        }
+    }
+}

dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/NvdCveInfo.java

@@ -0,0 +1,138 @@
+/*
+ * This file is part of dependency-check-core.
+ *
+ * Dependency-check-core is free software: you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation, either version 3 of the License, or (at your option) any
+ * later version.
+ *
+ * Dependency-check-core is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+ * details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * dependency-check-core. If not, see http://www.gnu.org/licenses/.
+ *
+ * Copyright (c) 2013 Jeremy Long. All Rights Reserved.
+ */
+package org.owasp.dependencycheck.data.update;
+
+/**
+ * A pojo that contains the Url and timestamp of the current NvdCve XML files.
+ *
+ * @author Jeremy Long (jeremy.long@owasp.org)
+ */
+public class NvdCveInfo {
+
+    /**
+     * an id.
+     */
+    private String id;
+
+    /**
+     * Get the value of id.
+     *
+     * @return the value of id
+     */
+    public String getId() {
+        return id;
+    }
+
+    /**
+     * Set the value of id.
+     *
+     * @param id new value of id
+     */
+    public void setId(String id) {
+        this.id = id;
+    }
+    /**
+     * a url.
+     */
+    private String url;
+
+    /**
+     * Get the value of url.
+     *
+     * @return the value of url
+     */
+    public String getUrl() {
+        return url;
+    }
+
+    /**
+     * Set the value of url.
+     *
+     * @param url new value of url
+     */
+    public void setUrl(String url) {
+        this.url = url;
+    }
+    /**
+     * The 1.2 schema URL.
+     */
+    private String oldSchemaVersionUrl;
+
+    /**
+     * Get the value of oldSchemaVersionUrl.
+     *
+     * @return the value of oldSchemaVersionUrl
+     */
+    public String getOldSchemaVersionUrl() {
+        return oldSchemaVersionUrl;
+    }
+
+    /**
+     * Set the value of oldSchemaVersionUrl.
+     *
+     * @param oldSchemaVersionUrl new value of oldSchemaVersionUrl
+     */
+    public void setOldSchemaVersionUrl(String oldSchemaVersionUrl) {
+        this.oldSchemaVersionUrl = oldSchemaVersionUrl;
+    }
+    /**
+     * a timestamp - epoch time.
+     */
+    private long timestamp;
+
+    /**
+     * Get the value of timestamp - epoch time.
+     *
+     * @return the value of timestamp - epoch time
+     */
+    public long getTimestamp() {
+        return timestamp;
+    }
+
+    /**
+     * Set the value of timestamp - epoch time.
+     *
+     * @param timestamp new value of timestamp - epoch time
+     */
+    public void setTimestamp(long timestamp) {
+        this.timestamp = timestamp;
+    }
+    /**
+     * indicates whether or not this item should be updated.
+     */
+    private boolean needsUpdate = true;
+
+    /**
+     * Get the value of needsUpdate.
+     *
+     * @return the value of needsUpdate
+     */
+    public boolean getNeedsUpdate() {
+        return needsUpdate;
+    }
+
+    /**
+     * Set the value of needsUpdate.
+     *
+     * @param needsUpdate new value of needsUpdate
+     */
+    public void setNeedsUpdate(boolean needsUpdate) {
+        this.needsUpdate = needsUpdate;
+    }
+}

dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/StandardUpdateTask.java

@@ -0,0 +1,294 @@
+/*
+ * This file is part of dependency-check-core.
+ *
+ * Dependency-check-core is free software: you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation, either version 3 of the License, or (at your option) any
+ * later version.
+ *
+ * Dependency-check-core is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+ * details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * dependency-check-core. If not, see http://www.gnu.org/licenses/.
+ *
+ * Copyright (c) 2012 Jeremy Long. All Rights Reserved.
+ */
+package org.owasp.dependencycheck.data.update;
+
+import org.owasp.dependencycheck.data.nvdcve.InvalidDataException;
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import javax.xml.parsers.ParserConfigurationException;
+import org.xml.sax.SAXException;
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.sql.SQLException;
+import java.util.Calendar;
+import java.util.Date;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+import org.owasp.dependencycheck.data.UpdateException;
+import org.owasp.dependencycheck.data.nvdcve.CveDB;
+import org.owasp.dependencycheck.utils.DownloadFailedException;
+import org.owasp.dependencycheck.utils.Downloader;
+import org.owasp.dependencycheck.utils.Settings;
+import org.owasp.dependencycheck.data.nvdcve.DatabaseException;
+import org.owasp.dependencycheck.utils.InvalidSettingException;
+import static org.owasp.dependencycheck.data.update.DataStoreMetaInfo.MODIFIED;
+
+/**
+ * Class responsible for updating the CPE and NVDCVE data stores.
+ *
+ * @author Jeremy Long (jeremy.long@owasp.org)
+ */
+public class StandardUpdateTask extends AbstractUpdateTask {
+
+    /**
+     * Constructs a new Standard Update Task.
+     *
+     * @param properties information about the data store
+     * @throws MalformedURLException thrown if a configured URL is malformed
+     * @throws DownloadFailedException thrown if a timestamp cannot be checked
+     * on a configured URL
+     * @throws UpdateException thrown if there is an exception generating the
+     * update task
+     */
+    public StandardUpdateTask(DataStoreMetaInfo properties) throws MalformedURLException, DownloadFailedException, UpdateException {
+        super(properties);
+    }
+
+    /**
+     * <p>Downloads the latest NVD CVE XML file from the web and imports it into
+     * the current CVE Database.</p>
+     *
+     * @throws UpdateException is thrown if there is an error updating the
+     * database
+     */
+    @Override
+    public void update() throws UpdateException {
+        try {
+            int maxUpdates = 0;
+            for (NvdCveInfo cve : getUpdateable()) {
+                if (cve.getNeedsUpdate()) {
+                    maxUpdates += 1;
+                }
+            }
+            if (maxUpdates > 3) {
+                Logger.getLogger(StandardUpdateTask.class.getName()).log(Level.INFO,
+                        "NVD CVE requires several updates; this could take a couple of minutes.");
+            }
+            if (maxUpdates > 0) {
+                openDataStores();
+            }
+
+            int count = 0;
+            for (NvdCveInfo cve : getUpdateable()) {
+                if (cve.getNeedsUpdate()) {
+                    count += 1;
+                    Logger.getLogger(StandardUpdateTask.class.getName()).log(Level.INFO,
+                            "Updating NVD CVE ({0} of {1})", new Object[]{count, maxUpdates});
+                    URL url = new URL(cve.getUrl());
+                    File outputPath = null;
+                    File outputPath12 = null;
+                    try {
+                        Logger.getLogger(StandardUpdateTask.class.getName()).log(Level.INFO,
+                                "Downloading {0}", cve.getUrl());
+                        outputPath = File.createTempFile("cve" + cve.getId() + "_", ".xml");
+                        Downloader.fetchFile(url, outputPath);
+
+                        url = new URL(cve.getOldSchemaVersionUrl());
+                        outputPath12 = File.createTempFile("cve_1_2_" + cve.getId() + "_", ".xml");
+                        Downloader.fetchFile(url, outputPath12);
+
+                        Logger.getLogger(StandardUpdateTask.class.getName()).log(Level.INFO,
+                                "Processing {0}", cve.getUrl());
+
+                        importXML(outputPath, outputPath12);
+
+                        getCveDB().commit();
+                        getCpeIndex().commit();
+                        getProperties().save(cve);
+
+                        Logger.getLogger(StandardUpdateTask.class.getName()).log(Level.INFO,
+                                "Completed update {0} of {1}", new Object[]{count, maxUpdates});
+                    } catch (FileNotFoundException ex) {
+                        throw new UpdateException(ex);
+                    } catch (ParserConfigurationException ex) {
+                        throw new UpdateException(ex);
+                    } catch (SAXException ex) {
+                        throw new UpdateException(ex);
+                    } catch (IOException ex) {
+                        throw new UpdateException(ex);
+                    } catch (SQLException ex) {
+                        throw new UpdateException(ex);
+                    } catch (DatabaseException ex) {
+                        throw new UpdateException(ex);
+                    } catch (ClassNotFoundException ex) {
+                        throw new UpdateException(ex);
+                    } finally {
+                        boolean deleted = false;
+                        try {
+                            if (outputPath != null && outputPath.exists()) {
+                                deleted = outputPath.delete();
+                            }
+                        } finally {
+                            if (outputPath != null && (outputPath.exists() || !deleted)) {
+                                outputPath.deleteOnExit();
+                            }
+                        }
+                        try {
+                            deleted = false;
+                            if (outputPath12 != null && outputPath12.exists()) {
+                                deleted = outputPath12.delete();
+                            }
+                        } finally {
+                            if (outputPath12 != null && (outputPath12.exists() || !deleted)) {
+                                outputPath12.deleteOnExit();
+                            }
+                        }
+                    }
+                }
+            }
+            if (maxUpdates >= 1) { //ensure the modified file date gets written
+                getProperties().save(getUpdateable().get(MODIFIED));
+                getCveDB().cleanupDatabase();
+            }
+        } catch (MalformedURLException ex) {
+            throw new UpdateException(ex);
+        } finally {
+            closeDataStores();
+        }
+    }
+
+    /**
+     * Determines if the index needs to be updated. This is done by fetching the
+     * NVD CVE meta data and checking the last update date. If the data needs to
+     * be refreshed this method will return the NvdCveUrl for the files that
+     * need to be updated.
+     *
+     * @return the collection of files that need to be updated
+     * @throws MalformedURLException is thrown if the URL for the NVD CVE Meta
+     * data is incorrect
+     * @throws DownloadFailedException is thrown if there is an error.
+     * downloading the NVD CVE download data file
+     * @throws UpdateException Is thrown if there is an issue with the last
+     * updated properties file
+     */
+    @Override
+    protected Updateable updatesNeeded() throws MalformedURLException, DownloadFailedException, UpdateException {
+        Updateable updates = null;
+        try {
+            updates = retrieveCurrentTimestampsFromWeb();
+        } catch (InvalidDataException ex) {
+            final String msg = "Unable to retrieve valid timestamp from nvd cve downloads page";
+            Logger.getLogger(StandardUpdateTask.class.getName()).log(Level.FINE, msg, ex);
+            throw new DownloadFailedException(msg, ex);
+        } catch (InvalidSettingException ex) {
+            Logger.getLogger(StandardUpdateTask.class.getName()).log(Level.FINE, "Invalid setting found when retrieving timestamps", ex);
+            throw new DownloadFailedException("Invalid settings", ex);
+        }
+
+        if (updates == null) {
+            throw new DownloadFailedException("Unable to retrieve the timestamps of the currently published NVD CVE data");
+        }
+        final DataStoreMetaInfo properties = getProperties();
+        if (!properties.isEmpty()) {
+            try {
+                float version;
+
+                if (properties.getProperty("version") == null) {
+                    setDeleteAndRecreate(true);
+                } else {
+                    try {
+                        version = Float.parseFloat(properties.getProperty("version"));
+                        final float currentVersion = Float.parseFloat(CveDB.DB_SCHEMA_VERSION);
+                        if (currentVersion > version) {
+                            setDeleteAndRecreate(true);
+                        }
+                    } catch (NumberFormatException ex) {
+                        setDeleteAndRecreate(true);
+                    }
+                }
+
+                if (shouldDeleteAndRecreate()) {
+                    return updates;
+                }
+
+                final long lastUpdated = Long.parseLong(properties.getProperty(DataStoreMetaInfo.LAST_UPDATED, "0"));
+                final Date now = new Date();
+                final int days = Settings.getInt(Settings.KEYS.CVE_MODIFIED_VALID_FOR_DAYS, 7);
+                if (lastUpdated == updates.getTimeStamp(MODIFIED)) {
+                    updates.clear(); //we don't need to update anything.
+                } else if (withinRange(lastUpdated, now.getTime(), days)) {
+                    for (NvdCveInfo entry : updates) {
+                        if (MODIFIED.equals(entry.getId())) {
+                            entry.setNeedsUpdate(true);
+                        } else {
+                            entry.setNeedsUpdate(false);
+                        }
+                    }
+                } else { //we figure out which of the several XML files need to be downloaded.
+                    for (NvdCveInfo entry : updates) {
+                        if (MODIFIED.equals(entry.getId())) {
+                            entry.setNeedsUpdate(true);
+                        } else {
+                            long currentTimestamp = 0;
+                            try {
+                                currentTimestamp = Long.parseLong(properties.getProperty(DataStoreMetaInfo.LAST_UPDATED_BASE + entry.getId(), "0"));
+                            } catch (NumberFormatException ex) {
+                                final String msg = String.format("Error parsing '%s' '%s' from nvdcve.lastupdated",
+                                        DataStoreMetaInfo.LAST_UPDATED_BASE, entry.getId());
+                                Logger.getLogger(StandardUpdateTask.class.getName()).log(Level.FINE, msg, ex);
+                            }
+                            if (currentTimestamp == entry.getTimestamp()) {
+                                entry.setNeedsUpdate(false);
+                            }
+                        }
+                    }
+                }
+            } catch (NumberFormatException ex) {
+                final String msg = "An invalid schema version or timestamp exists in the data.properties file.";
+                Logger.getLogger(StandardUpdateTask.class.getName()).log(Level.WARNING, msg);
+                Logger.getLogger(StandardUpdateTask.class.getName()).log(Level.FINE, null, ex);
+            }
+        }
+        return updates;
+    }
+
+    /**
+     * Retrieves the timestamps from the NVD CVE meta data file.
+     *
+     * @return the timestamp from the currently published nvdcve downloads page
+     * @throws MalformedURLException thrown if the URL for the NVD CCE Meta data
+     * is incorrect.
+     * @throws DownloadFailedException thrown if there is an error downloading
+     * the nvd cve meta data file
+     * @throws InvalidDataException thrown if there is an exception parsing the
+     * timestamps
+     * @throws InvalidSettingException thrown if the settings are invalid
+     */
+    private Updateable retrieveCurrentTimestampsFromWeb()
+            throws MalformedURLException, DownloadFailedException, InvalidDataException, InvalidSettingException {
+
+        final Updateable updates = new Updateable();
+        updates.add(MODIFIED, Settings.getString(Settings.KEYS.CVE_MODIFIED_20_URL),
+                Settings.getString(Settings.KEYS.CVE_MODIFIED_12_URL),
+                false);
+
+        final int start = Settings.getInt(Settings.KEYS.CVE_START_YEAR);
+        final int end = Calendar.getInstance().get(Calendar.YEAR);
+        final String baseUrl20 = Settings.getString(Settings.KEYS.CVE_SCHEMA_2_0);
+        final String baseUrl12 = Settings.getString(Settings.KEYS.CVE_SCHEMA_1_2);
+        for (int i = start; i <= end; i++) {
+            updates.add(Integer.toString(i), String.format(baseUrl20, i),
+                    String.format(baseUrl12, i),
+                    true);
+        }
+
+        return updates;
+    }
+}

dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/UpdateTask.java

@@ -0,0 +1,51 @@
+/*
+ * This file is part of dependency-check-core.
+ *
+ * Dependency-check-core is free software: you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation, either version 3 of the License, or (at your option) any
+ * later version.
+ *
+ * Dependency-check-core is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+ * details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * dependency-check-core. If not, see http://www.gnu.org/licenses/.
+ *
+ * Copyright (c) 2012 Jeremy Long. All Rights Reserved.
+ */
+package org.owasp.dependencycheck.data.update;
+
+import org.owasp.dependencycheck.data.UpdateException;
+
+/**
+ * An interface defining an update task.
+ *
+ * @author Jeremy Long (jeremy.long@owasp.org)
+ */
+public interface UpdateTask {
+
+    /**
+     * <p>Updates the data store to the latest version.</p>
+     *
+     * @throws UpdateException is thrown if there is an error updating the
+     * database
+     */
+    void update() throws UpdateException;
+
+    /**
+     * Get the value of deleteAndRecreate.
+     *
+     * @return the value of deleteAndRecreate
+     */
+    boolean shouldDeleteAndRecreate();
+
+    /**
+     * Gets whether or not an update is needed.
+     *
+     * @return true or false depending on whether an update is needed
+     */
+    boolean isUpdateNeeded();
+}

dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/UpdateTaskFactory.java

@@ -0,0 +1,60 @@
+/*
+ * This file is part of dependency-check-core.
+ *
+ * Dependency-check-core is free software: you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation, either version 3 of the License, or (at your option) any
+ * later version.
+ *
+ * Dependency-check-core is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+ * details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * dependency-check-core. If not, see http://www.gnu.org/licenses/.
+ *
+ * Copyright (c) 2012 Jeremy Long. All Rights Reserved.
+ */
+package org.owasp.dependencycheck.data.update;
+
+import java.net.MalformedURLException;
+import org.owasp.dependencycheck.data.UpdateException;
+import org.owasp.dependencycheck.utils.DownloadFailedException;
+
+/**
+ * An UpdateTask Factory that instantiates the correct UpdateTask based on the
+ * given configuration.
+ *
+ * @author Jeremy Long (jeremy.long@owasp.org)
+ */
+public final class UpdateTaskFactory {
+
+    /**
+     * private constructor for a utility class.
+     */
+    private UpdateTaskFactory() {
+        //empty contrusctor for utility class
+    }
+
+    /**
+     * Constructs the appropriate update task based on configuration.
+     *
+     * @return an UpdateTask
+     * @throws MalformedURLException thrown if a configured URL is malformed
+     * @throws DownloadFailedException thrown if a timestamp cannot be checked
+     * on a configured URL
+     * @throws UpdateException thrown if there is an exception generating the
+     * update task
+     */
+    public static UpdateTask getUpdateTask() throws MalformedURLException, DownloadFailedException, UpdateException {
+        final UpdateTask task;
+        final DataStoreMetaInfo properties = new DataStoreMetaInfo();
+        if (properties.isBatchUpdateMode()) {
+            task = new BatchUpdateTask(properties);
+        } else {
+            task = new StandardUpdateTask(properties);
+        }
+        return task;
+    }
+}

dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/Updateable.java

@@ -0,0 +1,181 @@
+/*
+ * This file is part of dependency-check-core.
+ *
+ * Dependency-check-core is free software: you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation, either version 3 of the License, or (at your option) any
+ * later version.
+ *
+ * Dependency-check-core is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+ * details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * dependency-check-core. If not, see http://www.gnu.org/licenses/.
+ *
+ * Copyright (c) 2012 Jeremy Long. All Rights Reserved.
+ */
+package org.owasp.dependencycheck.data.update;
+
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.TreeMap;
+import org.owasp.dependencycheck.utils.DownloadFailedException;
+import org.owasp.dependencycheck.utils.Downloader;
+
+/**
+ * Contains a collection of updateable NvdCveInfo objects. This is used to
+ * determine which files need to be downloaded and processed.
+ *
+ * @author Jeremy Long (jeremy.long@owasp.org)
+ */
+public class Updateable implements java.lang.Iterable<NvdCveInfo>, Iterator<NvdCveInfo> {
+
+    /**
+     * A collection of sources of data.
+     */
+    private Map<String, NvdCveInfo> collection = new TreeMap<String, NvdCveInfo>();
+
+    /**
+     * Returns the collection of NvdCveInfo objects. This method is mainly used
+     * for testing.
+     *
+     * @return the collection of NvdCveInfo objects
+     */
+    protected Map<String, NvdCveInfo> getCollection() {
+        return collection;
+    }
+
+    /**
+     * Gets whether or not an update is needed.
+     *
+     * @return true or false depending on whether an update is needed
+     */
+    public boolean isUpdateNeeded() {
+        for (NvdCveInfo item : this) {
+            if (item.getNeedsUpdate()) {
+                return true;
+            }
+        }
+        return false;
+    }
+
+    /**
+     * Adds a new entry of updateable information to the contained collection.
+     *
+     * @param id the key for the item to be added
+     * @param url the URL to download the item
+     * @param oldUrl the URL for the old version of the item (the NVD CVE old
+     * schema still contains useful data we need).
+     * @throws MalformedURLException thrown if the URL provided is invalid
+     * @throws DownloadFailedException thrown if the download fails.
+     */
+    public void add(String id, String url, String oldUrl) throws MalformedURLException, DownloadFailedException {
+        add(id, url, oldUrl, false);
+    }
+
+    /**
+     * Adds a new entry of updateable information to the contained collection.
+     *
+     * @param id the key for the item to be added
+     * @param url the URL to download the item
+     * @param oldUrl the URL for the old version of the item (the NVD CVE old
+     * schema still contains useful data we need).
+     * @param needsUpdate whether or not the data needs to be updated
+     * @throws MalformedURLException thrown if the URL provided is invalid
+     * @throws DownloadFailedException thrown if the download fails.
+     */
+    public void add(String id, String url, String oldUrl, boolean needsUpdate) throws MalformedURLException, DownloadFailedException {
+        final NvdCveInfo item = new NvdCveInfo();
+        item.setNeedsUpdate(needsUpdate); //the others default to true, to make life easier later this should default to false.
+        item.setId(id);
+        item.setUrl(url);
+        item.setOldSchemaVersionUrl(oldUrl);
+        item.setTimestamp(Downloader.getLastModified(new URL(url)));
+        collection.put(id, item);
+    }
+
+    /**
+     * Clears the contained collection of NvdCveInfo entries.
+     */
+    public void clear() {
+        collection.clear();
+    }
+
+    /**
+     * Returns the timestamp for the given entry.
+     *
+     * @param key the key to lookup in the collection of NvdCveInfo items
+     * @return the timestamp for the given entry
+     */
+    public long getTimeStamp(String key) {
+        return collection.get(key).getTimestamp();
+    }
+    /**
+     * An internal iterator used to implement iterable.
+     */
+    private Iterator<Entry<String, NvdCveInfo>> iterableContent = null;
+
+    /**
+     * <p>Returns an iterator for the NvdCveInfo contained.</p>
+     * <p><b>This method is not thread safe.</b></p>
+     *
+     * @return an NvdCveInfo Iterator
+     */
+    @Override
+    public Iterator<NvdCveInfo> iterator() {
+        iterableContent = collection.entrySet().iterator();
+        return this;
+    }
+
+    /**
+     * <p>Returns whether or not there is another item in the collection.</p>
+     * <p><b>This method is not thread safe.</b></p>
+     *
+     * @return true or false depending on whether or not another item exists in
+     * the collection
+     */
+    @Override
+    public boolean hasNext() {
+        return iterableContent.hasNext();
+    }
+
+    /**
+     * <p>Returns the next item in the collection.</p>
+     * <p><b>This method is not thread safe.</b></p>
+     *
+     * @return the next NvdCveInfo item in the collection
+     */
+    @Override
+    public NvdCveInfo next() {
+        return iterableContent.next().getValue();
+    }
+
+    /**
+     * <p>Removes the current NvdCveInfo object from the collection.</p>
+     * <p><b>This method is not thread safe.</b></p>
+     */
+    @Override
+    public void remove() {
+        iterableContent.remove();
+    }
+
+    /**
+     * Returns the specified item from the collection.
+     *
+     * @param key the key to lookup the return value
+     * @return the NvdCveInfo object stored using the specified key
+     */
+    NvdCveInfo get(String key) {
+        return collection.get(key);
+    }
+
+    @Override
+    public String toString() {
+        return "Updateable{" + "size=" + collection.size() + '}';
+    }
+}

dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/package-info.java

@@ -15,4 +15,4 @@
  * </html>
 */
 
-package org.owasp.dependencycheck.data.nvdcve.xml;
+package org.owasp.dependencycheck.data.update;

dependency-check-core/src/main/java/org/owasp/dependencycheck/utils/Downloader.java

@@ -23,10 +23,13 @@ import java.io.File;
 import java.io.FileOutputStream;
 import java.io.IOException;
 import java.io.InputStream;
+import java.net.Authenticator;
 import java.net.HttpURLConnection;
 import java.net.InetSocketAddress;
+import java.net.PasswordAuthentication;
 import java.net.Proxy;
 import java.net.SocketAddress;
+import java.net.URISyntaxException;
 import java.net.URL;
 import java.util.logging.Level;
 import java.util.logging.Logger;
@@ -46,33 +49,6 @@ public final class Downloader {
     private Downloader() {
     }
 
-    /**
-     * Retrieves a file from a given URL and saves it to the outputPath.
-     *
-     * @param url the URL of the file to download.
-     * @param outputPath the path to the save the file to.
-     * @throws DownloadFailedException is thrown if there is an error
-     * downloading the file.
-     */
-    public static void fetchFile(URL url, String outputPath) throws DownloadFailedException {
-        fetchFile(url, outputPath, false);
-    }
-
-    /**
-     * Retrieves a file from a given URL and saves it to the outputPath.
-     *
-     * @param url the URL of the file to download.
-     * @param outputPath the path to the save the file to.
-     * @param unzip true/false indicating that the file being retrieved is
-     * gzipped and if true, should be uncompressed before writing to the file.
-     * @throws DownloadFailedException is thrown if there is an error
-     * downloading the file.
-     */
-    public static void fetchFile(URL url, String outputPath, boolean unzip) throws DownloadFailedException {
-        final File f = new File(outputPath);
-        fetchFile(url, f, unzip);
-    }
-
     /**
      * Retrieves a file from a given URL and saves it to the outputPath.
      *
@@ -82,20 +58,6 @@ public final class Downloader {
      * downloading the file.
      */
     public static void fetchFile(URL url, File outputPath) throws DownloadFailedException {
-        fetchFile(url, outputPath, false);
-    }
-
-    /**
-     * Retrieves a file from a given URL and saves it to the outputPath.
-     *
-     * @param url the URL of the file to download.
-     * @param outputPath the path to the save the file to.
-     * @param unzip true/false indicating that the file being retrieved is
-     * gzipped and if true, should be uncompressed before writing to the file.
-     * @throws DownloadFailedException is thrown if there is an error
-     * downloading the file.
-     */
-    public static void fetchFile(URL url, File outputPath, boolean unzip) throws DownloadFailedException {
         HttpURLConnection conn = null;
         try {
             conn = Downloader.getConnection(url);
@@ -116,7 +78,7 @@ public final class Downloader {
         BufferedOutputStream writer = null;
         InputStream reader = null;
         try {
-            if (unzip || (encoding != null && "gzip".equalsIgnoreCase(encoding))) {
+            if (encoding != null && "gzip".equalsIgnoreCase(encoding)) {
                 reader = new GZIPInputStream(conn.getInputStream());
             } else if (encoding != null && "deflate".equalsIgnoreCase(encoding)) {
                 reader = new InflaterInputStream(conn.getInputStream());
@@ -136,7 +98,6 @@ public final class Downloader {
             if (writer != null) {
                 try {
                     writer.close();
-                    writer = null;
                 } catch (Exception ex) {
                     Logger.getLogger(Downloader.class.getName()).log(Level.FINEST,
                             "Error closing the writer in Downloader.", ex);
@@ -145,9 +106,7 @@ public final class Downloader {
             if (reader != null) {
                 try {
                     reader.close();
-                    reader = null;
                 } catch (Exception ex) {
-
                     Logger.getLogger(Downloader.class.getName()).log(Level.FINEST,
                             "Error closing the reader in Downloader.", ex);
                 }
@@ -162,7 +121,8 @@ public final class Downloader {
 
     /**
      * Makes an HTTP Head request to retrieve the last modified date of the
-     * given URL.
+     * given URL. If the file:// protocol is specified, then the lastTimestamp
+     * of the file is returned.
      *
      * @param url the URL to retrieve the timestamp from
      * @return an epoch timestamp
@@ -170,21 +130,42 @@ public final class Downloader {
      * the HTTP request
      */
     public static long getLastModified(URL url) throws DownloadFailedException {
-        HttpURLConnection conn = null;
         long timestamp = 0;
-        try {
-            conn = Downloader.getConnection(url);
-            conn.setRequestMethod("HEAD");
-            conn.connect();
-            timestamp = conn.getLastModified();
-        } catch (Exception ex) {
-            throw new DownloadFailedException("Error making HTTP HEAD request.", ex);
-        } finally {
-            if (conn != null) {
-                try {
-                    conn.disconnect();
-                } finally {
-                    conn = null;
+        //TODO add the FPR protocol?
+        if ("file".equalsIgnoreCase(url.getProtocol())) {
+            File lastModifiedFile;
+            try {
+//                if (System.getProperty("os.name").toLowerCase().startsWith("windows")) {
+//                    String filePath = url.toString();
+//                    if (filePath.matches("file://[a-zA-Z]:.*")) {
+//                        f = new File(filePath.substring(7));
+//                    } else {
+//                        f = new File(url.toURI());
+//                    }
+//                } else {
+                lastModifiedFile = new File(url.toURI());
+//                }
+            } catch (URISyntaxException ex) {
+                final String msg = String.format("Unable to locate '%s'; is the cve.url-2.0.modified property set correctly?", url.toString());
+                throw new DownloadFailedException(msg);
+            }
+            timestamp = lastModifiedFile.lastModified();
+        } else {
+            HttpURLConnection conn = null;
+            try {
+                conn = Downloader.getConnection(url);
+                conn.setRequestMethod("HEAD");
+                conn.connect();
+                timestamp = conn.getLastModified();
+            } catch (Exception ex) {
+                throw new DownloadFailedException("Error making HTTP HEAD request.", ex);
+            } finally {
+                if (conn != null) {
+                    try {
+                        conn.disconnect();
+                    } finally {
+                        conn = null;
+                    }
                 }
             }
         }
@@ -208,16 +189,29 @@ public final class Downloader {
             if (proxyUrl != null) {
                 final int proxyPort = Settings.getInt(Settings.KEYS.PROXY_PORT);
                 final SocketAddress addr = new InetSocketAddress(proxyUrl, proxyPort);
+
+                final String username = Settings.getString(Settings.KEYS.PROXY_USERNAME);
+                final String password = Settings.getString(Settings.KEYS.PROXY_PASSWORD);
+                if (username != null && password != null) {
+                    final Authenticator auth = new Authenticator() {
+                        @Override
+                        public PasswordAuthentication getPasswordAuthentication() {
+                            if (getRequestorType().equals(RequestorType.PROXY)) {
+                                return new PasswordAuthentication(username, password.toCharArray());
+                            }
+                            return super.getPasswordAuthentication();
+                        }
+                    };
+                    Authenticator.setDefault(auth);
+                }
+
                 proxy = new Proxy(Proxy.Type.HTTP, addr);
                 conn = (HttpURLConnection) url.openConnection(proxy);
             } else {
                 conn = (HttpURLConnection) url.openConnection();
             }
-            //added a default timeout of 20000
-            //if (Settings.getString(Settings.KEYS.CONNECTION_TIMEOUT) != null) {
             final int timeout = Settings.getInt(Settings.KEYS.CONNECTION_TIMEOUT, 60000);
             conn.setConnectTimeout(timeout);
-            //}
         } catch (IOException ex) {
             if (conn != null) {
                 try {

dependency-check-core/src/main/java/org/owasp/dependencycheck/utils/ExtractionException.java

@@ -0,0 +1,69 @@
+/*
+ * This file is part of dependency-check-core.
+ *
+ * Dependency-check-core is free software: you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation, either version 3 of the License, or (at your option) any
+ * later version.
+ *
+ * Dependency-check-core is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+ * details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * dependency-check-core. If not, see http://www.gnu.org/licenses/.
+ *
+ * Copyright (c) 2013 Jeremy Long. All Rights Reserved.
+ */
+package org.owasp.dependencycheck.utils;
+
+import java.io.IOException;
+
+/**
+ * An exception used when a file is unable to be un-zipped.
+ *
+ * @author Jeremy Long (jeremy.long@owasp.org)
+ */
+public class ExtractionException extends IOException {
+
+    /**
+     * The serial version UID.
+     */
+    private static final long serialVersionUID = 1L;
+
+    /**
+     * Creates a new ExtractionException.
+     */
+    public ExtractionException() {
+        super();
+    }
+
+    /**
+     * Creates a new ExtractionException.
+     *
+     * @param msg a message for the exception.
+     */
+    public ExtractionException(String msg) {
+        super(msg);
+    }
+
+    /**
+     * Creates a new ExtractionException.
+     *
+     * @param ex the cause of the download failure.
+     */
+    public ExtractionException(Throwable ex) {
+        super(ex);
+    }
+
+    /**
+     * Creates a new ExtractionException.
+     *
+     * @param msg a message for the exception.
+     * @param ex the cause of the download failure.
+     */
+    public ExtractionException(String msg, Throwable ex) {
+        super(msg, ex);
+    }
+}

dependency-check-core/src/main/java/org/owasp/dependencycheck/utils/FileUtils.java

@@ -18,11 +18,20 @@
  */
 package org.owasp.dependencycheck.utils;
 
+import java.io.BufferedInputStream;
+import java.io.BufferedOutputStream;
 import java.io.File;
+import java.io.FileInputStream;
 import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
 import java.io.IOException;
 import java.io.UnsupportedEncodingException;
 import java.net.URLDecoder;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipInputStream;
+import org.owasp.dependencycheck.Engine;
 
 /**
  * A collection of utilities for processing information about files.
@@ -31,6 +40,11 @@ import java.net.URLDecoder;
  */
 public final class FileUtils {
 
+    /**
+     * The buffer size to use when extracting files from the archive.
+     */
+    private static final int BUFFER_SIZE = 4096;
+
     /**
      * Private constructor for a utility class.
      */
@@ -65,9 +79,15 @@ public final class FileUtils {
                 delete(c);
             }
         }
-        if (!file.delete()) {
+        if (!org.apache.commons.io.FileUtils.deleteQuietly(file)) {
             throw new FileNotFoundException("Failed to delete file: " + file);
         }
+        /* else {
+         //delete on exit was a bad idea. if for some reason the file can't be deleted
+         // this will cause a newly constructed file to be deleted and a subsequent run may fail.
+         // still not sure why a file fails to be deleted, but can be overwritten... odd.
+         file.deleteOnExit();
+         }*/
     }
 
     /**
@@ -114,4 +134,100 @@ public final class FileUtils {
         final File jarPath = new File(decodedPath);
         return jarPath.getParentFile();
     }
+
+    /**
+     * Extracts the contents of an archive into the specified directory.
+     *
+     * @param archive an archive file such as a WAR or EAR
+     * @param extractTo a directory to extract the contents to
+     * @throws ExtractionException thrown if an exception occurs while
+     * extracting the files
+     */
+    public static void extractFiles(File archive, File extractTo) throws ExtractionException {
+        extractFiles(archive, extractTo, null);
+    }
+
+    /**
+     * Extracts the contents of an archive into the specified directory. The
+     * files are only extracted if they are supported by the analyzers loaded
+     * into the specified engine. If the engine is specified as null then all
+     * files are extracted.
+     *
+     * @param archive an archive file such as a WAR or EAR
+     * @param extractTo a directory to extract the contents to
+     * @param engine the scanning engine
+     * @throws ExtractionException thrown if there is an error extracting the
+     * files
+     */
+    public static void extractFiles(File archive, File extractTo, Engine engine) throws ExtractionException {
+        if (archive == null || extractTo == null) {
+            return;
+        }
+
+        FileInputStream fis = null;
+        ZipInputStream zis = null;
+
+        try {
+            fis = new FileInputStream(archive);
+        } catch (FileNotFoundException ex) {
+            Logger.getLogger(FileUtils.class.getName()).log(Level.INFO, null, ex);
+            throw new ExtractionException("Archive file was not found.", ex);
+        }
+        zis = new ZipInputStream(new BufferedInputStream(fis));
+        ZipEntry entry;
+        try {
+            while ((entry = zis.getNextEntry()) != null) {
+                if (entry.isDirectory()) {
+                    final File d = new File(extractTo, entry.getName());
+                    if (!d.exists() && !d.mkdirs()) {
+                        final String msg = String.format("Unable to create '%s'.", d.getAbsolutePath());
+                        throw new ExtractionException(msg);
+                    }
+                } else {
+                    final File file = new File(extractTo, entry.getName());
+                    final String ext = getFileExtension(file.getName());
+                    if (engine == null || engine.supportsExtension(ext)) {
+                        BufferedOutputStream bos = null;
+                        FileOutputStream fos;
+                        try {
+                            fos = new FileOutputStream(file);
+                            bos = new BufferedOutputStream(fos, BUFFER_SIZE);
+                            int count;
+                            final byte data[] = new byte[BUFFER_SIZE];
+                            while ((count = zis.read(data, 0, BUFFER_SIZE)) != -1) {
+                                bos.write(data, 0, count);
+                            }
+                            bos.flush();
+                        } catch (FileNotFoundException ex) {
+                            Logger.getLogger(FileUtils.class.getName()).log(Level.FINE, null, ex);
+                            final String msg = String.format("Unable to find file '%s'.", file.getName());
+                            throw new ExtractionException(msg, ex);
+                        } catch (IOException ex) {
+                            Logger.getLogger(FileUtils.class.getName()).log(Level.FINE, null, ex);
+                            final String msg = String.format("IO Exception while parsing file '%s'.", file.getName());
+                            throw new ExtractionException(msg, ex);
+                        } finally {
+                            if (bos != null) {
+                                try {
+                                    bos.close();
+                                } catch (IOException ex) {
+                                    Logger.getLogger(FileUtils.class.getName()).log(Level.FINEST, null, ex);
+                                }
+                            }
+                        }
+                    }
+                }
+            }
+        } catch (IOException ex) {
+            final String msg = String.format("Exception reading archive '%s'.", archive.getName());
+            Logger.getLogger(FileUtils.class.getName()).log(Level.FINE, msg, ex);
+            throw new ExtractionException(msg, ex);
+        } finally {
+            try {
+                zis.close();
+            } catch (IOException ex) {
+                Logger.getLogger(FileUtils.class.getName()).log(Level.FINEST, null, ex);
+            }
+        }
+    }
 }

dependency-check-core/src/main/java/org/owasp/dependencycheck/utils/LogFilter.java

@@ -0,0 +1,44 @@
+/*
+ * This file is part of dependency-check-core.
+ *
+ * Dependency-check-core is free software: you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation, either version 3 of the License, or (at your option) any
+ * later version.
+ *
+ * Dependency-check-core is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+ * details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * dependency-check-core. If not, see http://www.gnu.org/licenses/.
+ *
+ * Copyright (c) 2013 Jeremy Long. All Rights Reserved.
+ */
+package org.owasp.dependencycheck.utils;
+
+import java.util.logging.Filter;
+import java.util.logging.LogRecord;
+
+/**
+ * A simple log filter to limit the entries written to the verbose log file. The
+ * verbose log file uses the root logger as I couldn't get anything else to
+ * work; as such, this filter limits the log entries to specific classes.
+ *
+ * @author Jeremy Long (jeremy.long@owasp.org)
+ */
+public class LogFilter implements Filter {
+
+    /**
+     * Determines if the record should be logged.
+     *
+     * @param record a log record to examine
+     * @return true if the record should be logged, otherwise false
+     */
+    @Override
+    public boolean isLoggable(LogRecord record) {
+        final String name = record.getSourceClassName();
+        return name.startsWith("org.owasp.dependencycheck") && !name.contains("generated") && !name.contains("VelocityLoggerRedirect");
+    }
+}

dependency-check-core/src/main/java/org/owasp/dependencycheck/utils/LogUtils.java

@@ -0,0 +1,89 @@
+/*
+ * This file is part of dependency-check-core.
+ *
+ * Dependency-check-core is free software: you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation, either version 3 of the License, or (at your option) any
+ * later version.
+ *
+ * Dependency-check-core is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+ * details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * dependency-check-core. If not, see http://www.gnu.org/licenses/.
+ *
+ * Copyright (c) 2013 Jeremy Long. All Rights Reserved.
+ */
+package org.owasp.dependencycheck.utils;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.logging.FileHandler;
+import java.util.logging.Level;
+import java.util.logging.LogManager;
+import java.util.logging.Logger;
+import java.util.logging.SimpleFormatter;
+
+/**
+ * A utility class to aide in the setup of the logging mechanism.
+ *
+ * @author Jeremy Long (jeremy.long@owasp.org)
+ */
+public final class LogUtils {
+
+    /**
+     * Private constructor for a utility class.
+     */
+    private LogUtils() {
+    }
+
+    /**
+     * Configures the logger for use by the application.
+     *
+     * @param in the input stream to read the log settings from
+     * @param verboseLogFile the file path for the verbose log
+     */
+    public static void prepareLogger(InputStream in, String verboseLogFile) {
+        try {
+            LogManager.getLogManager().reset();
+            LogManager.getLogManager().readConfiguration(in);
+            if (verboseLogFile != null && !verboseLogFile.isEmpty()) {
+                verboseLoggingEnabled = true;
+                final Logger logger = Logger.getLogger("");
+                final FileHandler handler = new FileHandler(verboseLogFile, true);
+                handler.setFormatter(new SimpleFormatter());
+                handler.setLevel(Level.FINE);
+                handler.setFilter(new LogFilter());
+                logger.addHandler(handler);
+                logger.setLevel(Level.FINE);
+            }
+        } catch (IOException ex) {
+            Logger.getLogger(LogUtils.class.getName()).log(Level.FINE, "IO Error preparing the logger", ex);
+        } catch (SecurityException ex) {
+            Logger.getLogger(LogUtils.class.getName()).log(Level.FINE, "Error preparing the logger", ex);
+        } finally {
+            if (in != null) {
+                try {
+                    in.close();
+                } catch (Exception ex) {
+                    Logger.getLogger(LogUtils.class.getName()).log(Level.FINEST, "Error closing resource stream", ex);
+                }
+            }
+        }
+    }
+    /**
+     * Whether or not verbose logging is enabled.
+     */
+    private static boolean verboseLoggingEnabled = false;
+
+    /**
+     * Get the value of verboseLoggingEnabled.
+     *
+     * @return the value of verboseLoggingEnabled
+     */
+    public static boolean isVerboseLoggingEnabled() {
+        return verboseLoggingEnabled;
+    }
+}

dependency-check-core/src/main/java/org/owasp/dependencycheck/utils/Settings.java

@@ -67,6 +67,11 @@ public final class Settings {
          * The base path to use for the data directory.
          */
         public static final String DATA_DIRECTORY = "data.directory";
+        /**
+         * The location of the batch update URL. This is a zip file that
+         * contains the contents of the data directory.
+         */
+        public static final String BATCH_UPDATE_URL = "batch.update.url";
         /**
          * The properties key for the path where the CPE Lucene Index will be
          * stored.
@@ -77,14 +82,6 @@ public final class Settings {
          * stored.
          */
         public static final String CVE_DATA_DIRECTORY = "data.cve";
-        /**
-         * The properties key for the URL to the CPE.
-         */
-        public static final String CPE_URL = "cpe.url";
-        /**
-         * The properties key for the URL to the CPE.
-         */
-        public static final String CPE_META_URL = "cpe.meta.url";
         /**
          * The properties key for the URL to retrieve the "meta" data from about
          * the CVE entries.
@@ -128,6 +125,14 @@ public final class Settings {
          * value.
          */
         public static final String PROXY_PORT = "proxy.port";
+        /**
+         * The properties key for the proxy username.
+         */
+        public static final String PROXY_USERNAME = "proxy.username";
+        /**
+         * The properties key for the proxy password.
+         */
+        public static final String PROXY_PASSWORD = "proxy.password";
         /**
          * The properties key for the connection timeout.
          */
@@ -198,6 +203,23 @@ public final class Settings {
         }
     }
 
+    /**
+     * Merges a new properties file into the current properties. This method
+     * allows for the loading of a user provided properties file.<br/><br/>
+     * Note: even if using this method - system properties will be loaded before
+     * properties loaded from files.
+     *
+     * @param filePath the path to the properties file to merge.
+     * @throws FileNotFoundException is thrown when the filePath points to a
+     * non-existent file
+     * @throws IOException is thrown when there is an exception loading/merging
+     * the properties
+     */
+    public static void mergeProperties(File filePath) throws FileNotFoundException, IOException {
+        final FileInputStream fis = new FileInputStream(filePath);
+        mergeProperties(fis);
+    }
+
     /**
      * Merges a new properties file into the current properties. This method
      * allows for the loading of a user provided properties file.<br/><br/>
@@ -229,25 +251,6 @@ public final class Settings {
         INSTANCE.props.load(stream);
     }
 
-    /**
-     * Returns a value from the properties file as a File object. If the value
-     * was specified as a system property or passed in via the -Dprop=value
-     * argument - this method will return the value from the system properties
-     * before the values in the contained configuration file.
-     *
-     * @param key the key to lookup within the properties file
-     * @param defaultValue the default value for the requested property
-     * @return the property from the properties file as a File object
-     */
-    public static File getFile(String key, String defaultValue) {
-        final String baseDir = getString(Settings.KEYS.DATA_DIRECTORY);
-        final String str = getString(key, defaultValue);
-        if (baseDir != null) {
-            return new File(baseDir, str);
-        }
-        return new File(str);
-    }
-
     /**
      * Returns a value from the properties file as a File object. If the value
      * was specified as a system property or passed in via the -Dprop=value
@@ -259,17 +262,22 @@ public final class Settings {
      *
      * @param key the key to lookup within the properties file
      * @return the property from the properties file converted to a File object
-     * @throws IOException thrown if the file path to the JAR cannot be found
      */
-    public static File getFile(String key) throws IOException {
+    public static File getFile(String key) {
         final String file = getString(key);
         final String baseDir = getString(Settings.KEYS.DATA_DIRECTORY);
         if (baseDir != null) {
             if (baseDir.startsWith("[JAR]/")) {
                 final File jarPath = getJarPath();
-                final File newBase = new File(jarPath.getCanonicalPath(), baseDir.substring(6));
+                final File newBase = new File(jarPath, baseDir.substring(6));
+                if (Settings.KEYS.DATA_DIRECTORY.equals(key)) {
+                    return newBase;
+                }
                 return new File(newBase, file);
             }
+            if (Settings.KEYS.DATA_DIRECTORY.equals(key)) {
+                return new File(baseDir);
+            }
             return new File(baseDir, file);
         }
         return new File(file);
@@ -313,6 +321,15 @@ public final class Settings {
         return str;
     }
 
+    /**
+     * Returns the temporary directory.
+     *
+     * @return the temporary directory
+     */
+    public static File getTempDirectory() {
+        return new File(Settings.getString(Settings.KEYS.TEMP_DIRECTORY, System.getProperty("java.io.tmpdir")));
+    }
+
     /**
      * Returns a value from the properties file. If the value was specified as a
      * system property or passed in via the -Dprop=value argument - this method
@@ -326,6 +343,16 @@ public final class Settings {
         return System.getProperty(key, INSTANCE.props.getProperty(key));
     }
 
+    /**
+     * Removes a property from the local properties collection. This is mainly
+     * used in test cases.
+     *
+     * @param key the property key to remove
+     */
+    public static void removeProperty(String key) {
+        INSTANCE.props.remove(key);
+    }
+
     /**
      * Returns an int value from the properties file. If the value was specified
      * as a system property or passed in via the -Dprop=value argument - this

dependency-check-core/src/main/java/org/owasp/dependencycheck/utils/UrlStringUtils.java

@@ -21,6 +21,8 @@ package org.owasp.dependencycheck.utils;
 import java.net.MalformedURLException;
 import java.net.URL;
 import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashSet;
 import java.util.List;
 import java.util.regex.Pattern;
 
@@ -64,6 +66,12 @@ public final class UrlStringUtils {
     public static boolean isUrl(String text) {
         return IS_URL_TEST.matcher(text).matches();
     }
+    /**
+     * A listing of domain parts that shold not be used as evidence. Yes, this
+     * is an incomplete list.
+     */
+    private static final HashSet<String> IGNORE_LIST = new HashSet<String>(
+            Arrays.asList("www", "com", "org", "gov", "info", "name", "net", "pro", "tel", "mobi", "xxx"));
 
     /**
      * <p>Takes a URL, in String format, and adds the important parts of the URL
@@ -84,7 +92,7 @@ public final class UrlStringUtils {
         //add the domain except www and the tld.
         for (int i = 0; i < domain.length - 1; i++) {
             final String sub = domain[i];
-            if (!"www".equalsIgnoreCase(sub)) {
+            if (!IGNORE_LIST.contains(sub.toLowerCase())) {
                 importantParts.add(sub);
             }
         }

dependency-check-core/src/main/resources/META-INF/licenses/commons-compress/LICENSE.txt

@@ -0,0 +1,202 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

dependency-check-core/src/main/resources/META-INF/services/org.owasp.dependencycheck.analyzer.Analyzer

@@ -4,5 +4,5 @@ org.owasp.dependencycheck.analyzer.FileNameAnalyzer
 org.owasp.dependencycheck.analyzer.HintAnalyzer
 org.owasp.dependencycheck.analyzer.DependencyBundlingAnalyzer
 org.owasp.dependencycheck.analyzer.FalsePositiveAnalyzer
-org.owasp.dependencycheck.data.cpe.CPEAnalyzer
-org.owasp.dependencycheck.data.nvdcve.NvdCveAnalyzer
+org.owasp.dependencycheck.analyzer.CPEAnalyzer
+org.owasp.dependencycheck.analyzer.NvdCveAnalyzer

dependency-check-core/src/main/resources/META-INF/services/org.owasp.dependencycheck.data.CachedWebDataSource

@@ -1 +1 @@
-org.owasp.dependencycheck.data.nvdcve.xml.DatabaseUpdater
+org.owasp.dependencycheck.data.update.DatabaseUpdater

dependency-check-core/src/main/resources/dependencycheck.properties

@@ -23,6 +23,15 @@ cpe.meta.url=http://static.nvd.nist.gov/feeds/xml/cpe/dictionary/official-cpe-di
 # holds 8 days of updates, we are using 7 just to be safe.
 cve.url.modified.validfordays=7
 
+# The location of the zipped CVE H2 database and CPE Lucene index. If specified and
+# a full download of data is required this URL will be used and the data extracted
+# into the specified "data" directory. Additionally, after pulling the data the
+# system will attempt to update the modified. Thus, if one were maintaining an
+# internal copy of the data one would not need to update it nightly.
+# If the 'modified' URLs below for the CVE are removed and a batch url is provided
+# then if an update is required, the entre zip file will be downloaded.
+#batch.update.url=file:///C:/path/to/data.zip
+
 # the path to the modified nvd cve xml file.
 cve.url-1.2.modified=http://nvd.nist.gov/download/nvdcve-modified.xml
 cve.url-2.0.modified=http://static.nvd.nist.gov/feeds/xml/cve/nvdcve-2.0-modified.xml

dependency-check-core/src/main/resources/schema/DependencyCheck.xsd

@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="utf-8"?>
-<xs:schema id="analysis" xmlns="https://www.owasp.org/index.php/OWASP_Dependency_Check" xmlns:xs="http://www.w3.org/2001/XMLSchema">
+<xs:schema id="analysis" xmlns:xs="http://www.w3.org/2001/XMLSchema" elementFormDefault="qualified" targetNamespace="https://www.owasp.org/index.php/OWASP_Dependency_Check">
     <xs:element name="analysis">
         <xs:complexType>
             <xs:sequence minOccurs="0" maxOccurs="unbounded">

dependency-check-core/src/main/resources/templates/HtmlReport.vsl

@@ -52,6 +52,11 @@ Copyright (c) 2012 Jeremy Long. All Rights Reserved.
                 });
             });
         </script>
+        <script type="text/javascript">
+            function toggleVuln() {
+                $(".notvulnerable").toggle();
+            }
+        </script>
         <style type="text/css">
 
             .rounded-corners {
@@ -275,6 +280,15 @@ Copyright (c) 2012 Jeremy Long. All Rights Reserved.
                 margin-top:3px;
                 margin-bottom:3px;
             }
+            .vulnerable {
+            	color: #f00;
+            }
+            .vulnerable li {
+            	color: #000;
+            }
+            .notvulnerable {
+                display:none;
+            }
         </style>
     </head>
     <body>
@@ -292,13 +306,14 @@ Copyright (c) 2012 Jeremy Long. All Rights Reserved.
                         #set($vulnCount=$vulnCount+1)
                     #end
                 #end
-                Dependencies Scanned:&nbsp;$depCount<br/>
+                Dependencies Scanned:&nbsp;$depCount&nbsp;(<a href="#" onclick="toggleVuln()">show all</a>)<br/>
                 Vulnerable Dependencies:&nbsp;$vulnCount<br/><br/>
-                <div class="indent">
+                <ul class="indent">
                 #set($lnkcnt=0)
                 #foreach($dependency in $dependencies)
                     #set($lnkcnt=$lnkcnt+1)
-                    <a href="#l${lnkcnt}_$esc.html($esc.url($dependency.Sha1sum))">$esc.html($dependency.FileName)</a>#if($dependency.getVulnerabilities().size()>0)&nbsp;<b style="color:#ff0000;">&#8226;</b>#end<br/>
+                    <li class="#if($dependency.getVulnerabilities().size()==0)notvulnerable#else vulnerable#end">
+                    <a href="#l${lnkcnt}_$esc.html($esc.url($dependency.Sha1sum))">$esc.html($dependency.FileName)</a>
                     #if($dependency.getRelatedDependencies().size()>0)
                         <ul>
                         #foreach($related in $dependency.getRelatedDependencies())
@@ -306,15 +321,16 @@ Copyright (c) 2012 Jeremy Long. All Rights Reserved.
                         #end
                         </ul>
                     #end
+                    </li>
                 #end
-                </div>
+                </ul>
                 <h2>Dependencies</h2>
                 #set($lnkcnt=0)
                 #set($cnt=0)
                 #foreach($dependency in $dependencies)
                     #set($lnkcnt=$lnkcnt+1)
-                    <h3 class="subsectionheader standardsubsection"><a name="l${lnkcnt}_$esc.html($dependency.Sha1sum)"></a>$esc.html($dependency.FileName)</h3>
-                    <div class="subsectioncontent">
+                    <h3 class="subsectionheader standardsubsection#if($dependency.getVulnerabilities().size()==0) notvulnerable#end"><a name="l${lnkcnt}_$esc.html($dependency.Sha1sum)"></a>$esc.html($dependency.FileName)</h3>
+                    <div class="subsectioncontent#if($dependency.getVulnerabilities().size()==0) notvulnerable#end">
                         #if ($dependency.description)
                             <p><b>Description:</b>&nbsp;$esc.html($dependency.description)<br/></p>
                         #end
@@ -443,7 +459,7 @@ Copyright (c) 2012 Jeremy Long. All Rights Reserved.
                                     </ul>
                                 #end
                             </p>
-                            <p>Vulnerably Software:<ul>
+                            <p>Vulnerable Software &amp; Versions:<ul>
                             #foreach($vs in $vuln.getVulnerableSoftware())
                                 <li><a target="_blank" href="http://web.nvd.nist.gov/view/vuln/search-results?cpe=$esc.url($vs.name)">$esc.html($vs.name)</a> #if($vs.hasPreviousVersion()) and all previous versions.#end</li>
                             #end

dependency-check-core/src/main/resources/templates/XmlReport.vsl

@@ -21,30 +21,30 @@ Copyright (c) 2012 Jeremy Long. All Rights Reserved.
 *#<?xml version="1.0"?>
 <analysis xmlns="https://www.owasp.org/index.php/OWASP_Dependency_Check">
     <projectInfo>
-        <name>$esc.html($applicationName)</name>
+        <name>$esc.xml($applicationName)</name>
         <reportDate>$date</reportDate>
         <credits>This report contains data retrieved from the National Vulnerability Database: http://nvd.nist.gov</credits>
     </projectInfo>
     <dependencies>
 #foreach($dependency in $dependencies)
         <dependency>
-            <fileName>$esc.html($dependency.FileName)</fileName>
-            <filePath>$esc.html($dependency.FilePath)</filePath>
-            <md5>$esc.html($dependency.Md5sum)</md5>
-            <sha1>$esc.html($dependency.Sha1sum)</sha1>
+            <fileName>$esc.xml($dependency.FileName)</fileName>
+            <filePath>$esc.xml($dependency.FilePath)</filePath>
+            <md5>$esc.xml($dependency.Md5sum)</md5>
+            <sha1>$esc.xml($dependency.Sha1sum)</sha1>
 #if ($dependency.description)
-            <description>$esc.html($dependency.description)</description>
+            <description>$esc.xml($dependency.description)</description>
 #end
 #if ($dependency.license)
-            <license>$esc.html($dependency.license)</license>
+            <license>$esc.xml($dependency.license)</license>
 #end
 #if ($dependency.getRelatedDependencies().size()>0)
             <relatedDependencies>
 #foreach($related in $dependency.getRelatedDependencies())
                 <relatedDependency>
-                    <filePath>$esc.html($related.FilePath)</filePath>
-                    <sha1>$esc.html($related.Sha1sum)</sha1>
-                    <md5>$esc.html($related.Md5sum)</md5>
+                    <filePath>$esc.xml($related.FilePath)</filePath>
+                    <sha1>$esc.xml($related.Sha1sum)</sha1>
+                    <md5>$esc.xml($related.Md5sum)</md5>
                 </relatedDependency>
 #end
             </relatedDependencies>
@@ -53,21 +53,21 @@ Copyright (c) 2012 Jeremy Long. All Rights Reserved.
             <analysisExceptions>
 #foreach($ex in $dependency.analysisExceptions)
                 <exception>
-                    <message>$esc.html($ex.message)</message>
+                    <message>$esc.xml($ex.message)</message>
 #if ( $ex.stackTrace )
                     <stackTrace>
 #foreach ($st in $ex.stackTrace)
-                        <trace>$esc.html($st)</trace>
+                        <trace>$esc.xml($st)</trace>
 #end
                     </stackTrace>
 #end
 #if ( $ex.cause )
                     <innerException>
-                        <message>$esc.html($ex.cause.message)</message>
+                        <message>$esc.xml($ex.cause.message)</message>
 #if ( $ex.cause.stackTrace )
                         <stackTrace>
 #foreach ($st in $ex.cause.stackTrace)
-                            <trace>$esc.html($st)</trace>
+                            <trace>$esc.xml($st)</trace>
 #end
                         </stackTrace>
 #end
@@ -80,22 +80,22 @@ Copyright (c) 2012 Jeremy Long. All Rights Reserved.
             <evidenceCollected>
 #foreach($evidence in $dependency.getEvidenceUsed())
                 <evidence>
-                    <source>$esc.html($evidence.getSource())</source>
-                    <name>$esc.html($evidence.getName())</name>
-                    <value>$esc.html($evidence.getValue().trim())</value>
+                    <source>$esc.xml($evidence.getSource())</source>
+                    <name>$esc.xml($evidence.getName())</name>
+                    <value>$esc.xml($evidence.getValue().trim())</value>
                 </evidence>
 #end
             </evidenceCollected>
 #if($dependency.getIdentifiers().size()>0)
             <identifiers>
 #foreach($id in $dependency.getIdentifiers())
-                <identifier type="$esc.html($id.type)">
-                    <name>$esc.html($id.value)</name>
+                <identifier type="$esc.xml($id.type)">
+                    <name>($id.value)</name>
 #if( $id.url )
-                    <url>$esc.html($id.url)</url>
+                    <url>$esc.xml($id.url)</url>
 #end
 #if( $id.description )
-                    <description>$esc.html($id.description)</description>
+                    <description>$esc.xml($id.description)</description>
 #end
                 </identifier>
 #end
@@ -105,7 +105,7 @@ Copyright (c) 2012 Jeremy Long. All Rights Reserved.
             <vulnerabilities>
 #foreach($vuln in $dependency.getVulnerabilities())
                 <vulnerability>
-                    <name>$esc.html($vuln.name)</name>
+                    <name>$esc.xml($vuln.name)</name>
                     <cvssScore>$vuln.cvssScore</cvssScore>
 #if ($vuln.cvssScore<4.0)
                     <severity>Low</severity>
@@ -115,21 +115,21 @@ Copyright (c) 2012 Jeremy Long. All Rights Reserved.
                     <severity>Medium</severity>
 #end
 #if ($vuln.cwe)
-                    <cwe>$esc.html($vuln.cwe)</cwe>
+                    <cwe>$esc.xml($vuln.cwe)</cwe>
 #end
-                    <description>$esc.html($vuln.description)</description>
+                    <description>$esc.xml($vuln.description)</description>
                     <references>
 #foreach($ref in $vuln.getReferences())
                         <reference>
-                            <source>$esc.html($ref.source)</source>
-                            <url>$esc.html($ref.url)</url>
-                            <name>$esc.html($ref.name)</name>
+                            <source>$esc.xml($ref.source)</source>
+                            <url>$esc.xml($ref.url)</url>
+                            <name>$esc.xml($ref.name)</name>
                         </reference>
 #end
                     </references>
                     <vulnerableSoftware>
 #foreach($vs in $vuln.getVulnerableSoftware())
-                        <software#if($vs.hasPreviousVersion()) allPreviousVersion="true"#end>$esc.html($vs.name)</software>
+                        <software#if($vs.hasPreviousVersion()) allPreviousVersion="true"#end>$esc.xml($vs.name)</software>
 #end
                     </vulnerableSoftware>
                 </vulnerability>

dependency-check-core/src/site/markdown/index.md

@@ -13,4 +13,4 @@ The engine is currently exposed via:
 - [Command Line Tool](../dependency-check-cli/installation.html)
 - [Maven Plugin](../dependency-check-maven/usage.html)
 - [Ant Task](../dependency-check-ant/installation.html)
-- Jenkins Plugin
+- [Jenkins Plugin](../dependency-check-jenkins/index.html)

dependency-check-core/src/test/java/org/owasp/dependencycheck/EngineIntegrationTest.java

@@ -44,7 +44,6 @@ public class EngineIntegrationTest {
     @Before
     public void setUp() throws Exception {
         org.owasp.dependencycheck.data.nvdcve.BaseDBTestCase.ensureDBExists();
-        org.owasp.dependencycheck.data.cpe.BaseIndexTestCase.ensureIndexExists();
     }
 
     @After
@@ -58,13 +57,13 @@ public class EngineIntegrationTest {
      */
     @Test
     public void testScan() throws Exception {
-        String path = "target/test-classes";
+        String testClasses = "target/test-classes";
         Engine instance = new Engine();
-        instance.scan(path);
+        instance.scan(testClasses);
         assertTrue(instance.getDependencies().size() > 0);
         instance.analyzeDependencies();
         ReportGenerator rg = new ReportGenerator("DependencyCheck",
                 instance.getDependencies(), instance.getAnalyzers());
-        rg.generateReports("./target/", "HTML");
+        rg.generateReports("./target/", "ALL");
     }
 }

dependency-check-core/src/test/java/org/owasp/dependencycheck/analyzer/ArchiveAnalyzerTest.java

@@ -28,6 +28,7 @@ import org.junit.BeforeClass;
 import org.junit.Test;
 import static org.junit.Assert.*;
 import org.owasp.dependencycheck.Engine;
+import org.owasp.dependencycheck.data.cpe.BaseIndexTestCase;
 import org.owasp.dependencycheck.dependency.Dependency;
 import org.owasp.dependencycheck.utils.Settings;
 
@@ -35,7 +36,7 @@ import org.owasp.dependencycheck.utils.Settings;
  *
  * @author Jeremy Long (jeremy.long@owasp.org)
  */
-public class ArchiveAnalyzerTest {
+public class ArchiveAnalyzerTest extends BaseIndexTestCase {
 
     public ArchiveAnalyzerTest() {
     }
@@ -49,11 +50,13 @@ public class ArchiveAnalyzerTest {
     }
 
     @Before
-    public void setUp() {
+    public void setUp() throws Exception {
+        super.setUp();
     }
 
     @After
-    public void tearDown() {
+    public void tearDown() throws Exception {
+        super.tearDown();
     }
 
     /**
@@ -66,6 +69,9 @@ public class ArchiveAnalyzerTest {
         expResult.add("zip");
         expResult.add("war");
         expResult.add("ear");
+        expResult.add("tar");
+        expResult.add("gz");
+        expResult.add("tgz");
         Set result = instance.getSupportedExtensions();
         assertEquals(expResult, result);
     }
@@ -86,7 +92,7 @@ public class ArchiveAnalyzerTest {
      */
     @Test
     public void testSupportsExtension() {
-        String extension = "tar"; //not supported
+        String extension = "7z"; //not supported
         ArchiveAnalyzer instance = new ArchiveAnalyzer();
         boolean expResult = false;
         boolean result = instance.supportsExtension(extension);
@@ -139,7 +145,7 @@ public class ArchiveAnalyzerTest {
         try {
             instance.initialize();
 
-            File file = new File(this.getClass().getClassLoader().getResource("opensso.war").getPath());
+            File file = new File(this.getClass().getClassLoader().getResource("daytrader-ear-2.1.7.ear").getPath());
             Dependency dependency = new Dependency(file);
             Settings.setBoolean(Settings.KEYS.AUTO_UPDATE, false);
             Engine engine = new Engine();
@@ -154,4 +160,109 @@ public class ArchiveAnalyzerTest {
             instance.close();
         }
     }
+
+    /**
+     * Test of analyze method, of class ArchiveAnalyzer.
+     */
+    @Test
+    public void testAnalyzeTar() throws Exception {
+        ArchiveAnalyzer instance = new ArchiveAnalyzer();
+        try {
+            instance.initialize();
+
+            File file = new File(this.getClass().getClassLoader().getResource("file.tar").getPath());
+            Dependency dependency = new Dependency(file);
+            Settings.setBoolean(Settings.KEYS.AUTO_UPDATE, false);
+            Engine engine = new Engine();
+
+            int initial_size = engine.getDependencies().size();
+            instance.analyze(dependency, engine);
+            int ending_size = engine.getDependencies().size();
+
+            assertTrue(initial_size < ending_size);
+
+        } finally {
+            instance.close();
+        }
+    }
+
+    /**
+     * Test of analyze method, of class ArchiveAnalyzer.
+     */
+    @Test
+    public void testAnalyzeTarGz() throws Exception {
+        ArchiveAnalyzer instance = new ArchiveAnalyzer();
+        try {
+            instance.initialize();
+
+            File file = new File(this.getClass().getClassLoader().getResource("file.tar.gz").getPath());
+            //Dependency dependency = new Dependency(file);
+            Settings.setBoolean(Settings.KEYS.AUTO_UPDATE, false);
+            Engine engine = new Engine();
+
+            int initial_size = engine.getDependencies().size();
+            //instance.analyze(dependency, engine);
+            engine.scan(file);
+            engine.analyzeDependencies();
+            int ending_size = engine.getDependencies().size();
+
+            assertTrue(initial_size < ending_size);
+
+        } finally {
+            instance.close();
+        }
+    }
+
+    /**
+     * Test of analyze method, of class ArchiveAnalyzer.
+     */
+    @Test
+    public void testAnalyzeTgz() throws Exception {
+        ArchiveAnalyzer instance = new ArchiveAnalyzer();
+        try {
+            instance.initialize();
+
+            File file = new File(this.getClass().getClassLoader().getResource("file.tgz").getPath());
+            Settings.setBoolean(Settings.KEYS.AUTO_UPDATE, false);
+            Engine engine = new Engine();
+
+            int initial_size = engine.getDependencies().size();
+            engine.scan(file);
+            engine.analyzeDependencies();
+            int ending_size = engine.getDependencies().size();
+
+            assertTrue(initial_size < ending_size);
+
+        } finally {
+            instance.close();
+        }
+    }
+
+    /**
+     * Test of analyze method, of class ArchiveAnalyzer.
+     */
+    @Test
+    public void testAnalyze_badZip() throws Exception {
+        ArchiveAnalyzer instance = new ArchiveAnalyzer();
+        try {
+            instance.initialize();
+
+            File file = new File(this.getClass().getClassLoader().getResource("test.zip").getPath());
+            Dependency dependency = new Dependency(file);
+            Settings.setBoolean(Settings.KEYS.AUTO_UPDATE, false);
+            Engine engine = new Engine();
+            int initial_size = engine.getDependencies().size();
+//            boolean failed = false;
+//            try {
+            instance.analyze(dependency, engine);
+//            } catch (java.lang.UnsupportedClassVersionError ex) {
+//                failed = true;
+//            }
+//            assertTrue(failed);
+            int ending_size = engine.getDependencies().size();
+            assertEquals(initial_size, ending_size);
+        } finally {
+            instance.close();
+        }
+    }
 }

dependency-check-core/src/test/java/org/owasp/dependencycheck/analyzer/CPEAnalyzerTest.java

@@ -16,10 +16,10 @@
  *
  * Copyright (c) 2012 Jeremy Long. All Rights Reserved.
  */
-package org.owasp.dependencycheck.data.cpe;
+package org.owasp.dependencycheck.analyzer;
 
 import org.owasp.dependencycheck.data.cpe.IndexEntry;
-import org.owasp.dependencycheck.data.cpe.CPEAnalyzer;
+import org.owasp.dependencycheck.analyzer.CPEAnalyzer;
 import java.io.File;
 import java.io.IOException;
 import java.util.HashSet;
@@ -30,15 +30,12 @@ import org.apache.lucene.queryparser.classic.ParseException;
 import org.junit.After;
 import org.junit.AfterClass;
 import org.owasp.dependencycheck.dependency.Dependency;
-import org.owasp.dependencycheck.analyzer.JarAnalyzer;
 import org.junit.Assert;
 import org.junit.Before;
 import org.junit.BeforeClass;
 import org.junit.Test;
-import org.owasp.dependencycheck.analyzer.FalsePositiveAnalyzer;
-import org.owasp.dependencycheck.analyzer.FileNameAnalyzer;
-import org.owasp.dependencycheck.analyzer.HintAnalyzer;
-import static org.owasp.dependencycheck.data.cpe.BaseIndexTestCase.ensureIndexExists;
+import org.owasp.dependencycheck.data.cpe.BaseIndexTestCase;
+import org.owasp.dependencycheck.data.cpe.IndexEntry;
 import org.owasp.dependencycheck.dependency.Identifier;
 
 /**
@@ -56,11 +53,13 @@ public class CPEAnalyzerTest extends BaseIndexTestCase {
     }
 
     @Before
+    @Override
     public void setUp() throws Exception {
         super.setUp();
     }
 
     @After
+    @Override
     public void tearDown() throws Exception {
         super.tearDown();
     }
@@ -124,6 +123,7 @@ public class CPEAnalyzerTest extends BaseIndexTestCase {
      */
     @Test
     public void testDetermineCPE_full() throws Exception {
+        callDetermineCPE_full("hazelcast-2.5.jar", null);
         callDetermineCPE_full("spring-context-support-2.5.5.jar", "cpe:/a:vmware:springsource_spring_framework:2.5.5");
         callDetermineCPE_full("spring-core-3.0.0.RELEASE.jar", "cpe:/a:vmware:springsource_spring_framework:3.0.0");
         callDetermineCPE_full("org.mortbay.jetty.jar", "cpe:/a:mortbay_jetty:jetty:4.2");
@@ -139,6 +139,7 @@ public class CPEAnalyzerTest extends BaseIndexTestCase {
     public void callDetermineCPE_full(String depName, String expResult) throws Exception {
 
         File file = new File(this.getClass().getClassLoader().getResource(depName).getPath());
+
         Dependency dep = new Dependency(file);
 
         FileNameAnalyzer fnAnalyzer = new FileNameAnalyzer();
@@ -163,8 +164,10 @@ public class CPEAnalyzerTest extends BaseIndexTestCase {
         if (expResult != null) {
             Identifier expIdentifier = new Identifier("cpe", expResult, expResult);
             Assert.assertTrue("Incorrect match: { dep:'" + dep.getFileName() + "' }", dep.getIdentifiers().contains(expIdentifier));
-        } else {
+        } else if (dep.getIdentifiers().isEmpty()) {
             Assert.assertTrue("Match found when an Identifier should not have been found: { dep:'" + dep.getFileName() + "' }", dep.getIdentifiers().isEmpty());
+        } else {
+            Assert.assertTrue("Match found when an Identifier should not have been found: { dep:'" + dep.getFileName() + "', identifier:'" + dep.getIdentifiers().iterator().next().getValue() + "' }", dep.getIdentifiers().isEmpty());
         }
     }
 

dependency-check-core/src/test/java/org/owasp/dependencycheck/analyzer/JarAnalyzerTest.java

@@ -118,6 +118,7 @@ public class JarAnalyzerTest {
         JarAnalyzer instance = new JarAnalyzer();
         Set expResult = new HashSet();
         expResult.add("jar");
+        expResult.add("war");
         Set result = instance.getSupportedExtensions();
         assertEquals(expResult, result);
     }

dependency-check-core/src/test/java/org/owasp/dependencycheck/concurrency/DirectorySpinLockTest.java

@@ -0,0 +1,116 @@
+/*
+ * This file is part of dependency-check-core.
+ *
+ * Dependency-check-core is free software: you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation, either version 3 of the License, or (at your option) any
+ * later version.
+ *
+ * Dependency-check-core is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+ * details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * dependency-check-core. If not, see http://www.gnu.org/licenses/.
+ *
+ * Copyright (c) 2013 Jeremy Long. All Rights Reserved.
+ */
+package org.owasp.dependencycheck.concurrency;
+
+import java.io.File;
+import java.net.URL;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import static org.junit.Assert.*;
+
+/**
+ *
+ * @author Jeremy Long (jeremy.long@owasp.org)
+ */
+public class DirectorySpinLockTest {
+
+    public DirectorySpinLockTest() {
+    }
+
+    @BeforeClass
+    public static void setUpClass() {
+    }
+
+    @AfterClass
+    public static void tearDownClass() {
+    }
+
+    @Before
+    public void setUp() {
+    }
+
+    @After
+    public void tearDown() {
+    }
+
+    /**
+     * Test of obtainSharedLock method, of class DirectorySpinLock.
+     * Specifically, this test uses the SpinLockTask to obtain an exclusive lock
+     * that is held for 5 seconds. We then try to obtain a shared lock while
+     * that task is running. It should take longer then 5 seconds to obtain the
+     * shared lock.
+     */
+    @Test
+    public void testObtainSharedLock_withContention() throws Exception {
+        URL location = this.getClass().getProtectionDomain().getCodeSource().getLocation();
+        File directory = new File(location.getFile());
+        DirectorySpinLock instance = new DirectorySpinLock(directory);
+        SpinLockTask task = new SpinLockTask(directory, 5000, false, 2);
+        long start = System.currentTimeMillis();
+        task.run();
+        instance.obtainSharedLock();
+        long end = System.currentTimeMillis();
+        instance.close();
+        if (task.getException() != null) {
+            throw task.getException();
+        }
+        long timeElapsed = end - start;
+        assertTrue("no lock contention occured?", timeElapsed >= 5000);
+        //no exceptions means everything worked.
+    }
+
+    /**
+     * Test of obtainSharedLock method, of class DirectorySpinLock. This method
+     * obtains two shared locks by using the SpinLockTask to obtain a lock in
+     * another thread.
+     */
+    @Test
+    public void testObtainSharedLock() throws Exception {
+        URL location = this.getClass().getProtectionDomain().getCodeSource().getLocation();
+        File directory = new File(location.getFile());
+        DirectorySpinLock instance = new DirectorySpinLock(directory);
+        SpinLockTask task = new SpinLockTask(directory, 1000, true, 2);
+        task.run();
+        instance.obtainSharedLock();
+        instance.close();
+        if (task.getException() != null) {
+            throw task.getException();
+        }
+        //no exceptions means everything worked.
+    }
+
+    /**
+     * Test of obtainExclusiveLock method, of class DirectorySpinLock.
+     */
+    @Test
+    public void testObtainExclusiveLock() throws Exception {
+        URL location = this.getClass().getProtectionDomain().getCodeSource().getLocation();
+        File directory = new File(location.getFile());
+        DirectorySpinLock instance = new DirectorySpinLock(directory);
+        SpinLockTask task = new SpinLockTask(directory, 1000, true, 1);
+        instance.obtainExclusiveLock();
+        task.run();
+        instance.close();
+        assertNotNull("No exception thrown due to exclusive lock failure?", task.getException());
+        assertEquals("Incorrect exception when obtaining exclusive lock", "Unable to obtain lock", task.getException().getMessage());
+    }
+}

dependency-check-core/src/test/java/org/owasp/dependencycheck/concurrency/SpinLockTask.java

@@ -0,0 +1,84 @@
+/*
+ * This file is part of dependency-check-core.
+ *
+ * Dependency-check-core is free software: you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation, either version 3 of the License, or (at your option) any
+ * later version.
+ *
+ * Dependency-check-core is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+ * details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * dependency-check-core. If not, see http://www.gnu.org/licenses/.
+ *
+ * Copyright (c) 2013 Jeremy Long. All Rights Reserved.
+ */
+package org.owasp.dependencycheck.concurrency;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+/**
+ * A simple task that obtains a lock on a directory. This is used in testing of
+ * the shared and exclusive locks.
+ *
+ * @author Jeremy Long (jeremy.long@owasp.org)
+ */
+public class SpinLockTask implements Runnable {
+
+    DirectorySpinLock lock = null;
+    int holdLockFor;
+    long maxWait;
+    boolean shared;
+    private Exception exception = null;
+
+    /**
+     * Get the value of exception
+     *
+     * @return the value of exception
+     */
+    public Exception getException() {
+        return exception;
+    }
+
+    /**
+     * Set the value of exception
+     *
+     * @param exception new value of exception
+     */
+    public void setException(Exception exception) {
+        this.exception = exception;
+    }
+
+    public SpinLockTask(File directory, int holdLockFor, boolean shared, long maxWait) throws InvalidDirectoryException, DirectoryLockException {
+        this.holdLockFor = holdLockFor;
+        this.shared = shared;
+        this.maxWait = maxWait;
+        lock = new DirectorySpinLock(directory);
+    }
+
+    @Override
+    public void run() {
+        try {
+            lock.obtainLock(shared, maxWait);
+            Thread.sleep(holdLockFor);
+        } catch (DirectoryLockException ex) {
+            exception = ex;
+        } catch (InterruptedException ex) {
+            exception = ex;
+        } finally {
+            if (lock != null) {
+                try {
+                    lock.close();
+                } catch (IOException ex) {
+                    exception = ex;
+                }
+            }
+        }
+    }
+}

dependency-check-core/src/test/java/org/owasp/dependencycheck/data/cpe/BaseIndexTest.java

@@ -0,0 +1,63 @@
+/*
+ * This file is part of dependency-check-core.
+ *
+ * Dependency-check-core is free software: you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation, either version 3 of the License, or (at your option) any
+ * later version.
+ *
+ * Dependency-check-core is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+ * details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * dependency-check-core. If not, see http://www.gnu.org/licenses/.
+ *
+ * Copyright (c) 2012 Jeremy Long. All Rights Reserved.
+ */
+package org.owasp.dependencycheck.data.cpe;
+
+import org.owasp.dependencycheck.data.cpe.BaseIndex;
+import java.io.File;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import static org.junit.Assert.*;
+
+/**
+ *
+ * @author Jeremy Long (jeremy.long@owasp.org)
+ */
+public class BaseIndexTest {
+
+    @BeforeClass
+    public static void setUpClass() throws Exception {
+    }
+
+    @AfterClass
+    public static void tearDownClass() throws Exception {
+    }
+
+    @Before
+    public void setUp() {
+    }
+
+    @After
+    public void tearDown() {
+    }
+
+    /**
+     * Test of getDataDirectory method, of class BaseIndex.
+     *
+     * @throws Exception
+     */
+    @Test
+    public void testGetDataDirectory() throws Exception {
+        String file = BaseIndex.getDataDirectory().getPath();
+        String exp = File.separatorChar + "target" + File.separatorChar + "data" + File.separatorChar + "cpe";
+        assertTrue(file.contains(exp));
+    }
+}

dependency-check-core/src/test/java/org/owasp/dependencycheck/data/cpe/BaseIndexTestCase.java

@@ -18,30 +18,18 @@
  */
 package org.owasp.dependencycheck.data.cpe;
 
-import java.io.BufferedInputStream;
-import java.io.BufferedOutputStream;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.util.logging.Level;
-import java.util.logging.Logger;
-import java.util.zip.ZipEntry;
-import java.util.zip.ZipInputStream;
+import junit.framework.TestCase;
 import org.junit.After;
 import org.junit.AfterClass;
 import org.junit.Before;
 import org.junit.BeforeClass;
 import org.owasp.dependencycheck.data.nvdcve.BaseDBTestCase;
-import org.owasp.dependencycheck.utils.Settings;
 
 /**
  *
  * @author Jeremy Long (jeremy.long@owasp.org)
  */
-public abstract class BaseIndexTestCase {
-
-    protected static final int BUFFER_SIZE = 2048;
+public abstract class BaseIndexTestCase extends TestCase {
 
     @BeforeClass
     public static void setUpClass() throws Exception {
@@ -52,88 +40,15 @@ public abstract class BaseIndexTestCase {
     }
 
     @Before
+    @Override
     public void setUp() throws Exception {
-        ensureIndexExists();
         BaseDBTestCase.ensureDBExists();
+        super.setUp();
     }
 
     @After
+    @Override
     public void tearDown() throws Exception {
-    }
-
-    protected static File getDataDirectory() throws IOException {
-        final String fileName = Settings.getString(Settings.KEYS.CPE_DATA_DIRECTORY);
-        final String dataDirectory = Settings.getString(Settings.KEYS.DATA_DIRECTORY);
-        return new File(dataDirectory, fileName);
-        //return FileUtils.getDataDirectory(fileName, Index.class);
-    }
-
-    public static void ensureIndexExists() throws Exception {
-        //String indexPath = Settings.getString(Settings.KEYS.CPE_DATA_DIRECTORY);
-        String indexPath = getDataDirectory().getCanonicalPath();
-        java.io.File f = new File(indexPath);
-
-        if (!f.exists() || (f.isDirectory() && f.listFiles().length == 0)) {
-            f.mkdirs();
-            FileInputStream fis = null;
-            ZipInputStream zin = null;
-            try {
-                File path = new File(BaseIndexTestCase.class.getClassLoader().getResource("index.cpe.zip").getPath());
-                fis = new FileInputStream(path);
-                zin = new ZipInputStream(new BufferedInputStream(fis));
-                ZipEntry entry;
-                while ((entry = zin.getNextEntry()) != null) {
-                    if (entry.isDirectory()) {
-                        continue;
-                    }
-                    FileOutputStream fos = null;
-                    BufferedOutputStream dest = null;
-                    try {
-                        File o = new File(indexPath, entry.getName());
-                        o.createNewFile();
-                        fos = new FileOutputStream(o, false);
-                        dest = new BufferedOutputStream(fos, BUFFER_SIZE);
-                        byte data[] = new byte[BUFFER_SIZE];
-                        int count;
-                        while ((count = zin.read(data, 0, BUFFER_SIZE)) != -1) {
-                            dest.write(data, 0, count);
-                        }
-                    } catch (Exception ex) {
-                        Logger.getLogger(BaseIndexTestCase.class.getName()).log(Level.FINEST, null, ex);
-                    } finally {
-                        if (dest != null) {
-                            try {
-                                dest.flush();
-                                dest.close();
-                            } catch (Throwable ex) {
-                                Logger.getLogger(BaseIndexTestCase.class.getName()).log(Level.FINEST, null, ex);
-                            }
-                        }
-                        if (fos != null) {
-                            try {
-                                fos.close();
-                            } catch (Throwable ex) {
-                                Logger.getLogger(BaseIndexTestCase.class.getName()).log(Level.FINEST, null, ex);
-                            }
-                        }
-                    }
-                }
-            } finally {
-                try {
-                    if (zin != null) {
-                        zin.close();
-                    }
-                } catch (Throwable ex) {
-                    Logger.getLogger(BaseIndexTestCase.class.getName()).log(Level.FINEST, null, ex);
-                }
-                try {
-                    if (fis != null) {
-                        fis.close();
-                    }
-                } catch (Throwable ex) {
-                    Logger.getLogger(BaseIndexTestCase.class.getName()).log(Level.FINEST, null, ex);
-                }
-            }
-        }
+        super.tearDown();
     }
 }

dependency-check-core/src/test/java/org/owasp/dependencycheck/data/cpe/IndexEntryTest.java

@@ -19,6 +19,7 @@
 package org.owasp.dependencycheck.data.cpe;
 
 import org.owasp.dependencycheck.data.cpe.IndexEntry;
+import junit.framework.TestCase;
 import org.junit.After;
 import org.junit.AfterClass;
 import org.junit.Before;
@@ -30,7 +31,7 @@ import org.junit.Assert;
  *
  * @author Jeremy Long (jeremy.long@owasp.org)
  */
-public class IndexEntryTest {
+public class IndexEntryTest extends TestCase {
 
     @BeforeClass
     public static void setUpClass() throws Exception {
@@ -41,11 +42,15 @@ public class IndexEntryTest {
     }
 
     @Before
-    public void setUp() {
+    @Override
+    public void setUp() throws Exception {
+        super.setUp();
     }
 
     @After
-    public void tearDown() {
+    @Override
+    public void tearDown() throws Exception {
+        super.tearDown();
     }
 
     /**

dependency-check-core/src/test/java/org/owasp/dependencycheck/data/cpe/IndexTest.java

@@ -1,103 +0,0 @@
-/*
- * This file is part of dependency-check-core.
- *
- * Dependency-check-core is free software: you can redistribute it and/or modify it
- * under the terms of the GNU General Public License as published by the Free
- * Software Foundation, either version 3 of the License, or (at your option) any
- * later version.
- *
- * Dependency-check-core is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
- * details.
- *
- * You should have received a copy of the GNU General Public License along with
- * dependency-check-core. If not, see http://www.gnu.org/licenses/.
- *
- * Copyright (c) 2012 Jeremy Long. All Rights Reserved.
- */
-package org.owasp.dependencycheck.data.cpe;
-
-import org.owasp.dependencycheck.data.cpe.Index;
-import java.io.File;
-import java.io.IOException;
-import java.util.logging.Level;
-import java.util.logging.Logger;
-import org.apache.lucene.document.Document;
-import org.apache.lucene.queryparser.classic.ParseException;
-import org.apache.lucene.search.ScoreDoc;
-import org.apache.lucene.search.TopDocs;
-import org.apache.lucene.store.Directory;
-import org.junit.After;
-import org.junit.AfterClass;
-import org.junit.Before;
-import org.junit.BeforeClass;
-import org.junit.Test;
-import static org.junit.Assert.*;
-
-/**
- *
- * @author Jeremy Long (jeremy.long@owasp.org)
- */
-public class IndexTest {
-
-    @BeforeClass
-    public static void setUpClass() throws Exception {
-    }
-
-    @AfterClass
-    public static void tearDownClass() throws Exception {
-    }
-
-    @Before
-    public void setUp() {
-    }
-
-    @After
-    public void tearDown() {
-    }
-
-    /**
-     * Test of open method, of class Index.
-     */
-    @Test
-    public void testOpen() {
-        Index instance = new Index();
-        try {
-            instance.open();
-            //TODO research why are we getting multiple documents for the same documentId. is the update method not working?
-//            try {
-//                instance.createSearchingAnalyzer();
-//                TopDocs docs = instance.search("product:( project\\-open )", 20);
-//                for (ScoreDoc d : docs.scoreDocs) {
-//                    final Document doc = instance.getDocument(d.doc);
-//                    String vendor = doc.getField(Fields.VENDOR).stringValue();
-//                    String product = doc.getField(Fields.PRODUCT).stringValue();
-//                    System.out.print(d.doc);
-//                    System.out.print(" : ");
-//                    System.out.print(vendor + ":");
-//                    System.out.println(product);
-//                }
-//            } catch (ParseException ex) {
-//                Logger.getLogger(IndexTest.class.getName()).log(Level.SEVERE, null, ex);
-//            }
-        } catch (IOException ex) {
-            assertNull(ex.getMessage(), ex);
-        }
-        instance.close();
-    }
-
-    /**
-     * Test of getDirectory method, of class Index.
-     *
-     * @throws Exception
-     */
-    @Test
-    public void testGetDirectory() throws Exception {
-        Index index = new Index();
-        Directory result = index.getDirectory();
-
-        String exp = File.separatorChar + "target" + File.separatorChar + "data" + File.separatorChar + "cpe";
-        assertTrue(result.toString().contains(exp));
-    }
-}

dependency-check-core/src/test/java/org/owasp/dependencycheck/data/lucene/TokenPairConcatenatingFilterTest.java

@@ -6,12 +6,18 @@ package org.owasp.dependencycheck.data.lucene;
 
 import java.io.IOException;
 import java.io.Reader;
+import java.io.StringReader;
 import org.apache.lucene.analysis.Analyzer;
 import org.apache.lucene.analysis.BaseTokenStreamTestCase;
 import static org.apache.lucene.analysis.BaseTokenStreamTestCase.assertAnalyzesTo;
+import static org.apache.lucene.analysis.BaseTokenStreamTestCase.assertTokenStreamContents;
 import static org.apache.lucene.analysis.BaseTokenStreamTestCase.checkOneTerm;
 import org.apache.lucene.analysis.MockTokenizer;
+import org.apache.lucene.analysis.TokenStream;
 import org.apache.lucene.analysis.Tokenizer;
+import org.apache.lucene.analysis.core.WhitespaceTokenizer;
+import org.apache.lucene.analysis.tokenattributes.TypeAttributeImpl;
+import org.apache.lucene.util.Version;
 import org.junit.After;
 import org.junit.AfterClass;
 import org.junit.Before;
@@ -25,19 +31,6 @@ import static org.junit.Assert.*;
  */
 public class TokenPairConcatenatingFilterTest extends BaseTokenStreamTestCase {
 
-    private Analyzer analyzer;
-
-    public TokenPairConcatenatingFilterTest() {
-        analyzer = new Analyzer() {
-            @Override
-            protected Analyzer.TokenStreamComponents createComponents(String fieldName,
-                    Reader reader) {
-                Tokenizer source = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
-                return new Analyzer.TokenStreamComponents(source, new TokenPairConcatenatingFilter(source));
-            }
-        };
-    }
-
     @BeforeClass
     public static void setUpClass() {
     }
@@ -60,21 +53,25 @@ public class TokenPairConcatenatingFilterTest extends BaseTokenStreamTestCase {
      * test some examples
      */
     public void testExamples() throws IOException {
-        //TODO figure outwhy I am getting "Failed: incrementtoken() called while in wrong state"
-//        String[] expected = new String[3];
-//        expected[0] = "one";
-//        expected[1] = "onetwo";
-//        expected[2] = "two";
-//        checkOneTerm(analyzer, "one", "one");
-//        assertAnalyzesTo(analyzer, "two", new String[]{"onetwo", "two"});
-        //checkOneTerm(analyzer, "two", "onetwo");
-        //checkOneTerm(analyzer, "three", "two");
+        Tokenizer wsTokenizer = new WhitespaceTokenizer(Version.LUCENE_43, new StringReader("one two three"));
+        TokenStream filter = new TokenPairConcatenatingFilter(wsTokenizer);
+        assertTokenStreamContents(filter,
+                new String[]{"one", "onetwo", "two", "twothree", "three"});
     }
 
     /**
      * Test of clear method, of class TokenPairConcatenatingFilter.
      */
     @Test
-    public void testClear() {
+    public void testClear() throws IOException {
+
+        TokenStream ts = new WhitespaceTokenizer(Version.LUCENE_43, new StringReader("one two three"));
+        TokenPairConcatenatingFilter filter = new TokenPairConcatenatingFilter(ts);
+        assertTokenStreamContents(filter, new String[]{"one", "onetwo", "two", "twothree", "three"});
+
+        assertNotNull(filter.getPreviousWord());
+        filter.clear();
+        assertNull(filter.getPreviousWord());
+        assertTrue(filter.getWords().isEmpty());
     }
 }

dependency-check-core/src/test/java/org/owasp/dependencycheck/data/nvdcve/BaseDBTestCase.java

@@ -30,6 +30,7 @@ import java.util.logging.Logger;
 import java.util.zip.ZipEntry;
 import java.util.zip.ZipInputStream;
 import junit.framework.TestCase;
+import org.owasp.dependencycheck.data.update.DataStoreMetaInfo;
 import org.owasp.dependencycheck.utils.Settings;
 
 /**
@@ -46,32 +47,28 @@ public abstract class BaseDBTestCase extends TestCase {
         ensureDBExists();
     }
 
-    protected static File getDataDirectory() throws IOException {
-        final String fileName = Settings.getString(Settings.KEYS.CVE_DATA_DIRECTORY);
-        final String dataDirectory = Settings.getString(Settings.KEYS.DATA_DIRECTORY);
-        return new File(dataDirectory, fileName);
-    }
-
     public static void ensureDBExists() throws Exception {
-        String indexPath = getDataDirectory().getCanonicalPath();
-        java.io.File f = new File(indexPath);
-        if (!f.exists() || (f.isDirectory() && f.listFiles().length == 0)) {
-            f.mkdirs();
+
+        java.io.File dataPath = Settings.getFile(Settings.KEYS.DATA_DIRECTORY);
+        if (!dataPath.exists() || (dataPath.isDirectory() && dataPath.listFiles().length < 3)) {
+            dataPath.mkdirs();
             FileInputStream fis = null;
             ZipInputStream zin = null;
             try {
-                File path = new File(BaseDBTestCase.class.getClassLoader().getResource("db.cve.zip").getPath());
+                File path = new File(BaseDBTestCase.class.getClassLoader().getResource("data.zip").getPath());
                 fis = new FileInputStream(path);
                 zin = new ZipInputStream(new BufferedInputStream(fis));
                 ZipEntry entry;
                 while ((entry = zin.getNextEntry()) != null) {
                     if (entry.isDirectory()) {
+                        final File d = new File(dataPath, entry.getName());
+                        d.mkdir();
                         continue;
                     }
                     FileOutputStream fos = null;
                     BufferedOutputStream dest = null;
                     try {
-                        File o = new File(indexPath, entry.getName());
+                        File o = new File(dataPath, entry.getName());
                         o.createNewFile();
                         fos = new FileOutputStream(o, false);
                         dest = new BufferedOutputStream(fos, BUFFER_SIZE);
@@ -81,7 +78,7 @@ public abstract class BaseDBTestCase extends TestCase {
                             dest.write(data, 0, count);
                         }
                     } catch (Exception ex) {
-                        Logger.getLogger(BaseDBTestCase.class.getName()).log(Level.FINEST, null, ex);
+                        Logger.getLogger(BaseDBTestCase.class.getName()).log(Level.SEVERE, null, ex);
                     } finally {
                         try {
                             if (dest != null) {

dependency-check-core/src/test/java/org/owasp/dependencycheck/data/nvdcve/NvdCve_1_2_HandlerTest.java

@@ -16,9 +16,9 @@
  *
  * Copyright (c) 2012 Jeremy Long. All Rights Reserved.
  */
-package org.owasp.dependencycheck.data.nvdcve.xml;
+package org.owasp.dependencycheck.data.nvdcve;
 
-import org.owasp.dependencycheck.data.nvdcve.xml.NvdCve12Handler;
+import org.owasp.dependencycheck.data.nvdcve.NvdCve12Handler;
 import java.io.File;
 import java.util.List;
 import java.util.Map;

dependency-check-core/src/test/java/org/owasp/dependencycheck/data/nvdcve/NvdCve_2_0_HandlerTest.java

@@ -16,9 +16,9 @@
  *
  * Copyright (c) 2012 Jeremy Long. All Rights Reserved.
  */
-package org.owasp.dependencycheck.data.nvdcve.xml;
+package org.owasp.dependencycheck.data.nvdcve;
 
-import org.owasp.dependencycheck.data.nvdcve.xml.NvdCve20Handler;
+import org.owasp.dependencycheck.data.nvdcve.NvdCve20Handler;
 import java.io.File;
 import javax.xml.parsers.SAXParser;
 import javax.xml.parsers.SAXParserFactory;

dependency-check-core/src/test/java/org/owasp/dependencycheck/data/update/AbstractUpdateTaskTest.java

@@ -0,0 +1,140 @@
+/*
+ * This file is part of dependency-check-core.
+ *
+ * Dependency-check-core is free software: you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation, either version 3 of the License, or (at your option) any
+ * later version.
+ *
+ * Dependency-check-core is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+ * details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * dependency-check-core. If not, see http://www.gnu.org/licenses/.
+ *
+ * Copyright (c) 2013 Jeremy Long. All Rights Reserved.
+ */
+package org.owasp.dependencycheck.data.update;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.MalformedURLException;
+import java.text.DateFormat;
+import java.util.Calendar;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import static org.junit.Assert.*;
+import org.owasp.dependencycheck.data.UpdateException;
+import org.owasp.dependencycheck.utils.DownloadFailedException;
+
+/**
+ *
+ * @author Jeremy Long (jeremy.long@owasp.org)
+ */
+public class AbstractUpdateTaskTest {
+
+    public AbstractUpdateTaskTest() {
+    }
+
+    @BeforeClass
+    public static void setUpClass() {
+    }
+
+    @AfterClass
+    public static void tearDownClass() {
+    }
+
+    @Before
+    public void setUp() {
+    }
+
+    @After
+    public void tearDown() {
+    }
+
+    public AbstractUpdateTask getAbstractUpdateImpl() throws Exception {
+        DataStoreMetaInfo props = new DataStoreMetaInfo();
+        AbstractUpdateTask instance = new AbstractUpdateImpl(props);
+        return instance;
+    }
+
+    /**
+     * Test of setDeleteAndRecreate method, of class AbstractUpdateTask.
+     */
+    @Test
+    public void testSetDeleteAndRecreate() throws Exception {
+        boolean deleteAndRecreate = false;
+        boolean expResult = false;
+        AbstractUpdateTask instance = getAbstractUpdateImpl();
+        instance.setDeleteAndRecreate(deleteAndRecreate);
+        boolean result = instance.shouldDeleteAndRecreate();
+        assertEquals(expResult, result);
+    }
+
+    /**
+     * Test of deleteExistingData method, of class AbstractUpdateTask.
+     */
+    @Test
+    public void testDeleteExistingData() throws Exception {
+        AbstractUpdateTask instance = getAbstractUpdateImpl();
+        Exception result = null;
+        try {
+            instance.deleteExistingData();
+        } catch (IOException ex) {
+            result = ex;
+        }
+        assertNull(result);
+    }
+
+    /**
+     * Test of openDataStores method, of class AbstractUpdateTask.
+     */
+    @Test
+    public void testOpenDataStores() throws Exception {
+        AbstractUpdateTask instance = getAbstractUpdateImpl();
+        instance.openDataStores();
+        instance.closeDataStores();
+    }
+
+    /**
+     * Test of withinRange method, of class AbstractUpdateTask.
+     */
+    @Test
+    public void testWithinRange() throws Exception {
+        Calendar c = Calendar.getInstance();
+
+        long current = c.getTimeInMillis();
+        long lastRun = c.getTimeInMillis() - (3 * (1000 * 60 * 60 * 24));
+        int range = 7; // 7 days
+        AbstractUpdateTask instance = getAbstractUpdateImpl();
+        boolean expResult = true;
+        boolean result = instance.withinRange(lastRun, current, range);
+        assertEquals(expResult, result);
+
+        lastRun = c.getTimeInMillis() - (8 * (1000 * 60 * 60 * 24));
+        expResult = false;
+        result = instance.withinRange(lastRun, current, range);
+        assertEquals(expResult, result);
+    }
+
+    public class AbstractUpdateImpl extends AbstractUpdateTask {
+
+        public AbstractUpdateImpl(DataStoreMetaInfo props) throws Exception {
+            super(props);
+        }
+
+        public Updateable updatesNeeded() throws MalformedURLException, DownloadFailedException, UpdateException {
+            return null;
+        }
+
+        public void update() throws UpdateException {
+        }
+    }
+}

dependency-check-core/src/test/java/org/owasp/dependencycheck/data/update/BatchUpdateTaskTest.java

@@ -0,0 +1,121 @@
+/*
+ * This file is part of dependency-check-core.
+ *
+ * Dependency-check-core is free software: you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation, either version 3 of the License, or (at your option) any
+ * later version.
+ *
+ * Dependency-check-core is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+ * details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * dependency-check-core. If not, see http://www.gnu.org/licenses/.
+ *
+ * Copyright (c) 2013 Jeremy Long. All Rights Reserved.
+ */
+package org.owasp.dependencycheck.data.update;
+
+import java.io.File;
+import java.net.MalformedURLException;
+import org.apache.commons.io.FileUtils;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import static org.junit.Assert.*;
+import org.owasp.dependencycheck.data.UpdateException;
+import org.owasp.dependencycheck.utils.DownloadFailedException;
+import org.owasp.dependencycheck.utils.Settings;
+
+/**
+ *
+ * @author Jeremy Long (jeremy.long@owasp.org)
+ */
+public class BatchUpdateTaskTest {
+
+    public BatchUpdateTaskTest() {
+    }
+
+    @BeforeClass
+    public static void setUpClass() {
+    }
+
+    @AfterClass
+    public static void tearDownClass() {
+    }
+    private String old12;
+    private String old20;
+
+    @Before
+    public void setUp() throws Exception {
+        old12 = Settings.getString(Settings.KEYS.CVE_MODIFIED_12_URL);
+        old20 = Settings.getString(Settings.KEYS.CVE_MODIFIED_20_URL);
+
+        File tmp = Settings.getTempDirectory();
+        if (!tmp.exists()) {
+            tmp.mkdirs();
+        }
+
+        File dest = new File(tmp, "data.zip");
+        File file = new File(this.getClass().getClassLoader().getResource("data.zip").toURI());
+        FileUtils.copyFile(file, dest);
+        String path = "file:///" + dest.getCanonicalPath();
+        Settings.setString(Settings.KEYS.BATCH_UPDATE_URL, path);
+
+        dest = new File(tmp, "nvdcve-2012.xml");
+        file = new File(this.getClass().getClassLoader().getResource("nvdcve-2012.xml").toURI());
+        FileUtils.copyFile(file, dest);
+        path = "file:///" + dest.getCanonicalPath();
+        Settings.setString(Settings.KEYS.CVE_MODIFIED_12_URL, path);
+
+        dest = new File(tmp, "nvdcve-2.0-2012.xml");
+        file = new File(this.getClass().getClassLoader().getResource("nvdcve-2.0-2012.xml").toURI());
+        FileUtils.copyFile(file, dest);
+        path = "file:///" + dest.getCanonicalPath();
+        Settings.setString(Settings.KEYS.CVE_MODIFIED_20_URL, path);
+    }
+
+    @After
+    public void tearDown() {
+        Settings.setString(Settings.KEYS.CVE_MODIFIED_12_URL, old12);
+        Settings.setString(Settings.KEYS.CVE_MODIFIED_20_URL, old20);
+        Settings.setString(Settings.KEYS.BATCH_UPDATE_URL, "");
+    }
+
+    public BatchUpdateTask getBatchUpdateTask() throws MalformedURLException, DownloadFailedException, UpdateException {
+        DataStoreMetaInfo props = new DataStoreMetaInfo();
+        BatchUpdateTask instance = new BatchUpdateTask(props);
+        return instance;
+    }
+
+    /**
+     * Test of setDoBatchUpdate method, of class BatchUpdateTask.
+     */
+    @Test
+    public void testSetDoBatchUpdate() throws DownloadFailedException, MalformedURLException, UpdateException {
+        boolean expected = false;
+        BatchUpdateTask instance = getBatchUpdateTask();
+        instance.setDoBatchUpdate(expected);
+        boolean results = instance.isDoBatchUpdate();
+        assertEquals(results, expected);
+    }
+
+    /**
+     * Test of update method, of class BatchUpdateTask.
+     */
+    @Test
+    public void testUpdate() throws Exception {
+        BatchUpdateTask instance = getBatchUpdateTask();
+
+        //do some setup
+        instance.setDoBatchUpdate(true);
+        instance.deleteExistingData();
+
+        instance.update(); //no exceptions it worked?
+        //todo add some actual asserts to check things.
+    }
+}

dependency-check-core/src/test/java/org/owasp/dependencycheck/data/update/DataStoreMetaInfoTest.java

@@ -0,0 +1,118 @@
+/*
+ * This file is part of dependency-check-core.
+ *
+ * Dependency-check-core is free software: you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation, either version 3 of the License, or (at your option) any
+ * later version.
+ *
+ * Dependency-check-core is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+ * details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * dependency-check-core. If not, see http://www.gnu.org/licenses/.
+ *
+ * Copyright (c) 2013 Jeremy Long. All Rights Reserved.
+ */
+package org.owasp.dependencycheck.data.update;
+
+import java.io.File;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import static org.junit.Assert.*;
+
+/**
+ *
+ * @author Jeremy Long (jeremy.long@owasp.org)
+ */
+public class DataStoreMetaInfoTest {
+
+    public DataStoreMetaInfoTest() {
+    }
+
+    @BeforeClass
+    public static void setUpClass() {
+    }
+
+    @AfterClass
+    public static void tearDownClass() {
+    }
+
+    @Before
+    public void setUp() {
+    }
+
+    @After
+    public void tearDown() {
+    }
+
+    /**
+     * Test of isBatchUpdateMode method, of class DataStoreMetaInfo.
+     */
+    @Test
+    public void testIsBatchUpdateMode() {
+        DataStoreMetaInfo instance = new DataStoreMetaInfo();
+        boolean expResult = false;
+        instance.setBatchUpdateMode(expResult);
+        boolean result = instance.isBatchUpdateMode();
+        assertEquals(expResult, result);
+    }
+
+    /**
+     * Test of isEmpty method, of class DataStoreMetaInfo.
+     */
+    @Test
+    public void testIsEmpty() {
+        DataStoreMetaInfo instance = new DataStoreMetaInfo();
+        boolean expResult = false;
+        boolean result = instance.isEmpty();
+        assertEquals(expResult, result);
+    }
+
+    /**
+     * Test of save method, of class DataStoreMetaInfo.
+     */
+    @Test
+    public void testSave() throws Exception {
+        NvdCveInfo updatedValue = new NvdCveInfo();
+        String key = "test";
+        long expected = 1337;
+        updatedValue.setId(key);
+        updatedValue.setTimestamp(expected);
+        DataStoreMetaInfo instance = new DataStoreMetaInfo();
+        instance.save(updatedValue);
+        //reload the properties
+        instance = new DataStoreMetaInfo();
+        long results = Long.parseLong(instance.getProperty("lastupdated." + key));
+        assertEquals(expected, results);
+
+    }
+
+    /**
+     * Test of getProperty method, of class DataStoreMetaInfo.
+     */
+    @Test
+    public void testGetProperty_String_String() {
+        String key = "doesn't exist";
+        String defaultValue = "default";
+        DataStoreMetaInfo instance = new DataStoreMetaInfo();
+        String expResult = "default";
+        String result = instance.getProperty(key, defaultValue);
+        assertEquals(expResult, result);
+    }
+
+    /**
+     * Test of getPropertiesFile method, of class DataStoreMetaInfo.
+     */
+    @Test
+    public void testGetPropertiesFile() {
+        File result = DataStoreMetaInfo.getPropertiesFile();
+        //wow... rigorous!
+        assertNotNull(result);
+    }
+}

dependency-check-core/src/test/java/org/owasp/dependencycheck/data/update/DatabaseUpdaterIntegrationTest.java

@@ -14,16 +14,16 @@
  * You should have received a copy of the GNU General Public License along with
  * dependency-check-core. If not, see http://www.gnu.org/licenses/.
  *
- * Copyright (c) 2012 Jeremy Long. All Rights Reserved.
+ * Copyright (c) 2013 Jeremy Long. All Rights Reserved.
  */
-package org.owasp.dependencycheck.data.nvdcve.xml;
+package org.owasp.dependencycheck.data.update;
 
-import org.owasp.dependencycheck.data.nvdcve.xml.DatabaseUpdater;
 import org.junit.After;
 import org.junit.AfterClass;
 import org.junit.Before;
 import org.junit.BeforeClass;
 import org.junit.Test;
+import static org.junit.Assert.*;
 
 /**
  *
@@ -35,11 +35,11 @@ public class DatabaseUpdaterIntegrationTest {
     }
 
     @BeforeClass
-    public static void setUpClass() throws Exception {
+    public static void setUpClass() {
     }
 
     @AfterClass
-    public static void tearDownClass() throws Exception {
+    public static void tearDownClass() {
     }
 
     @Before
@@ -52,8 +52,6 @@ public class DatabaseUpdaterIntegrationTest {
 
     /**
      * Test of update method, of class DatabaseUpdater.
-     *
-     * @throws Exception
      */
     @Test
     public void testUpdate() throws Exception {

dependency-check-core/src/test/java/org/owasp/dependencycheck/data/update/NvdCveInfoTest.java

@@ -0,0 +1,113 @@
+/*
+ * This file is part of dependency-check-core.
+ *
+ * Dependency-check-core is free software: you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation, either version 3 of the License, or (at your option) any
+ * later version.
+ *
+ * Dependency-check-core is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+ * details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * dependency-check-core. If not, see http://www.gnu.org/licenses/.
+ *
+ * Copyright (c) 2013 Jeremy Long. All Rights Reserved.
+ */
+package org.owasp.dependencycheck.data.update;
+
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import static org.junit.Assert.*;
+
+/**
+ * Rigorous test of setters/getters.
+ *
+ * @author Jeremy Long (jeremy.long@owasp.org)
+ */
+public class NvdCveInfoTest {
+
+    public NvdCveInfoTest() {
+    }
+
+    @BeforeClass
+    public static void setUpClass() {
+    }
+
+    @AfterClass
+    public static void tearDownClass() {
+    }
+
+    @Before
+    public void setUp() {
+    }
+
+    @After
+    public void tearDown() {
+    }
+
+    /**
+     * Test of setId and getId method, of class NvdCveInfo.
+     */
+    @Test
+    public void testSetGetId() {
+        NvdCveInfo instance = new NvdCveInfo();
+        String expResult = "id";
+        instance.setId(expResult);
+        String result = instance.getId();
+        assertEquals(expResult, result);
+    }
+
+    /**
+     * Test of getUrl method, of class NvdCveInfo.
+     */
+    @Test
+    public void testSetGetUrl() {
+        NvdCveInfo instance = new NvdCveInfo();
+        String expResult = "http://www.someurl.com/something";
+        instance.setUrl(expResult);
+        String result = instance.getUrl();
+        assertEquals(expResult, result);
+    }
+
+    /**
+     * Test of getOldSchemaVersionUrl method, of class NvdCveInfo.
+     */
+    @Test
+    public void testSetGetOldSchemaVersionUrl() {
+        NvdCveInfo instance = new NvdCveInfo();
+        String expResult = "http://www.someurl.com/something";
+        instance.setOldSchemaVersionUrl(expResult);
+        String result = instance.getOldSchemaVersionUrl();
+        assertEquals(expResult, result);
+    }
+
+    /**
+     * Test of getTimestamp method, of class NvdCveInfo.
+     */
+    @Test
+    public void testSetGetTimestamp() {
+        NvdCveInfo instance = new NvdCveInfo();
+        long expResult = 1337L;
+        instance.setTimestamp(expResult);
+        long result = instance.getTimestamp();
+        assertEquals(expResult, result);
+    }
+
+    /**
+     * Test of getNeedsUpdate method, of class NvdCveInfo.
+     */
+    @Test
+    public void testSetGetNeedsUpdate() {
+        NvdCveInfo instance = new NvdCveInfo();
+        boolean expResult = true;
+        instance.setNeedsUpdate(expResult);
+        boolean result = instance.getNeedsUpdate();
+        assertEquals(expResult, result);
+    }
+}

dependency-check-core/src/test/java/org/owasp/dependencycheck/data/update/StandardUpdateTaskIntegrationTest.java

@@ -0,0 +1,81 @@
+/*
+ * This file is part of dependency-check-core.
+ *
+ * Dependency-check-core is free software: you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation, either version 3 of the License, or (at your option) any
+ * later version.
+ *
+ * Dependency-check-core is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+ * details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * dependency-check-core. If not, see http://www.gnu.org/licenses/.
+ *
+ * Copyright (c) 2013 Jeremy Long. All Rights Reserved.
+ */
+package org.owasp.dependencycheck.data.update;
+
+import java.net.MalformedURLException;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import static org.junit.Assert.*;
+import org.owasp.dependencycheck.data.UpdateException;
+import org.owasp.dependencycheck.utils.DownloadFailedException;
+
+/**
+ *
+ * @author Jeremy Long (jeremy.long@owasp.org)
+ */
+public class StandardUpdateTaskIntegrationTest {
+
+    public StandardUpdateTaskIntegrationTest() {
+    }
+
+    @BeforeClass
+    public static void setUpClass() {
+    }
+
+    @AfterClass
+    public static void tearDownClass() {
+    }
+
+    @Before
+    public void setUp() {
+    }
+
+    @After
+    public void tearDown() {
+    }
+
+    public StandardUpdateTask getStandardUpdateTask() throws MalformedURLException, DownloadFailedException, UpdateException {
+        DataStoreMetaInfo props = new DataStoreMetaInfo();
+        StandardUpdateTask instance = new StandardUpdateTask(props);
+        return instance;
+    }
+
+    /**
+     * Test of update method, of class StandardUpdateTask.
+     */
+    @Test
+    public void testUpdate() throws Exception {
+        StandardUpdateTask instance = getStandardUpdateTask();
+        instance.update();
+        //TODO make this an actual test
+    }
+
+    /**
+     * Test of updatesNeeded method, of class StandardUpdateTask.
+     */
+    @Test
+    public void testUpdatesNeeded() throws Exception {
+        StandardUpdateTask instance = getStandardUpdateTask();
+        Updateable result = instance.updatesNeeded();
+        assertNotNull(result);
+    }
+}

dependency-check-core/src/test/java/org/owasp/dependencycheck/data/update/UpdateableTest.java

@@ -0,0 +1,160 @@
+/*
+ * This file is part of dependency-check-core.
+ *
+ * Dependency-check-core is free software: you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation, either version 3 of the License, or (at your option) any
+ * later version.
+ *
+ * Dependency-check-core is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+ * details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * dependency-check-core. If not, see http://www.gnu.org/licenses/.
+ *
+ * Copyright (c) 2013 Jeremy Long. All Rights Reserved.
+ */
+package org.owasp.dependencycheck.data.update;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.MalformedURLException;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import static org.junit.Assert.*;
+import org.owasp.dependencycheck.utils.DownloadFailedException;
+
+/**
+ *
+ * @author Jeremy Long (jeremy.long@owasp.org)
+ */
+public class UpdateableTest {
+
+    public UpdateableTest() {
+    }
+
+    @BeforeClass
+    public static void setUpClass() {
+    }
+
+    @AfterClass
+    public static void tearDownClass() {
+    }
+
+    @Before
+    public void setUp() {
+    }
+
+    @After
+    public void tearDown() {
+    }
+
+    /**
+     * Test of isUpdateNeeded method, of class Updateable.
+     */
+    @Test
+    public void testIsUpdateNeeded() throws MalformedURLException, DownloadFailedException, IOException {
+        String id = "key";
+        //use a local file as this test will load the result and check the timestamp
+        File f = new File("target/test-classes/nvdcve-2.0-2012.xml");
+        String url = "file:///" + f.getCanonicalPath();
+        Updateable instance = new Updateable();
+        instance.add(id, url, url, false);
+
+        boolean expResult = false;
+        boolean result = instance.isUpdateNeeded();
+        assertEquals(expResult, result);
+
+        instance.add("nextId", url, url, true);
+
+        expResult = true;
+        result = instance.isUpdateNeeded();
+        assertEquals(expResult, result);
+    }
+
+    /**
+     * Test of add method, of class Updateable.
+     */
+    @Test
+    public void testAdd_3args() throws Exception {
+        String id = "key";
+        File f = new File("target/test-classes/nvdcve-2.0-2012.xml");
+        //use a local file as this test will load the result and check the timestamp
+        String url = "file:///" + f.getCanonicalPath();
+        Updateable instance = new Updateable();
+        instance.add(id, url, url);
+        NvdCveInfo results = instance.get(id);
+        assertEquals(id, results.getId());
+        assertEquals(url, results.getUrl());
+        assertEquals(url, results.getOldSchemaVersionUrl());
+    }
+
+    /**
+     * Test of add method, of class Updateable.
+     */
+    @Test
+    public void testAdd_4args() throws Exception {
+        String id = "key";
+        //use a local file as this test will load the result and check the timestamp
+        File f = new File("target/test-classes/nvdcve-2.0-2012.xml");
+        String url = "file:///" + f.getCanonicalPath();
+        Updateable instance = new Updateable();
+        instance.add(id, url, url, false);
+
+        boolean expResult = false;
+        boolean result = instance.isUpdateNeeded();
+        assertEquals(expResult, result);
+
+        instance.add("nextId", url, url, false);
+        NvdCveInfo results = instance.get(id);
+
+        assertEquals(id, results.getId());
+        assertEquals(url, results.getUrl());
+        assertEquals(url, results.getOldSchemaVersionUrl());
+
+    }
+
+    /**
+     * Test of clear method, of class Updateable.
+     */
+    @Test
+    public void testClear() throws MalformedURLException, DownloadFailedException, IOException {
+        String id = "key";
+        //use a local file as this test will load the result and check the timestamp
+        File f = new File("target/test-classes/nvdcve-2.0-2012.xml");
+        String url = "file:///" + f.getCanonicalPath();
+        Updateable instance = new Updateable();
+        instance.add(id, url, url, false);
+        assertFalse(instance.getCollection().isEmpty());
+        instance.clear();
+        assertTrue(instance.getCollection().isEmpty());
+    }
+
+    /**
+     * Test of iterator method, of class Updateable.
+     */
+    @Test
+    public void testIterator() throws IOException {
+        //use a local file as this test will load the result and check the timestamp
+        File f = new File("target/test-classes/nvdcve-2.0-2012.xml");
+        String url = "file:///" + f.getCanonicalPath();
+        Updateable instance = new Updateable();
+        instance.add("one", url, url, false);
+        instance.add("two", url, url, false);
+        instance.add("three", url, url, false);
+        int itemsProcessed = 0;
+        for (NvdCveInfo item : instance) {
+            if ("one".equals(item.getId())) {
+                instance.remove();
+            }
+            itemsProcessed += 1;
+        }
+        assertEquals(3, itemsProcessed);
+        assertEquals(2, instance.getCollection().size());
+    }
+}

dependency-check-core/src/test/java/org/owasp/dependencycheck/reporting/ReportGeneratorTest.java

@@ -18,6 +18,7 @@
  */
 package org.owasp.dependencycheck.reporting;
 
+import org.owasp.dependencycheck.Engine;
 import org.owasp.dependencycheck.data.cpe.BaseIndexTestCase;
 import org.junit.After;
 import org.junit.AfterClass;
@@ -25,6 +26,14 @@ import org.junit.Before;
 import org.junit.BeforeClass;
 import org.junit.Test;
 
+import javax.xml.XMLConstants;
+import javax.xml.transform.stream.StreamSource;
+import javax.xml.validation.Schema;
+import javax.xml.validation.SchemaFactory;
+import javax.xml.validation.Validator;
+import java.io.File;
+import java.io.InputStream;
+
 /**
  *
  * @author Jeremy Long (jeremy.long@owasp.org)
@@ -107,4 +116,41 @@ public class ReportGeneratorTest {
 //        instance.generateReport(templateName, writeTo, properties);
         //assertTrue("need to add a real check here", false);
     }
+
+    /**
+     * Generates an XML report containing known vulnerabilities and realistic
+     * data and validates the generated XML document against the XSD.
+     * @throws Exception
+     */
+    @Test
+    public void testGenerateXMLReport() throws Exception {
+        String templateName = "XmlReport";
+
+        File f = new File("target/test-reports");
+        if (!f.exists()) {
+            f.mkdir();
+        }
+        String writeTo = "target/test-reports/Report.xml";
+
+        File struts = new File(this.getClass().getClassLoader().getResource("struts2-core-2.1.2.jar").getPath());
+        File axis = new File(this.getClass().getClassLoader().getResource("axis2-adb-1.4.1.jar").getPath());
+        File jetty = new File(this.getClass().getClassLoader().getResource("org.mortbay.jetty.jar").getPath());
+
+        Engine engine = new Engine();
+        engine.scan(struts);
+        engine.scan(axis);
+        engine.scan(jetty);
+        engine.analyzeDependencies();
+
+        ReportGenerator generator = new ReportGenerator("Test Report", engine.getDependencies(), engine.getAnalyzers());
+        generator.generateReport(templateName, writeTo);
+
+        InputStream xsdStream = ReportGenerator.class.getClassLoader().getResourceAsStream("schema/DependencyCheck.xsd");
+        StreamSource xsdSource = new StreamSource(xsdStream);
+        StreamSource xmlSource =  new StreamSource(new File(writeTo));
+        SchemaFactory sf = SchemaFactory.newInstance(XMLConstants.W3C_XML_SCHEMA_NS_URI);
+        Schema schema = sf.newSchema(xsdSource);
+        Validator validator = schema.newValidator();
+        validator.validate(xmlSource);
+    }
 }

dependency-check-core/src/test/java/org/owasp/dependencycheck/utils/DependencyVersionTest.java

@@ -109,7 +109,6 @@ public class DependencyVersionTest {
      */
     @Test
     public void testHashCode() {
-        System.out.println("hashCode");
         DependencyVersion instance = new DependencyVersion("3.2.1");
         int expResult = 80756;
         int result = instance.hashCode();

dependency-check-core/src/test/java/org/owasp/dependencycheck/utils/DownloaderIntegrationTest.java

@@ -18,6 +18,7 @@
  */
 package org.owasp.dependencycheck.utils;
 
+import java.io.File;
 import org.owasp.dependencycheck.utils.Settings;
 import org.owasp.dependencycheck.utils.Downloader;
 import java.net.URL;
@@ -59,17 +60,12 @@ public class DownloaderIntegrationTest {
     public void testFetchFile() throws Exception {
 
 //        Settings.setString(Settings.KEYS.CONNECTION_TIMEOUT, "1000");
-
 //        Settings.setString(Settings.KEYS.PROXY_PORT, "8080");
 //        Settings.setString(Settings.KEYS.PROXY_URL, "127.0.0.1");
 
-        URL url = new URL(Settings.getString(Settings.KEYS.CPE_URL));
-        String outputPath = "target/downloaded_cpe.xml";
-        Downloader.fetchFile(url, outputPath, true);
-
-        url = new URL(Settings.getString(Settings.KEYS.CVE_MODIFIED_20_URL));
-        outputPath = "target/downloaded_cve.xml";
-        Downloader.fetchFile(url, outputPath, false);
+        URL url = new URL(Settings.getString(Settings.KEYS.CVE_MODIFIED_20_URL));
+        File outputPath = new File("target/downloaded_cve.xml");
+        Downloader.fetchFile(url, outputPath);
 
     }
 

dependency-check-core/src/test/java/org/owasp/dependencycheck/utils/DownloaderTest.java

@@ -0,0 +1,60 @@
+/*
+ * This file is part of dependency-check-core.
+ *
+ * Dependency-check-core is free software: you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation, either version 3 of the License, or (at your option) any
+ * later version.
+ *
+ * Dependency-check-core is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+ * details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * dependency-check-core. If not, see http://www.gnu.org/licenses/.
+ *
+ * Copyright (c) 2012 Jeremy Long. All Rights Reserved.
+ */
+package org.owasp.dependencycheck.utils;
+
+import java.io.File;
+import org.owasp.dependencycheck.utils.Downloader;
+import java.net.URL;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import static org.junit.Assert.*;
+
+/**
+ *
+ * @author Jeremy Long (jeremy.long@owasp.org)
+ */
+public class DownloaderTest {
+
+    @BeforeClass
+    public static void setUpClass() throws Exception {
+    }
+
+    @AfterClass
+    public static void tearDownClass() throws Exception {
+    }
+
+    @Before
+    public void setUp() {
+    }
+
+    @After
+    public void tearDown() {
+    }
+
+    @Test
+    public void testGetLastModified_file() throws Exception {
+        File f = new File("target/test-classes/nvdcve-2.0-2012.xml");
+        URL url = new URL("file:///" + f.getCanonicalPath());
+        long timestamp = Downloader.getLastModified(url);
+        assertTrue("timestamp equal to zero?", timestamp > 0);
+    }
+}

dependency-check-core/src/test/java/org/owasp/dependencycheck/utils/SettingsTest.java

@@ -73,9 +73,9 @@ public class SettingsTest {
         File result = Settings.getFile(key);
         Assert.assertTrue(result.getAbsolutePath().endsWith(expResult));
 
-        key = "an invalid key!!!";
-        result = Settings.getFile(key, expResult);
-        Assert.assertTrue(result.getAbsolutePath().endsWith(expResult));
+        result = Settings.getFile(Settings.KEYS.DATA_DIRECTORY);
+        String path = result.getPath();
+        Assert.assertTrue(path.endsWith("data") || path.endsWith("data" + File.separator));
     }
 
     /**
@@ -163,4 +163,20 @@ public class SettingsTest {
         boolean result = Settings.getBoolean(key);
         Assert.assertEquals(expResult, result);
     }
+
+    /**
+     * Test of removeProperty method, of class Settings.
+     */
+    @Test
+    public void testRemoveProperty() {
+        String key = "SomeKey";
+        String value = "value";
+        String dfault = "default";
+        Settings.setString(key, value);
+        String ret = Settings.getString(key);
+        Assert.assertEquals(value, ret);
+        Settings.removeProperty(key);
+        ret = Settings.getString(key, dfault);
+        Assert.assertEquals(dfault, ret);
+    }
 }

dependency-check-core/src/test/resources/dependencycheck.properties

@@ -26,7 +26,7 @@ cve.url.modified.validfordays=7
 # the path to the modified nvd cve xml file.
 cve.url-1.2.modified=http://nvd.nist.gov/download/nvdcve-modified.xml
 cve.url-2.0.modified=http://static.nvd.nist.gov/feeds/xml/cve/nvdcve-2.0-modified.xml
-cve.startyear=2002
+cve.startyear=2013
 cve.url-2.0.base=http://static.nvd.nist.gov/feeds/xml/cve/nvdcve-2.0-%d.xml
 cve.url-1.2.base=http://nvd.nist.gov/download/nvdcve-%d.xml
 #cve.url-2.0.base=file:///C:/data/xml/nvdcve-2.0-%d.xml

dependency-check-jenkins/README.md

@@ -0,0 +1,33 @@
+Dependency-Check Jenkins Plugin
+==============================
+
+The Dependency-Check Jenkins Plugin features the ability to perform a dependency
+analysis build and later view results post build. The plugin is built using [analysis-core]
+and features many of the same features that Jenkins static analysis plugins offer,
+including thresholds, charts and the ability to view vulnerability information should
+a dependency have one identified.
+
+The main repository is located at [jenkins-cli/dependency-check-jenkins](https://github.com/jenkinsci/dependency-check-jenkins).
+
+The main site for documentation is located at [OWASP Dependency-Check-Jenkins](https://wiki.jenkins-ci.org/display/JENKINS/OWASP+Dependency-Check+Plugin).
+
+More information can be found on the [wiki].
+
+Mailing List
+------------
+
+Subscribe: [dependency-check+subscribe@googlegroups.com](mailto:dependency-check+subscribe@googlegroups.com)
+
+Post: [dependency-check@googlegroups.com](mailto:dependency-check@googlegroups.com)
+
+Copyright & License
+-------------------
+
+Dependency-Check Jenkins Plugin is Copyright (c) 2013 Steve Springett. All Rights Reserved.
+
+Permission to modify and redistribute is granted under the terms of the GPLv3 license. See the [LICENSE.txt] [GPLv3] file for the full license.
+
+  [wiki]: https://github.com/jenkinsci/dependency-check-jenkins/wiki
+  [analysis-core]: http://wiki.jenkins-ci.org/x/CwDgAQ
+  [GPLv3]: https://github.com/jenkinsci/dependency-check-jenkins/blob/master/LICENSE.txt
+  [notices]: https://github.com/jenkinsci/dependency-check-jenkins/blob/master/NOTICES.txt

dependency-check-jenkins/pom.xml

@@ -0,0 +1,94 @@
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+
+    <description>This plug-in can independently execute a Dependency-Check analysis and visualize the results.</description>
+    <url>http://wiki.jenkins-ci.org/display/JENKINS/OWASP+Dependency-Check+Plugin</url>
+    <parent>
+        <groupId>org.owasp</groupId>
+        <artifactId>dependency-check-parent</artifactId>
+        <version>1.0.5</version>
+    </parent>
+
+    <groupId>org.owasp</groupId>
+    <artifactId>dependency-check-jenkins</artifactId>
+    <name>Dependency-Check Jenkins Plugin</name>
+    <packaging>pom</packaging>
+    <inceptionYear>2012</inceptionYear>
+    <organization>
+        <name>OWASP</name>
+        <url>http://www.owasp.org</url>
+    </organization>
+
+    <developers>
+        <developer>
+            <name>Steve Springett</name>
+            <email>steve.springett@owasp.org</email>
+            <organization>OWASP</organization>
+            <organizationUrl>https://www.owasp.org/index.php/OWASP_Dependency_Check</organizationUrl>
+            <roles>
+                <role>architect</role>
+                <role>developer</role>
+            </roles>
+        </developer>
+    </developers>
+    <!-- begin copy from http://minds.coremedia.com/2012/09/11/problem-solved-deploy-multi-module-maven-project-site-as-github-pages/ -->
+    <distributionManagement>
+        <site>
+            <id>github-pages-site</id>
+            <name>Deployment through GitHub's site deployment plugin</name>
+            <url>${basedir}/../target/site/${project.version}/dependency-check-maven</url>
+        </site>
+    </distributionManagement>
+    <!-- end copy -->
+    <scm>
+        <connection>scm:git:git@github.com:jenkinsci/dependency-check-jenkins.git</connection>
+        <url>https://github.com/jenkinsci/dependency-check-jenkins</url>
+        <developerConnection>scm:git:git@github.com:jenkinsci/dependency-check-jenkins.git</developerConnection>
+    </scm>
+    <issueManagement>
+        <system>github</system>
+        <url>https://github.com/jenkinsci/dependency-check-jenkins/issues</url>
+    </issueManagement>
+    <licenses>
+        <license>
+            <name>GNU General Public License version 3</name>
+            <url>http://www.gnu.org/licenses/</url>
+        </license>
+    </licenses>
+    <build>
+        <plugins>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-site-plugin</artifactId>
+                <version>3.3</version>
+                <dependencies>
+                    <dependency>
+                        <groupId>org.apache.maven.doxia</groupId>
+                        <artifactId>doxia-module-markdown</artifactId>
+                        <version>1.4</version>
+                    </dependency>
+                </dependencies>
+                <configuration>
+                    <skipDeploy>true</skipDeploy>
+                    <reportPlugins>
+                        <plugin>
+                            <groupId>org.apache.maven.plugins</groupId>
+                            <artifactId>maven-project-info-reports-plugin</artifactId>
+                            <version>2.6</version>
+                            <reportSets>
+                                <reportSet>
+                                    <reports>
+                                        <report>index</report>
+                                        <report>summary</report>
+                                        <report>license</report>
+                                        <report>help</report>
+                                    </reports>
+                                </reportSet>
+                            </reportSets>
+                        </plugin>
+                    </reportPlugins>
+                </configuration>
+            </plugin>
+        </plugins>
+    </build>
+</project>

dependency-check-jenkins/src/site/markdown/index.md

@@ -0,0 +1,33 @@
+Dependency-Check Jenkins Plugin
+==============================
+
+The Dependency-Check Jenkins Plugin features the ability to perform a dependency
+analysis build and later view results post build. The plugin is built using [analysis-core]
+and features many of the same features that Jenkins static analysis plugins offer,
+including thresholds, charts and the ability to view vulnerability information should
+a dependency have one identified.
+
+The main repository is located at [jenkins-cli/dependency-check-jenkins](https://github.com/jenkinsci/dependency-check-jenkins).
+
+The main site for documentation is located at [OWASP Dependency-Check-Jenkins](https://wiki.jenkins-ci.org/display/JENKINS/OWASP+Dependency-Check+Plugin).
+
+More information can be found on the [wiki].
+
+Mailing List
+------------
+
+Subscribe: [dependency-check+subscribe@googlegroups.com](mailto:dependency-check+subscribe@googlegroups.com)
+
+Post: [dependency-check@googlegroups.com](mailto:dependency-check@googlegroups.com)
+
+Copyright & License
+-------------------
+
+Dependency-Check Jenkins Plugin is Copyright (c) 2013 Steve Springett. All Rights Reserved.
+
+Permission to modify and redistribute is granted under the terms of the GPLv3 license. See the [LICENSE.txt] [GPLv3] file for the full license.
+
+  [wiki]: https://github.com/jenkinsci/dependency-check-jenkins/wiki
+  [analysis-core]: http://wiki.jenkins-ci.org/x/CwDgAQ
+  [GPLv3]: https://github.com/jenkinsci/dependency-check-jenkins/blob/master/LICENSE.txt
+  [notices]: https://github.com/jenkinsci/dependency-check-jenkins/blob/master/NOTICES.txt

dependency-check-jenkins/src/site/site.xml

@@ -0,0 +1,31 @@
+<?xml version="1.0" encoding="ISO-8859-1"?>
+<!--
+This file is part of dependency-check-jenkins.
+
+Dependency-check-jenkins is free software: you can redistribute it and/or modify it
+under the terms of the GNU General Public License as published by the Free
+Software Foundation, either version 3 of the License, or (at your option) any
+later version.
+
+Dependency-check-jenkins is distributed in the hope that it will be useful, but
+WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+details.
+
+You should have received a copy of the GNU General Public License along with
+dependency-check-jenkins. If not, see http://www.gnu.org/licenses/.
+
+Copyright (c) 2013 Jeremy Long. All Rights Reserved.
+-->
+<project name="dependency-check-jenkins">
+    <bannerLeft>
+        <name>dependency-check-jenkins</name>
+    </bannerLeft>
+    <body>
+        <breadcrumbs>
+            <item name="dependency-check" href="../index.html"/>
+        </breadcrumbs>
+        <menu ref="Project Documentation" />
+        <menu ref="reports" />
+    </body>
+</project>

dependency-check-maven/pom.xml

@@ -24,13 +24,13 @@ Copyright (c) 2013 Jeremy Long. All Rights Reserved.
     <parent>
         <groupId>org.owasp</groupId>
         <artifactId>dependency-check-parent</artifactId>
-        <version>1.0.1</version>
+        <version>1.0.5</version>
     </parent>
 
     <artifactId>dependency-check-maven</artifactId>
     <packaging>maven-plugin</packaging>
 
-    <name>dependency-check-maven</name>
+    <name>Dependency-Check Maven Plugin</name>
     <description>Dependency-Check-Maven is a Maven Plugin that attempts to detect publicly disclosed vulnerabilities contained within project dependencies. It does this by determining if there is a Common Platform Enumeration (CPE) identifier for a given dependency. If found, it will generate a report linking to the associated CVE entries.</description>
     <inceptionYear>2013</inceptionYear>
     <!-- begin copy from http://minds.coremedia.com/2012/09/11/problem-solved-deploy-multi-module-maven-project-site-as-github-pages/ -->

dependency-check-maven/src/main/java/org/owasp/dependencycheck/maven/DependencyCheckMojo.java

@@ -34,7 +34,6 @@ import org.apache.maven.plugin.AbstractMojo;
 import org.apache.maven.plugin.MojoExecutionException;
 import org.apache.maven.project.MavenProject;
 import java.util.Set;
-import java.util.logging.LogManager;
 import org.apache.maven.artifact.Artifact;
 import org.apache.maven.plugins.annotations.Component;
 import org.apache.maven.plugins.annotations.LifecyclePhase;
@@ -54,6 +53,7 @@ import org.owasp.dependencycheck.dependency.Reference;
 import org.owasp.dependencycheck.dependency.Vulnerability;
 import org.owasp.dependencycheck.dependency.VulnerableSoftware;
 import org.owasp.dependencycheck.reporting.ReportGenerator;
+import org.owasp.dependencycheck.utils.LogUtils;
 import org.owasp.dependencycheck.utils.Settings;
 
 /**
@@ -90,6 +90,11 @@ public class DependencyCheckMojo extends AbstractMojo implements MavenMultiPageR
      */
     @Parameter(property = "report-name", defaultValue = "dependency-check-report")
     private String reportName;
+    /**
+     * The path to the verbose log
+     */
+    @Parameter(property = "logfile", defaultValue = "")
+    private String logFile;
     /**
      * The name of the report to be displayed in the Maven Generated Reports
      * page
@@ -155,6 +160,18 @@ public class DependencyCheckMojo extends AbstractMojo implements MavenMultiPageR
     @SuppressWarnings("CanBeFinal")
     @Parameter(property = "proxyPort", defaultValue = "", required = false)
     private String proxyPort = null;
+    /**
+     * The Proxy username.
+     */
+    @SuppressWarnings("CanBeFinal")
+    @Parameter(property = "proxyUsername", defaultValue = "", required = false)
+    private String proxyUsername = null;
+    /**
+     * The Proxy password.
+     */
+    @SuppressWarnings("CanBeFinal")
+    @Parameter(property = "proxyPassword", defaultValue = "", required = false)
+    private String proxyPassword = null;
     /**
      * The Connection Timeout.
      */
@@ -163,44 +180,16 @@ public class DependencyCheckMojo extends AbstractMojo implements MavenMultiPageR
     private String connectionTimeout = null;
 
     // </editor-fold>
-    /**
-     * Configures the logger for use by the application.
-     */
-    private static void prepareLogger() {
-        InputStream in = null;
-        try {
-            in = DependencyCheckMojo.class.getClassLoader().getResourceAsStream(LOG_PROPERTIES_FILE);
-            LogManager.getLogManager().reset();
-            LogManager.getLogManager().readConfiguration(in);
-            //TODO add code to disable fine grained log file.
-//            Logger logger = LogManager.getLogManager().getLogger("");
-//            for (Handler h : logger.getHandlers()) {
-//                if (h.getFormatter(). h.toString());
-//            }
-        } catch (IOException ex) {
-            System.err.println(ex.toString());
-            Logger.getLogger(DependencyCheckMojo.class.getName()).log(Level.SEVERE, null, ex);
-        } catch (SecurityException ex) {
-            Logger.getLogger(DependencyCheckMojo.class.getName()).log(Level.SEVERE, null, ex);
-        } finally {
-            if (in != null) {
-                try {
-                    in.close();
-                } catch (Exception ex) {
-                    //noinspection UnusedAssignment
-                    in = null;
-                }
-            }
-        }
-    }
-
     /**
      * Executes the Dependency-Check on the dependent libraries.
      *
      * @return the Engine used to scan the dependencies.
      */
     private Engine executeDependencyCheck() {
-        prepareLogger();
+
+        final InputStream in = DependencyCheckMojo.class.getClassLoader().getResourceAsStream(LOG_PROPERTIES_FILE);
+        LogUtils.prepareLogger(in, logFile);
+
         populateSettings();
         final Engine engine = new Engine();
         final Set<Artifact> artifacts = project.getArtifacts();
@@ -648,6 +637,12 @@ public class DependencyCheckMojo extends AbstractMojo implements MavenMultiPageR
         if (proxyPort != null && !proxyPort.isEmpty()) {
             Settings.setString(Settings.KEYS.PROXY_PORT, proxyPort);
         }
+        if (proxyUsername != null && !proxyUsername.isEmpty()) {
+            Settings.setString(Settings.KEYS.PROXY_USERNAME, proxyUsername);
+        }
+        if (proxyPassword != null && !proxyPassword.isEmpty()) {
+            Settings.setString(Settings.KEYS.PROXY_PASSWORD, proxyPassword);
+        }
         if (connectionTimeout != null && !connectionTimeout.isEmpty()) {
             Settings.setString(Settings.KEYS.CONNECTION_TIMEOUT, connectionTimeout);
         }

dependency-check-maven/src/main/resources/log.properties

@@ -4,7 +4,7 @@ handlers=java.util.logging.ConsoleHandler, java.util.logging.FileHandler
 # FINEST, FINER, FINE, CONFIG, INFO, WARNING and SEVERE.
 
 # Configure the ConsoleHandler.
-java.util.logging.ConsoleHandler.level=WARNING
+java.util.logging.ConsoleHandler.level=INFO
 
 #org.owasp.dependencycheck.data.nvdcve.xml