Updated to version 1.0.2

Former-commit-id: 2bc5dbe4cae0c2da1b3bbae36a435e6847f7d487

dependency-check-ant/README.md

@@ -1,29 +1,25 @@
-Dependency-Check-Ant
-===================
+Dependency-Check Ant Task
+=========
 
-Dependency-Check is a utility that attempts to detect publicly disclosed vulnerabilities contained within project dependencies. It does this by determining if there is a Common Platform Enumeration (CPE) identifier for a given dependency. If found, it will generate a report linking to the associated CVE entries.
+Dependency-Check Ant Task can be used to check the project dependencies for published security vulnerabilities. The checks
+performed are a "best effort" and as such, there could be false positives as well as false negatives. However,
+vulnerabilities in 3rd party components is a well-known problem and is currently documented in the 2013 OWASP
+Top 10 as [A9 - Using Components with Known Vulnerabilities](https://www.owasp.org/index.php/Top_10_2013-A9-Using_Components_with_Known_Vulnerabilities).
 
 Documentation and links to production binary releases can be found on the [github pages](http://jeremylong.github.io/DependencyCheck/dependency-check-ant/installation.html).
 
 Mailing List
 ------------
 
-Subscribe: [dependency-check+subscribe@googlegroups.com] [subscribe]
+Subscribe: [dependency-check+subscribe@googlegroups.com](mailto:dependency-check+subscribe@googlegroups.com)
 
-Post: [dependency-check@googlegroups.com] [post]
+Post: [dependency-check@googlegroups.com](mailto:dependency-check@googlegroups.com)
 
 Copyright & License
--
+-------------------
 
 Dependency-Check is Copyright (c) 2012-2013 Jeremy Long. All Rights Reserved.
 
-Permission to modify and redistribute is granted under the terms of the GPLv3 license. See the [LICENSE.txt] [GPLv3] file for the full license.
+Permission to modify and redistribute is granted under the terms of the GPLv3 license. See the [LICENSE.txt](https://github.com/jeremylong/DependencyCheck/dependency-check-ant/blob/master/LICENSE.txt) file for the full license.
 
-Dependency-Check makes use of several other open source libraries. Please see the [NOTICE.txt] [notices] file for more information.
-
-
-  [wiki]: https://github.com/jeremylong/DependencyCheck/wiki
-  [subscribe]: mailto:dependency-check+subscribe@googlegroups.com
-  [post]: mailto:dependency-check@googlegroups.com
-  [GPLv3]: https://github.com/jeremylong/DependencyCheck/blob/master/LICENSE.txt
-  [notices]: https://github.com/jeremylong/DependencyCheck/blob/master/NOTICES.txt
+Dependency-Check-Ant makes use of other open source libraries. Please see the [NOTICE.txt](https://github.com/jeremylong/DependencyCheck/dependency-check-ant/blob/master/NOTICES.txt) file for more information.

dependency-check-ant/pom.xml

@@ -22,13 +22,13 @@ Copyright (c) 2013 - Jeremy Long. All Rights Reserved.
     <parent>
         <groupId>org.owasp</groupId>
         <artifactId>dependency-check-parent</artifactId>
-        <version>1.0.1</version>
+        <version>1.0.2</version>
     </parent>
 
     <artifactId>dependency-check-ant</artifactId>
     <packaging>jar</packaging>
 
-    <name>dependency-check-ant</name>
+    <name>Dependency-Check Ant Task</name>
     <description>Dependency-check is a utility that attempts to detect publicly disclosed vulnerabilities contained within project dependencies. It does this by determining if there is a Common Platform Enumeration (CPE) identifier for a given dependency. If found, it will generate a report linking to the associated CVE entries.</description>
     <!-- begin copy from http://minds.coremedia.com/2012/09/11/problem-solved-deploy-multi-module-maven-project-site-as-github-pages/ -->
     <distributionManagement>
@@ -76,6 +76,25 @@ Copyright (c) 2013 - Jeremy Long. All Rights Reserved.
                 </configuration>
                 <executions>
                     <!-- the following executions are solely to setup the test environment -->
+                    <execution>
+                        <id>copy-test-data.zip</id>
+                        <phase>validate</phase>
+                        <goals>
+                            <goal>copy-resources</goal>
+                        </goals>
+                        <configuration>
+                            <outputDirectory>${project.build.directory}/test-classes</outputDirectory>
+                            <resources>
+                                <resource>
+                                    <directory>${basedir}/../src/test/resources</directory>
+                                    <filtering>false</filtering>
+                                    <includes>
+                                        <include>data.zip</include>
+                                    </includes>
+                                </resource>
+                            </resources>
+                        </configuration>
+                    </execution>
                     <execution>
                         <id>copy-test-resources-1</id>
                         <phase>validate</phase>
@@ -89,7 +108,7 @@ Copyright (c) 2013 - Jeremy Long. All Rights Reserved.
                                     <directory>${basedir}/../src/test/resources</directory>
                                     <filtering>false</filtering>
                                     <includes>
-                                        <include>axis2*.jar</include>
+                                        <include>org.mortbay.*.jar</include>
                                     </includes>
                                 </resource>
                             </resources>
@@ -127,7 +146,7 @@ Copyright (c) 2013 - Jeremy Long. All Rights Reserved.
                                     <directory>${basedir}/../src/test/resources</directory>
                                     <filtering>false</filtering>
                                     <includes>
-                                        <include>*.war</include>
+                                        <include>struts.jar</include>
                                     </includes>
                                 </resource>
                             </resources>
@@ -146,13 +165,32 @@ Copyright (c) 2013 - Jeremy Long. All Rights Reserved.
                                     <directory>${basedir}/../src/test/resources</directory>
                                     <filtering>false</filtering>
                                     <includes>
-                                        <include>jetty-6.1.0.jar</include>
                                         <include>org.mortbay.jetty.jar</include>
                                     </includes>
                                 </resource>
                             </resources>
                         </configuration>
                     </execution>
+                    <execution>
+                        <id>copy-data</id>
+                        <phase>validate</phase>
+                        <goals>
+                            <goal>copy-resources</goal>
+                        </goals>
+                        <configuration>
+                            <outputDirectory>${project.build.directory}/test-classes</outputDirectory>
+                            <resources>
+                                <resource>
+                                    <directory>${basedir}/../src/test/resources</directory>
+                                    <filtering>false</filtering>
+                                    <includes>
+                                        <include>db.cve.zip</include>
+                                        <include>index.cpe.zip</include>
+                                    </includes>
+                                </resource>
+                            </resources>
+                        </configuration>
+                    </execution>
                 </executions>
             </plugin>
             <plugin>
@@ -238,6 +276,10 @@ Copyright (c) 2013 - Jeremy Long. All Rights Reserved.
                             <value>${project.build.directory}/cobertura/cobertura.ser</value>
                             <workingDirectory>target</workingDirectory>
                         </property>
+                        <property>
+                            <name>data.directory</name>
+                            <value>${project.build.directory}/dependency-check-data</value>
+                        </property>
                     </systemProperties>
                 </configuration>
             </plugin>
@@ -389,6 +431,13 @@ Copyright (c) 2013 - Jeremy Long. All Rights Reserved.
             <artifactId>dependency-check-core</artifactId>
             <version>${project.parent.version}</version>
         </dependency>
+        <dependency>
+            <groupId>org.owasp</groupId>
+            <artifactId>dependency-check-core</artifactId>
+            <version>${project.parent.version}</version>
+            <type>test-jar</type>
+            <scope>test</scope>
+        </dependency>
         <dependency>
             <groupId>org.apache.ant</groupId>
             <artifactId>ant</artifactId>

dependency-check-ant/src/test/java/org/owasp/dependencycheck/taskdefs/DependencyCheckTaskTest.java

@@ -26,6 +26,7 @@ import org.junit.Before;
 import org.junit.BeforeClass;
 import org.junit.Test;
 import org.apache.tools.ant.BuildFileTest;
+import org.owasp.dependencycheck.data.nvdcve.BaseDBTestCase;
 
 /**
  *
@@ -46,7 +47,8 @@ public class DependencyCheckTaskTest extends BuildFileTest {
 
     @Before
     @Override
-    public void setUp() {
+    public void setUp() throws Exception {
+        BaseDBTestCase.ensureDBExists();
         final String buildFile = this.getClass().getClassLoader().getResource("build.xml").getPath();
         configureProject(buildFile);
     }

dependency-check-cli/README.md

@@ -1,29 +1,24 @@
-Dependency-Check
+Dependency-Check Command Line
 ================
-
-Dependency-Check is a utility that attempts to detect publicly disclosed vulnerabilities contained within project dependencies. It does this by determining if there is a Common Platform Enumeration (CPE) identifier for a given dependency. If found, it will generate a report linking to the associated CVE entries.
+Dependency-Check Command Line can be used to check project dependencies for published security vulnerabilities. The checks
+performed are a "best effort" and as such, there could be false positives as well as false negatives. However,
+vulnerabilities in 3rd party components is a well-known problem and is currently documented in the 2013 OWASP
+Top 10 as [A9 - Using Components with Known Vulnerabilities](https://www.owasp.org/index.php/Top_10_2013-A9-Using_Components_with_Known_Vulnerabilities).
 
 Documentation and links to production binary releases can be found on the [github pages](http://jeremylong.github.io/DependencyCheck/dependency-check-cli/installation.html).
 
 Mailing List
 ------------
 
-Subscribe: [dependency-check+subscribe@googlegroups.com] [subscribe]
+Subscribe: [dependency-check+subscribe@googlegroups.com](mailto:dependency-check+subscribe@googlegroups.com)
 
-Post: [dependency-check@googlegroups.com] [post]
+Post: [dependency-check@googlegroups.com](mailto:dependency-check@googlegroups.com)
 
 Copyright & License
 ------------
 
 Dependency-Check is Copyright (c) 2012-2013 Jeremy Long. All Rights Reserved.
 
-Permission to modify and redistribute is granted under the terms of the GPLv3 license. See the [LICENSE.txt] [GPLv3] file for the full license.
+Permission to modify and redistribute is granted under the terms of the GPLv3 license. See the [LICENSE.txt](https://github.com/jeremylong/DependencyCheck/dependency-check-cli/blob/master/LICENSE.txt) file for the full license.
 
-Dependency-Check makes use of several other open source libraries. Please see the [NOTICE.txt] [notices] file for more information.
-
-
-  [wiki]: https://github.com/jeremylong/DependencyCheck/wiki
-  [subscribe]: mailto:dependency-check+subscribe@googlegroups.com
-  [post]: mailto:dependency-check@googlegroups.com
-  [GPLv3]: https://github.com/jeremylong/DependencyCheck/blob/master/LICENSE.txt
-  [notices]: https://github.com/jeremylong/DependencyCheck/blob/master/NOTICES.txt
+Dependency-Check Command Line makes use of other open source libraries. Please see the [NOTICE.txt](https://github.com/jeremylong/DependencyCheck/dependency-check-cli/blob/master/NOTICES.txt) file for more information.

dependency-check-cli/pom.xml

@@ -22,13 +22,13 @@ Copyright (c) 2012 - Jeremy Long. All Rights Reserved.
     <parent>
         <groupId>org.owasp</groupId>
         <artifactId>dependency-check-parent</artifactId>
-        <version>1.0.1</version>
+        <version>1.0.2</version>
     </parent>
 
     <artifactId>dependency-check-cli</artifactId>
     <packaging>jar</packaging>
 
-    <name>dependency-check-cli</name>
+    <name>Dependency-Check Command Line</name>
     <description>Dependency-Check-Maven is a Maven Plugin that attempts to detect publicly disclosed vulnerabilities contained within project dependencies. It does this by determining if there is a Common Platform Enumeration (CPE) identifier for a given dependency. If found, it will generate a report linking to the associated CVE entries.</description>
     <!-- begin copy from http://minds.coremedia.com/2012/09/11/problem-solved-deploy-multi-module-maven-project-site-as-github-pages/ -->
     <distributionManagement>

dependency-check-core/pom.xml

@@ -22,13 +22,13 @@ along with Dependency-Check.  If not, see <http://www.gnu.org/licenses />.
     <parent>
         <groupId>org.owasp</groupId>
         <artifactId>dependency-check-parent</artifactId>
-        <version>1.0.1</version>
+        <version>1.0.2</version>
     </parent>
 
     <artifactId>dependency-check-core</artifactId>
     <packaging>jar</packaging>
 
-    <name>dependency-check-core</name>
+    <name>Dependency-Check Core</name>
     <!-- begin copy from http://minds.coremedia.com/2012/09/11/problem-solved-deploy-multi-module-maven-project-site-as-github-pages/ -->
     <distributionManagement>
         <site>
@@ -83,16 +83,16 @@ along with Dependency-Check.  If not, see <http://www.gnu.org/licenses />.
             <plugin>
                 <groupId>org.apache.maven.plugins</groupId>
                 <artifactId>maven-dependency-plugin</artifactId>
-                <version>2.7</version>
+                <version>2.8</version>
                 <executions>
                     <execution>
-                        <phase>package</phase>
+                        <phase>generate-resources</phase>
                         <goals>
                             <goal>copy-dependencies</goal>
                         </goals>
                         <configuration>
-                            <outputDirectory>${project.build.directory}/lib</outputDirectory>
-                            <excludeScope>provided</excludeScope>
+                            <outputDirectory>${project.build.directory}/test-classes</outputDirectory>
+                            <includeScope>provided</includeScope>
                         </configuration>
                     </execution>
                 </executions>
@@ -101,6 +101,22 @@ along with Dependency-Check.  If not, see <http://www.gnu.org/licenses />.
                 <groupId>org.apache.maven.plugins</groupId>
                 <artifactId>maven-jar-plugin</artifactId>
                 <version>2.4</version>
+                <executions>
+                    <execution>
+                        <id>jar</id>
+                        <phase>package</phase>
+                        <goals>
+                            <goal>jar</goal>
+                        </goals>
+                    </execution>
+                    <execution>
+                        <id>test-jar</id>
+                        <phase>package</phase>
+                        <goals>
+                            <goal>test-jar</goal>
+                        </goals>
+                    </execution>
+                </executions>
                 <configuration>
                     <archive>
                         <manifest>
@@ -177,6 +193,10 @@ along with Dependency-Check.  If not, see <http://www.gnu.org/licenses />.
                             <name>data.directory</name>
                             <value>${project.build.directory}/data</value>
                         </property>
+                        <property>
+                            <name>temp.directory</name>
+                            <value>${project.build.directory}/temp</value>
+                        </property>
                     </systemProperties>
                     <excludes>
                         <exclude>**/*IntegrationTest.java</exclude>
@@ -369,7 +389,7 @@ along with Dependency-Check.  If not, see <http://www.gnu.org/licenses />.
             <groupId>com.google.code.findbugs</groupId>
             <artifactId>annotations</artifactId>
             <version>2.0.1</version>
-            <scope>provided</scope><!-- don't include this in the libs-->
+            <optional>true</optional>
         </dependency>
         <dependency>
             <groupId>commons-cli</groupId>
@@ -461,12 +481,84 @@ along with Dependency-Check.  If not, see <http://www.gnu.org/licenses />.
             <version>1.7.2</version>
             <type>jar</type>
         </dependency>
-        <!-- The following dependencies are only scanned during integration testing -->
-        <!--<dependency>
+        <dependency>
+            <groupId>org.apache.commons</groupId>
+            <artifactId>commons-compress</artifactId>
+            <version>1.5</version>
+        </dependency>
+        <!-- The following dependencies are only used during testing -->
+        <dependency>
             <groupId>org.springframework</groupId>
             <artifactId>spring-webmvc</artifactId>
             <version>2.5.5</version>
-            <scope>test</scope>
-        </dependency>-->
+            <scope>provided</scope>
+            <optional>true</optional>
+        </dependency>
+        <dependency>
+            <groupId>com.hazelcast</groupId>
+            <artifactId>hazelcast</artifactId>
+            <version>2.5</version>
+            <scope>provided</scope>
+            <optional>true</optional>
+        </dependency>
+        <dependency>
+            <groupId>net.sf.ehcache</groupId>
+            <artifactId>ehcache-core</artifactId>
+            <version>2.2.0</version>
+            <scope>provided</scope>
+            <optional>true</optional>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.struts</groupId>
+            <artifactId>struts2-core</artifactId>
+            <version>2.1.2</version>
+            <scope>provided</scope>
+            <optional>true</optional>
+        </dependency>
+        <dependency>
+            <groupId>org.mortbay.jetty</groupId>
+            <artifactId>jetty</artifactId>
+            <version>6.1.0</version>
+            <scope>provided</scope>
+            <optional>true</optional>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.axis2</groupId>
+            <artifactId>axis2-spring</artifactId>
+            <version>1.4.1</version>
+            <scope>provided</scope>
+            <optional>true</optional>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.axis2</groupId>
+            <artifactId>axis2-adb</artifactId>
+            <version>1.4.1</version>
+            <scope>provided</scope>
+            <optional>true</optional>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.geronimo.daytrader</groupId>
+            <artifactId>daytrader-ear</artifactId>
+            <version>2.1.7</version>
+            <type>ear</type>
+            <scope>provided</scope>
+            <optional>true</optional>
+        </dependency>
+        <dependency>
+            <groupId>org.glassfish.main.admingui</groupId>
+            <artifactId>war</artifactId>
+            <version>4.0</version>
+            <type>war</type>
+            <scope>provided</scope>
+            <optional>true</optional>
+        </dependency>
+        <dependency>
+            <groupId>org.dojotoolkit</groupId>
+            <artifactId>dojo-war</artifactId>
+            <version>1.3.0</version>
+            <type>war</type>
+            <scope>provided</scope>
+            <optional>true</optional>
+        </dependency>
     </dependencies>
 </project>

dependency-check-core/src/main/java/org/owasp/dependencycheck/analyzer/ArchiveAnalyzer.java

@@ -32,8 +32,11 @@ import java.util.List;
 import java.util.Set;
 import java.util.logging.Level;
 import java.util.logging.Logger;
-import java.util.zip.ZipEntry;
-import java.util.zip.ZipInputStream;
+//import java.util.zip.ZipEntry;
+//import java.util.zip.ZipException;
+//import java.util.zip.ZipInputStream;
+import org.apache.commons.compress.archivers.zip.ZipArchiveEntry;
+import org.apache.commons.compress.archivers.zip.ZipArchiveInputStream;
 import org.h2.store.fs.FileUtils;
 import org.owasp.dependencycheck.Engine;
 import org.owasp.dependencycheck.dependency.Dependency;
@@ -140,8 +143,10 @@ public class ArchiveAnalyzer extends AbstractAnalyzer implements Analyzer {
      */
     @Override
     public void initialize() throws Exception {
-        final String tmpDir = Settings.getString(Settings.KEYS.TEMP_DIRECTORY, System.getProperty("java.io.tmpdir"));
-        final File baseDir = new File(tmpDir);
+        final File baseDir = Settings.getTempDirectory();
+        if (!baseDir.exists()) {
+            baseDir.mkdirs();
+        }
         tempFileLocation = File.createTempFile("check", "tmp", baseDir);
         if (!tempFileLocation.delete()) {
             throw new AnalysisException("Unable to delete temporary file '" + tempFileLocation.getAbsolutePath() + "'.");
@@ -243,7 +248,8 @@ public class ArchiveAnalyzer extends AbstractAnalyzer implements Analyzer {
         }
 
         FileInputStream fis = null;
-        ZipInputStream zis = null;
+        //ZipInputStream zis = null;
+        ZipArchiveInputStream zis = null;
 
         try {
             fis = new FileInputStream(archive);
@@ -251,10 +257,11 @@ public class ArchiveAnalyzer extends AbstractAnalyzer implements Analyzer {
             Logger.getLogger(ArchiveAnalyzer.class.getName()).log(Level.INFO, null, ex);
             throw new AnalysisException("Archive file was not found.", ex);
         }
-        zis = new ZipInputStream(new BufferedInputStream(fis));
-        ZipEntry entry;
+        zis = new ZipArchiveInputStream(new BufferedInputStream(fis));
+        ZipArchiveEntry entry;
+
         try {
-            while ((entry = zis.getNextEntry()) != null) {
+            while ((entry = zis.getNextZipEntry()) != null) {
                 if (entry.isDirectory()) {
                     final File d = new File(extractTo, entry.getName());
                     if (!d.mkdirs()) {
@@ -295,7 +302,13 @@ public class ArchiveAnalyzer extends AbstractAnalyzer implements Analyzer {
             }
         } catch (IOException ex) {
             final String msg = String.format("Exception reading archive '%s'.", archive.getName());
-            Logger.getLogger(ArchiveAnalyzer.class.getName()).log(Level.FINE, msg, ex);
+            Logger.getLogger(ArchiveAnalyzer.class.getName()).log(Level.WARNING, msg);
+            Logger.getLogger(ArchiveAnalyzer.class.getName()).log(Level.FINE, null, ex);
+            throw new AnalysisException(msg, ex);
+        } catch (Throwable ex) {
+            final String msg = String.format("Exception reading archive '%s'.", archive.getName());
+            Logger.getLogger(ArchiveAnalyzer.class.getName()).log(Level.WARNING, msg);
+            Logger.getLogger(ArchiveAnalyzer.class.getName()).log(Level.WARNING, null, ex);
             throw new AnalysisException(msg, ex);
         } finally {
             try {

dependency-check-core/src/main/java/org/owasp/dependencycheck/analyzer/CPEAnalyzer.java

@@ -16,7 +16,7 @@
  *
  * Copyright (c) 2012 Jeremy Long. All Rights Reserved.
  */
-package org.owasp.dependencycheck.data.cpe;
+package org.owasp.dependencycheck.analyzer;
 
 import java.io.IOException;
 import java.io.UnsupportedEncodingException;
@@ -35,14 +35,14 @@ import org.apache.lucene.queryparser.classic.ParseException;
 import org.apache.lucene.search.ScoreDoc;
 import org.apache.lucene.search.TopDocs;
 import org.owasp.dependencycheck.Engine;
-import org.owasp.dependencycheck.analyzer.AnalysisException;
-import org.owasp.dependencycheck.analyzer.AnalysisPhase;
 import org.owasp.dependencycheck.data.lucene.LuceneUtils;
 import org.owasp.dependencycheck.dependency.Dependency;
 import org.owasp.dependencycheck.dependency.Evidence;
 import org.owasp.dependencycheck.dependency.Evidence.Confidence;
 import org.owasp.dependencycheck.dependency.EvidenceCollection;
-import org.owasp.dependencycheck.analyzer.Analyzer;
+import org.owasp.dependencycheck.data.cpe.CpeIndexReader;
+import org.owasp.dependencycheck.data.cpe.Fields;
+import org.owasp.dependencycheck.data.cpe.IndexEntry;
 import org.owasp.dependencycheck.data.nvdcve.CveDB;
 import org.owasp.dependencycheck.data.nvdcve.DatabaseException;
 import org.owasp.dependencycheck.dependency.Identifier;
@@ -83,9 +83,9 @@ public class CPEAnalyzer implements Analyzer {
      */
     static final int STRING_BUILDER_BUFFER = 20;
     /**
-     * The CPE Index.
+     * The CPE Index Reader.
      */
-    private Index cpe;
+    private CpeIndexReader cpe;
     /**
      * The CVE Database.
      */
@@ -100,7 +100,7 @@ public class CPEAnalyzer implements Analyzer {
      * usually occurs when the database is in use by another process.
      */
     public void open() throws IOException, DatabaseException {
-        cpe = new Index();
+        cpe = new CpeIndexReader();
         cpe.open();
         cve = new CveDB();
         try {
@@ -119,8 +119,12 @@ public class CPEAnalyzer implements Analyzer {
      */
     @Override
     public void close() {
-        cpe.close();
-        cve.close();
+        if (cpe != null) {
+            cpe.close();
+        }
+        if (cve != null) {
+            cve.close();
+        }
     }
 
     /**
@@ -162,7 +166,6 @@ public class CPEAnalyzer implements Analyzer {
         String vendors = addEvidenceWithoutDuplicateTerms("", dependency.getVendorEvidence(), vendorConf);
         String products = addEvidenceWithoutDuplicateTerms("", dependency.getProductEvidence(), productConf);
 
-        //boolean found = false;
         int ctr = 0;
         do {
             if (!vendors.isEmpty() && !products.isEmpty()) {
@@ -171,27 +174,20 @@ public class CPEAnalyzer implements Analyzer {
 
                 for (IndexEntry e : entries) {
                     if (verifyEntry(e, dependency)) {
-                        //found = true; // we found a vendor/product pair. Now find version from the cve db.
                         final String vendor = e.getVendor();
                         final String product = e.getProduct();
-                        // cve.getVersions(vendor, product);
                         determineIdentifiers(dependency, vendor, product);
                     }
                 }
             }
-            //if (!found) {
             vendorConf = reduceConfidence(vendorConf);
             if (dependency.getVendorEvidence().contains(vendorConf)) {
-                //vendors += " " + dependency.getVendorEvidence().toString(vendorConf);
                 vendors = addEvidenceWithoutDuplicateTerms(vendors, dependency.getVendorEvidence(), vendorConf);
             }
             productConf = reduceConfidence(productConf);
             if (dependency.getProductEvidence().contains(productConf)) {
-                //products += " " + dependency.getProductEvidence().toString(productConf);
                 products = addEvidenceWithoutDuplicateTerms(products, dependency.getProductEvidence(), productConf);
             }
-            //}
-            //} while (!found && (++ctr) < 4);
         } while ((++ctr) < 4);
     }
 
@@ -275,13 +271,22 @@ public class CPEAnalyzer implements Analyzer {
 
         final TopDocs docs = cpe.search(searchString, MAX_QUERY_RESULTS);
         for (ScoreDoc d : docs.scoreDocs) {
-            final Document doc = cpe.getDocument(d.doc);
-            final IndexEntry entry = new IndexEntry();
-            entry.setVendor(doc.get(Fields.VENDOR));
-            entry.setProduct(doc.get(Fields.PRODUCT));
-            entry.setSearchScore(d.score);
-            if (!ret.contains(entry)) {
-                ret.add(entry);
+            if (d.score >= 0.08) {
+                final Document doc = cpe.getDocument(d.doc);
+                final IndexEntry entry = new IndexEntry();
+                entry.setVendor(doc.get(Fields.VENDOR));
+                entry.setProduct(doc.get(Fields.PRODUCT));
+//                if (d.score < 0.08) {
+//                    System.out.print(entry.getVendor());
+//                    System.out.print(":");
+//                    System.out.print(entry.getProduct());
+//                    System.out.print(":");
+//                    System.out.println(d.score);
+//                }
+                entry.setSearchScore(d.score);
+                if (!ret.contains(entry)) {
+                    ret.add(entry);
+                }
             }
         }
         return ret;
@@ -454,9 +459,10 @@ public class CPEAnalyzer implements Analyzer {
                 list.add(word);
             }
         }
-//        if (tempWord != null) {
-//            //for now ignore any last single letter words...
-//        }
+        if (tempWord != null && !list.isEmpty()) {
+            String tmp = list.get(list.size() - 1) + tempWord;
+            list.add(tmp);
+        }
         boolean contains = true;
         for (String word : list) {
             contains &= ec.containsUsedString(word);

dependency-check-core/src/main/java/org/owasp/dependencycheck/analyzer/JarAnalyzer.java

@@ -159,7 +159,7 @@ public class JarAnalyzer extends AbstractAnalyzer implements Analyzer {
     /**
      * The set of file extensions supported by this analyzer.
      */
-    private static final Set<String> EXTENSIONS = newHashSet("jar");
+    private static final Set<String> EXTENSIONS = newHashSet("jar", "war");
 
     /**
      * Returns a list of file EXTENSIONS supported by this analyzer.

dependency-check-core/src/main/java/org/owasp/dependencycheck/analyzer/NvdCveAnalyzer.java

@@ -16,19 +16,18 @@
  *
  * Copyright (c) 2012 Jeremy Long. All Rights Reserved.
  */
-package org.owasp.dependencycheck.data.nvdcve;
+package org.owasp.dependencycheck.analyzer;
 
 import java.io.IOException;
 import java.sql.SQLException;
 import java.util.List;
 import java.util.Set;
 import org.owasp.dependencycheck.Engine;
-import org.owasp.dependencycheck.analyzer.AnalysisException;
-import org.owasp.dependencycheck.analyzer.AnalysisPhase;
 import org.owasp.dependencycheck.dependency.Dependency;
 import org.owasp.dependencycheck.dependency.Vulnerability;
 import org.owasp.dependencycheck.dependency.Identifier;
-import org.owasp.dependencycheck.analyzer.Analyzer;
+import org.owasp.dependencycheck.data.nvdcve.CveDB;
+import org.owasp.dependencycheck.data.nvdcve.DatabaseException;
 
 /**
  * NvdCveAnalyzer is a utility class that takes a project dependency and

dependency-check-core/src/main/java/org/owasp/dependencycheck/concurrency/DirectoryLockException.java

@@ -0,0 +1,67 @@
+/*
+ * This file is part of dependency-check-core.
+ *
+ * Dependency-check-core is free software: you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation, either version 3 of the License, or (at your option) any
+ * later version.
+ *
+ * Dependency-check-core is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+ * details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * dependency-check-core. If not, see http://www.gnu.org/licenses/.
+ *
+ * Copyright (c) 2013 Jeremy Long. All Rights Reserved.
+ */
+package org.owasp.dependencycheck.concurrency;
+
+/**
+ * If thrown, indicates that a problem occurred when locking a directory.
+ *
+ * @author Jeremy Long (jeremy.long@owasp.org)
+ */
+public class DirectoryLockException extends Exception {
+
+    /**
+     * Default serial version UID.
+     */
+    private static final long serialVersionUID = 1L;
+
+    /**
+     * Constructs a new Directory Lock Exception.
+     */
+    public DirectoryLockException() {
+        super();
+    }
+
+    /**
+     * Constructs a new Directory Lock Exception.
+     *
+     * @param msg the message describing the exception
+     */
+    public DirectoryLockException(String msg) {
+        super(msg);
+    }
+
+    /**
+     * Constructs a new Directory Lock Exception.
+     *
+     * @param ex the cause of the exception
+     */
+    public DirectoryLockException(Throwable ex) {
+        super(ex);
+    }
+
+    /**
+     * Constructs a new Directory Lock Exception.
+     *
+     * @param msg the message describing the exception
+     * @param ex the cause of the exception
+     */
+    public DirectoryLockException(String msg, Throwable ex) {
+        super(msg, ex);
+    }
+}

dependency-check-core/src/main/java/org/owasp/dependencycheck/concurrency/DirectorySpinLock.java

@@ -0,0 +1,267 @@
+/*
+ * This file is part of dependency-check-core.
+ *
+ * Dependency-check-core is free software: you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation, either version 3 of the License, or (at your option) any
+ * later version.
+ *
+ * Dependency-check-core is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+ * details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * dependency-check-core. If not, see http://www.gnu.org/licenses/.
+ *
+ * Copyright (c) 2013 Jeremy Long. All Rights Reserved.
+ */
+package org.owasp.dependencycheck.concurrency;
+
+import java.io.Closeable;
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.RandomAccessFile;
+import java.nio.channels.AsynchronousCloseException;
+import java.nio.channels.ClosedChannelException;
+import java.nio.channels.FileChannel;
+import java.nio.channels.FileLock;
+import java.nio.channels.FileLockInterruptionException;
+import java.nio.channels.NonWritableChannelException;
+import java.nio.channels.OverlappingFileLockException;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+/**
+ * Implements a spin lock on a given directory. If the lock cannot be obtained,
+ * the process will "spin" waiting for an opportunity to obtain the lock
+ * requested.
+ *
+ * @author Jeremy Long (jeremy.long@owasp.org)
+ */
+public class DirectorySpinLock implements Closeable /*, AutoCloseable*/ {
+
+    /**
+     * The name of the lock file.
+     */
+    public static final String LOCK_NAME = "data.lock";
+    /**
+     * The maximum wait period used when attempting to obtain a lock.
+     */
+    public static final int MAX_SPIN = 100;
+    /**
+     * The file channel used to perform the lock.
+     */
+    private FileChannel channel = null;
+    /**
+     * The file used to perform the lock.
+     */
+    private File lockFile = null;
+    /**
+     * The lock object.
+     */
+    private FileLock lock = null;
+    /**
+     * The maximum number of seconds that the spin lock will wait while trying
+     * to obtain a lock.
+     */
+    private long maxWait = MAX_SPIN;
+
+    /**
+     * Get the maximum wait time, in seconds, that the spin lock will wait while
+     * trying to obtain a lock.
+     *
+     * @return the number of seconds the spin lock will wait
+     */
+    public long getMaxWait() {
+        return maxWait / 2; //sleep is for 500, so / 2
+    }
+
+    /**
+     * Set the maximum wait time, in seconds, that the spin lock will wait while
+     * trying to obtain a lock.
+     *
+     * @param maxWait the number of seconds the spin lock will wait
+     */
+    public void setMaxWait(long maxWait) {
+        this.maxWait = maxWait * 2; //sleep is for 500, so * 2
+    }
+
+    /**
+     * Constructs a new spin lock on the given directory.
+     *
+     * @param directory the directory to monitor/lock
+     * @throws InvalidDirectoryException thrown if there is an issue with the
+     * directory provided
+     * @throws DirectoryLockException thrown there is an issue obtaining a
+     * handle to the lock file
+     */
+    public DirectorySpinLock(File directory) throws InvalidDirectoryException, DirectoryLockException {
+        checkDirectory(directory);
+        lockFile = new File(directory, LOCK_NAME);
+        RandomAccessFile file = null;
+        try {
+            file = new RandomAccessFile(lockFile, "rw");
+        } catch (FileNotFoundException ex) {
+            throw new DirectoryLockException("Lock file not found", ex);
+        }
+        channel = file.getChannel();
+    }
+
+    /**
+     * Attempts to obtain an exclusive lock; an exception is thrown if the lock
+     * could not be obtained. This method may block for a few seconds if a lock
+     * cannot be obtained.
+     *
+     * @throws DirectoryLockException thrown if there is an exception obtaining
+     * the lock
+     */
+    public void obtainSharedLock() throws DirectoryLockException {
+        obtainLock(true);
+    }
+
+    /**
+     * Attempts to obtain an exclusive lock; an exception is thrown if the lock
+     * could not be obtained. This method may block for a few seconds if a lock
+     * cannot be obtained.
+     *
+     * @throws DirectoryLockException thrown if there is an exception obtaining
+     * the lock
+     */
+    public void obtainExclusiveLock() throws DirectoryLockException {
+        obtainLock(false);
+    }
+
+    /**
+     * Attempts to obtain a lock; an exception is thrown if the lock could not
+     * be obtained. This method may block for a few seconds if a lock cannot be
+     * obtained.
+     *
+     * @param shared true if the lock is shared, otherwise false
+     * @param maxWait the maximum time to wait, in seconds, while trying to
+     * obtain the lock
+     * @throws DirectoryLockException thrown if there is an exception obtaining
+     * the lock
+     */
+    protected void obtainLock(boolean shared, long maxWait) throws DirectoryLockException {
+        setMaxWait(maxWait);
+        obtainLock(shared);
+    }
+
+    /**
+     * Attempts to obtain a lock; an exception is thrown if the lock could not
+     * be obtained. This method may block for a few seconds if a lock cannot be
+     * obtained.
+     *
+     * @param shared true if the lock is shared, otherwise false
+     * @throws DirectoryLockException thrown if there is an exception obtaining
+     * the lock
+     */
+    protected void obtainLock(boolean shared) throws DirectoryLockException {
+        if (lock != null) {
+            release();
+        }
+        if (channel == null) {
+            throw new DirectoryLockException("Unable to create lock, no file channel exists");
+        }
+        int count = 0;
+        Exception lastException = null;
+        while (lock == null && count++ < maxWait) {
+            try {
+                lock = channel.lock(0, Long.MAX_VALUE, shared);
+            } catch (AsynchronousCloseException ex) {
+                lastException = ex;
+            } catch (ClosedChannelException ex) {
+                lastException = ex;
+            } catch (FileLockInterruptionException ex) {
+                lastException = ex;
+            } catch (OverlappingFileLockException ex) {
+                lastException = ex;
+            } catch (NonWritableChannelException ex) {
+                lastException = ex;
+            } catch (IOException ex) {
+                lastException = ex;
+            }
+            try {
+                Thread.sleep(500);
+            } catch (InterruptedException ex) {
+                Thread.currentThread().interrupt();
+            }
+        }
+        if (lock == null) {
+            if (lastException == null) {
+                throw new DirectoryLockException("Unable to obtain lock");
+            } else {
+                throw new DirectoryLockException("Unable to obtain lock", lastException);
+            }
+        }
+    }
+
+    /**
+     * Performs a few simple rudimentary checks on the specified directory.
+     * Specifically, does the file exist and is it a directory.
+     *
+     * @param directory the File object to inspect
+     * @throws InvalidDirectoryException thrown if the directory is null or is
+     * not a directory
+     */
+    private void checkDirectory(File directory) throws InvalidDirectoryException {
+        if (directory == null) {
+            throw new InvalidDirectoryException("Unable to obtain lock on a null File");
+        }
+        if (!directory.isDirectory()) {
+            final String msg = String.format("File, '%s', does not exist or is not a directory", directory.getAbsolutePath());
+            throw new InvalidDirectoryException(msg);
+        }
+    }
+
+    /**
+     * Releases any locks and closes the underlying channel.
+     *
+     * @throws IOException if an IO Exception occurs
+     */
+    @Override
+    public void close() throws IOException {
+        release();
+// TODO uncomment this once support for 1.6 is dropped.
+//        if (lock != null) {
+//            try {
+//                lock.close();
+//            } catch (IOException ex) {
+//                Logger.getLogger(DirectorySpinLock.class.getName()).log(Level.FINEST, "Unable to close file lock due to IO Exception", ex);
+//            }
+//        }
+        if (channel != null) {
+            try {
+                channel.close();
+            } catch (IOException ex) {
+                Logger.getLogger(DirectorySpinLock.class.getName()).log(Level.FINEST, "Unable to close the channel for the file lock", ex);
+            }
+        }
+        if (lockFile != null) {
+            if (lockFile.exists()) {
+                /* yes, this delete could fail which is totally fine. The other
+                 * thread holding the lock while delete it.
+                 */
+                lockFile.delete();
+            }
+        }
+    }
+
+    /**
+     * Releases the lock. Any exceptions that are thrown by the underlying lock
+     * during the release are ignored.
+     */
+    public void release() {
+        if (lock != null) {
+            try {
+                lock.release();
+            } catch (ClosedChannelException ex) {
+                Logger.getLogger(DirectorySpinLock.class.getName()).log(Level.FINEST, "Uable to release file lock", ex);
+            } catch (IOException ex) {
+                Logger.getLogger(DirectorySpinLock.class.getName()).log(Level.FINEST, "Unable to release file lock due to IO Exception", ex);
+            }
+        }
+    }
+}

dependency-check-core/src/main/java/org/owasp/dependencycheck/concurrency/InvalidDirectoryException.java

@@ -0,0 +1,67 @@
+/*
+ * This file is part of dependency-check-core.
+ *
+ * Dependency-check-core is free software: you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation, either version 3 of the License, or (at your option) any
+ * later version.
+ *
+ * Dependency-check-core is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+ * details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * dependency-check-core. If not, see http://www.gnu.org/licenses/.
+ *
+ * Copyright (c) 2013 Jeremy Long. All Rights Reserved.
+ */
+package org.owasp.dependencycheck.concurrency;
+
+/**
+ * If thrown, indicates that there is a problem with a directory.
+ *
+ * @author Jeremy Long (jeremy.long@owasp.org)
+ */
+public class InvalidDirectoryException extends Exception {
+
+    /**
+     * Default serial version UID.
+     */
+    private static final long serialVersionUID = 1L;
+
+    /**
+     * Constructs a new Invalid Directory Exception.
+     */
+    public InvalidDirectoryException() {
+        super();
+    }
+
+    /**
+     * Constructs a new Invalid Directory Exception.
+     *
+     * @param msg the message describing the exception
+     */
+    public InvalidDirectoryException(String msg) {
+        super(msg);
+    }
+
+    /**
+     * Constructs a new Invalid Directory Exception.
+     *
+     * @param ex the cause of the exception
+     */
+    public InvalidDirectoryException(Throwable ex) {
+        super(ex);
+    }
+
+    /**
+     * Constructs a new Invalid Directory Exception.
+     *
+     * @param msg the message describing the exception
+     * @param ex the cause of the exception
+     */
+    public InvalidDirectoryException(String msg, Throwable ex) {
+        super(msg, ex);
+    }
+}

dependency-check-core/src/main/java/org/owasp/dependencycheck/concurrency/package-info.java

@@ -0,0 +1,11 @@
+/**
+ * <html>
+ * <head>
+ * <title>org.owasp.dependencycheck.concurrency</title>
+ * </head>
+ * <body>
+ * Contains classes used to create shared and exclusive locks on directories.
+ * </body>
+ * </html>
+ */
+package org.owasp.dependencycheck.concurrency;

dependency-check-core/src/main/java/org/owasp/dependencycheck/data/cpe/BaseIndex.java

@@ -0,0 +1,117 @@
+/*
+ * This file is part of dependency-check-core.
+ *
+ * Dependency-check-core is free software: you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation, either version 3 of the License, or (at your option) any
+ * later version.
+ *
+ * Dependency-check-core is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+ * details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * dependency-check-core. If not, see http://www.gnu.org/licenses/.
+ *
+ * Copyright (c) 2013 Jeremy Long. All Rights Reserved.
+ */
+package org.owasp.dependencycheck.data.cpe;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.FSDirectory;
+import org.owasp.dependencycheck.utils.Settings;
+
+/**
+ * The Base Index class used to access the CPE Index.
+ *
+ * @author Jeremy Long (jeremy.long@owasp.org)
+ */
+public abstract class BaseIndex {
+
+    /**
+     * The Lucene directory containing the index.
+     */
+    private Directory directory;
+    /**
+     * Indicates whether or not the Lucene Index is open.
+     */
+    private boolean indexOpen = false;
+
+    /**
+     * Gets the directory.
+     *
+     * @return the directory
+     */
+    public Directory getDirectory() {
+        return directory;
+    }
+
+    /**
+     * Opens the CPE Index.
+     *
+     * @throws IOException is thrown if an IOException occurs opening the index.
+     */
+    public void open() throws IOException {
+        directory = this.openDirectory();
+        indexOpen = true;
+    }
+
+    /**
+     * Closes the CPE Index.
+     */
+    public void close() {
+        try {
+            directory.close();
+        } catch (IOException ex) {
+            final String msg = "Unable to update database due to an IO error.";
+            Logger.getLogger(BaseIndex.class.getName()).log(Level.SEVERE, msg);
+            Logger.getLogger(BaseIndex.class.getName()).log(Level.FINE, null, ex);
+        } finally {
+            directory = null;
+        }
+        indexOpen = false;
+
+    }
+
+    /**
+     * Returns the status of the data source - is the index open.
+     *
+     * @return true or false.
+     */
+    public boolean isOpen() {
+        return indexOpen;
+    }
+
+    /**
+     * Returns the Lucene directory object for the CPE Index.
+     *
+     * @return the Lucene Directory object for the CPE Index.
+     * @throws IOException is thrown if an IOException occurs.
+     */
+    protected Directory openDirectory() throws IOException {
+        final File path = getDataDirectory();
+        return FSDirectory.open(path);
+    }
+
+    /**
+     * Retrieves the directory that the JAR file exists in so that we can ensure
+     * we always use a common data directory.
+     *
+     * @return the data directory for this index.
+     * @throws IOException is thrown if an IOException occurs of course...
+     */
+    public static File getDataDirectory() throws IOException {
+        final File path = Settings.getFile(Settings.KEYS.CPE_DATA_DIRECTORY);
+        if (!path.exists()) {
+            if (!path.mkdirs()) {
+                throw new IOException("Unable to create CPE Data directory");
+            }
+        }
+        return path;
+    }
+}

dependency-check-core/src/main/java/org/owasp/dependencycheck/data/cpe/CpeIndexReader.java

@@ -0,0 +1,179 @@
+/*
+ * This file is part of dependency-check-core.
+ *
+ * Dependency-check-core is free software: you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation, either version 3 of the License, or (at your option) any
+ * later version.
+ *
+ * Dependency-check-core is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+ * details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * dependency-check-core. If not, see http://www.gnu.org/licenses/.
+ *
+ * Copyright (c) 2013 Jeremy Long. All Rights Reserved.
+ */
+package org.owasp.dependencycheck.data.cpe;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.core.KeywordAnalyzer;
+import org.apache.lucene.analysis.miscellaneous.PerFieldAnalyzerWrapper;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.index.CorruptIndexException;
+import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.queryparser.classic.ParseException;
+import org.apache.lucene.queryparser.classic.QueryParser;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.TopDocs;
+import org.apache.lucene.util.Version;
+import org.owasp.dependencycheck.data.lucene.FieldAnalyzer;
+import org.owasp.dependencycheck.data.lucene.SearchFieldAnalyzer;
+
+/**
+ *
+ * @author Jeremy Long (jeremy.long@owasp.org)
+ */
+public class CpeIndexReader extends BaseIndex {
+
+    /**
+     * The Lucene IndexReader.
+     */
+    private IndexReader indexReader;
+    /**
+     * The Lucene IndexSearcher.
+     */
+    private IndexSearcher indexSearcher;
+    /**
+     * The Lucene Analyzer used for Searching.
+     */
+    private Analyzer searchingAnalyzer;
+    /**
+     * The Lucene QueryParser used for Searching.
+     */
+    private QueryParser queryParser;
+    /**
+     * The search field analyzer for the product field.
+     */
+    private SearchFieldAnalyzer productSearchFieldAnalyzer;
+    /**
+     * The search field analyzer for the vendor field.
+     */
+    private SearchFieldAnalyzer vendorSearchFieldAnalyzer;
+
+    /**
+     * Opens the CPE Index.
+     *
+     * @throws IOException is thrown if an IOException occurs opening the index.
+     */
+    @Override
+    public void open() throws IOException {
+        //TODO add spinlock (shared)
+        super.open();
+        indexReader = DirectoryReader.open(getDirectory());
+        indexSearcher = new IndexSearcher(indexReader);
+        searchingAnalyzer = createSearchingAnalyzer();
+        queryParser = new QueryParser(Version.LUCENE_43, Fields.DOCUMENT_KEY, searchingAnalyzer);
+    }
+
+    /**
+     * Closes the CPE Index.
+     */
+    @Override
+    public void close() {
+        //TODO remove spinlock (shared)
+        if (searchingAnalyzer != null) {
+            searchingAnalyzer.close();
+            searchingAnalyzer = null;
+        }
+        if (indexReader != null) {
+            try {
+                indexReader.close();
+            } catch (IOException ex) {
+                Logger.getLogger(CpeIndexReader.class.getName()).log(Level.FINEST, null, ex);
+            }
+            indexReader = null;
+        }
+        queryParser = null;
+        indexSearcher = null;
+        super.close();
+    }
+
+    /**
+     * Searches the index using the given search string.
+     *
+     * @param searchString the query text
+     * @param maxQueryResults the maximum number of documents to return
+     * @return the TopDocs found by the search
+     * @throws ParseException thrown when the searchString is invalid
+     * @throws IOException is thrown if there is an issue with the underlying
+     * Index
+     */
+    public TopDocs search(String searchString, int maxQueryResults) throws ParseException, IOException {
+        final Query query = queryParser.parse(searchString);
+        return indexSearcher.search(query, maxQueryResults);
+    }
+
+    /**
+     * Searches the index using the given query.
+     *
+     * @param query the query used to search the index
+     * @param maxQueryResults the max number of results to return
+     * @return the TopDocs found be the query
+     * @throws CorruptIndexException thrown if the Index is corrupt
+     * @throws IOException thrown if there is an IOException
+     */
+    public TopDocs search(Query query, int maxQueryResults) throws CorruptIndexException, IOException {
+        resetSearchingAnalyzer();
+        return indexSearcher.search(query, maxQueryResults);
+    }
+
+    /**
+     * Retrieves a document from the Index.
+     *
+     * @param documentId the id of the document to retrieve
+     * @return the Document
+     * @throws IOException thrown if there is an IOException
+     */
+    public Document getDocument(int documentId) throws IOException {
+        return indexSearcher.doc(documentId);
+    }
+
+    /**
+     * Creates an Analyzer for searching the CPE Index.
+     *
+     * @return the CPE Analyzer.
+     */
+    @SuppressWarnings("unchecked")
+    private Analyzer createSearchingAnalyzer() {
+        final Map fieldAnalyzers = new HashMap();
+        fieldAnalyzers.put(Fields.DOCUMENT_KEY, new KeywordAnalyzer());
+        productSearchFieldAnalyzer = new SearchFieldAnalyzer(Version.LUCENE_43);
+        vendorSearchFieldAnalyzer = new SearchFieldAnalyzer(Version.LUCENE_43);
+        fieldAnalyzers.put(Fields.PRODUCT, productSearchFieldAnalyzer);
+        fieldAnalyzers.put(Fields.VENDOR, vendorSearchFieldAnalyzer);
+
+        return new PerFieldAnalyzerWrapper(new FieldAnalyzer(Version.LUCENE_43), fieldAnalyzers);
+    }
+
+    /**
+     * Resets the searching analyzers
+     */
+    private void resetSearchingAnalyzer() {
+        if (productSearchFieldAnalyzer != null) {
+            productSearchFieldAnalyzer.clear();
+        }
+        if (vendorSearchFieldAnalyzer != null) {
+            vendorSearchFieldAnalyzer.clear();
+        }
+    }
+}

dependency-check-core/src/main/java/org/owasp/dependencycheck/data/cpe/CpeIndexWriter.java

@@ -0,0 +1,149 @@
+/*
+ * This file is part of dependency-check-core.
+ *
+ * Dependency-check-core is free software: you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation, either version 3 of the License, or (at your option) any
+ * later version.
+ *
+ * Dependency-check-core is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+ * details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * dependency-check-core. If not, see http://www.gnu.org/licenses/.
+ *
+ * Copyright (c) 2013 Jeremy Long. All Rights Reserved.
+ */
+package org.owasp.dependencycheck.data.cpe;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.core.KeywordAnalyzer;
+import org.apache.lucene.analysis.miscellaneous.PerFieldAnalyzerWrapper;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.document.StringField;
+import org.apache.lucene.document.TextField;
+import org.apache.lucene.index.CorruptIndexException;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.util.Version;
+import org.owasp.dependencycheck.data.lucene.FieldAnalyzer;
+
+/**
+ *
+ * @author Jeremy Long (jeremy.long@owasp.org)
+ */
+public class CpeIndexWriter extends BaseIndex {
+
+    /**
+     * The IndexWriter for the Lucene index.
+     */
+    private IndexWriter indexWriter;
+    /**
+     * The Lucene Analyzer used for Indexing.
+     */
+    private Analyzer indexingAnalyzer;
+
+    /**
+     * Opens the CPE Index.
+     *
+     * @throws IOException is thrown if an IOException occurs opening the index.
+     */
+    @Override
+    public void open() throws IOException {
+        //TODO add spinlock
+        super.open();
+        indexingAnalyzer = createIndexingAnalyzer();
+        final IndexWriterConfig conf = new IndexWriterConfig(Version.LUCENE_43, indexingAnalyzer);
+        indexWriter = new IndexWriter(getDirectory(), conf);
+    }
+
+    /**
+     * Closes the CPE Index.
+     */
+    @Override
+    public void close() {
+        //TODO remove spinlock
+        if (indexWriter != null) {
+            commit();
+            try {
+                indexWriter.close(true);
+            } catch (CorruptIndexException ex) {
+                final String msg = "Unable to update database, there is a corrupt index.";
+                Logger.getLogger(CpeIndexWriter.class.getName()).log(Level.SEVERE, msg);
+                Logger.getLogger(CpeIndexWriter.class.getName()).log(Level.FINE, null, ex);
+            } catch (IOException ex) {
+                final String msg = "Unable to update database due to an IO error.";
+                Logger.getLogger(CpeIndexWriter.class.getName()).log(Level.SEVERE, msg);
+                Logger.getLogger(CpeIndexWriter.class.getName()).log(Level.FINE, null, ex);
+            } finally {
+                indexWriter = null;
+            }
+        }
+        if (indexingAnalyzer != null) {
+            indexingAnalyzer.close();
+            indexingAnalyzer = null;
+        }
+        super.close();
+    }
+
+    /**
+     * Commits any pending changes.
+     */
+    public void commit() {
+        if (indexWriter != null) {
+            try {
+                indexWriter.forceMerge(1);
+                indexWriter.commit();
+            } catch (CorruptIndexException ex) {
+                final String msg = "Unable to update database, there is a corrupt index.";
+                Logger.getLogger(CpeIndexWriter.class.getName()).log(Level.SEVERE, msg);
+                Logger.getLogger(CpeIndexWriter.class.getName()).log(Level.FINE, null, ex);
+            } catch (IOException ex) {
+                final String msg = "Unable to update database due to an IO error.";
+                Logger.getLogger(CpeIndexWriter.class.getName()).log(Level.SEVERE, msg);
+                Logger.getLogger(CpeIndexWriter.class.getName()).log(Level.FINE, null, ex);
+            }
+        }
+    }
+
+    /**
+     * Creates the indexing analyzer for the CPE Index.
+     *
+     * @return the CPE Analyzer.
+     */
+    @SuppressWarnings("unchecked")
+    private Analyzer createIndexingAnalyzer() {
+        final Map fieldAnalyzers = new HashMap();
+        fieldAnalyzers.put(Fields.DOCUMENT_KEY, new KeywordAnalyzer());
+        return new PerFieldAnalyzerWrapper(new FieldAnalyzer(Version.LUCENE_43), fieldAnalyzers);
+    }
+
+    /**
+     * Saves a CPE IndexEntry into the Lucene index.
+     *
+     * @param entry a CPE entry.
+     * @throws CorruptIndexException is thrown if the index is corrupt.
+     * @throws IOException is thrown if an IOException occurs.
+     */
+    public void saveEntry(IndexEntry entry) throws CorruptIndexException, IOException {
+        final Document doc = new Document();
+        final Field documentKey = new StringField(Fields.DOCUMENT_KEY, entry.getDocumentId(), Field.Store.NO);
+        final Field vendor = new TextField(Fields.VENDOR, entry.getVendor(), Field.Store.YES);
+        final Field product = new TextField(Fields.PRODUCT, entry.getProduct(), Field.Store.YES);
+        doc.add(documentKey);
+        doc.add(vendor);
+        doc.add(product);
+
+        final Term term = new Term(Fields.DOCUMENT_KEY, entry.getDocumentId());
+        indexWriter.updateDocument(term, doc);
+    }
+}

dependency-check-core/src/main/java/org/owasp/dependencycheck/data/cpe/Index.java

@@ -1,170 +0,0 @@
-/*
- * This file is part of dependency-check-core.
- *
- * Dependency-check-core is free software: you can redistribute it and/or modify it
- * under the terms of the GNU General Public License as published by the Free
- * Software Foundation, either version 3 of the License, or (at your option) any
- * later version.
- *
- * Dependency-check-core is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
- * details.
- *
- * You should have received a copy of the GNU General Public License along with
- * dependency-check-core. If not, see http://www.gnu.org/licenses/.
- *
- * Copyright (c) 2012 Jeremy Long. All Rights Reserved.
- */
-package org.owasp.dependencycheck.data.cpe;
-
-import java.io.File;
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.Map;
-import org.apache.lucene.analysis.Analyzer;
-import org.apache.lucene.analysis.core.KeywordAnalyzer;
-import org.apache.lucene.analysis.miscellaneous.PerFieldAnalyzerWrapper;
-import org.apache.lucene.document.Document;
-import org.apache.lucene.document.Field;
-import org.apache.lucene.document.TextField;
-import org.apache.lucene.index.CorruptIndexException;
-import org.apache.lucene.index.Term;
-import org.apache.lucene.queryparser.classic.QueryParser;
-import org.apache.lucene.store.Directory;
-import org.apache.lucene.store.FSDirectory;
-import org.apache.lucene.util.Version;
-import org.owasp.dependencycheck.data.lucene.AbstractIndex;
-import org.owasp.dependencycheck.utils.Settings;
-import org.owasp.dependencycheck.data.lucene.FieldAnalyzer;
-import org.owasp.dependencycheck.data.lucene.SearchFieldAnalyzer;
-
-/**
- * The Index class is used to utilize and maintain the CPE Index.
- *
- * @author Jeremy Long (jeremy.long@owasp.org)
- */
-public class Index extends AbstractIndex {
-
-    /**
-     * Returns the directory that holds the CPE Index.
-     *
-     * @return the Directory containing the CPE Index.
-     * @throws IOException is thrown if an IOException occurs.
-     */
-    @Override
-    public Directory getDirectory() throws IOException {
-        final File path = getDataDirectory();
-        return FSDirectory.open(path);
-    }
-
-    /**
-     * Retrieves the directory that the JAR file exists in so that we can ensure
-     * we always use a common data directory.
-     *
-     * @return the data directory for this index.
-     * @throws IOException is thrown if an IOException occurs of course...
-     */
-    public File getDataDirectory() throws IOException {
-        final File path = Settings.getFile(Settings.KEYS.CPE_DATA_DIRECTORY);
-        if (!path.exists()) {
-            if (!path.mkdirs()) {
-                throw new IOException("Unable to create CPE Data directory");
-            }
-        }
-        return path;
-    }
-
-    /**
-     * Creates an Analyzer for the CPE Index.
-     *
-     * @return the CPE Analyzer.
-     */
-    @SuppressWarnings("unchecked")
-    @Override
-    public Analyzer createIndexingAnalyzer() {
-        final Map fieldAnalyzers = new HashMap();
-        fieldAnalyzers.put(Fields.DOCUMENT_KEY, new KeywordAnalyzer());
-        return new PerFieldAnalyzerWrapper(new FieldAnalyzer(Version.LUCENE_43), fieldAnalyzers);
-    }
-    /**
-     * The search field analyzer for the product field.
-     */
-    private SearchFieldAnalyzer productSearchFieldAnalyzer;
-    /**
-     * The search field analyzer for the vendor field.
-     */
-    private SearchFieldAnalyzer vendorSearchFieldAnalyzer;
-
-    /**
-     * Creates an Analyzer for searching the CPE Index.
-     *
-     * @return the CPE Analyzer.
-     */
-    @SuppressWarnings("unchecked")
-    @Override
-    public Analyzer createSearchingAnalyzer() {
-        final Map fieldAnalyzers = new HashMap();
-
-        fieldAnalyzers.put(Fields.DOCUMENT_KEY, new KeywordAnalyzer());
-        productSearchFieldAnalyzer = new SearchFieldAnalyzer(Version.LUCENE_43);
-        vendorSearchFieldAnalyzer = new SearchFieldAnalyzer(Version.LUCENE_43);
-        fieldAnalyzers.put(Fields.PRODUCT, productSearchFieldAnalyzer);
-        fieldAnalyzers.put(Fields.VENDOR, vendorSearchFieldAnalyzer);
-
-        return new PerFieldAnalyzerWrapper(new FieldAnalyzer(Version.LUCENE_43), fieldAnalyzers);
-    }
-
-    /**
-     * Creates the Lucene QueryParser used when querying the index.
-     *
-     * @return a QueryParser.
-     */
-    @Override
-    public QueryParser createQueryParser() {
-        return new QueryParser(Version.LUCENE_43, Fields.DOCUMENT_KEY, getSearchingAnalyzer());
-    }
-
-    /**
-     * Resets the searching analyzers
-     */
-    @Override
-    protected void resetSearchingAnalyzer() {
-        if (productSearchFieldAnalyzer != null) {
-            productSearchFieldAnalyzer.clear();
-        }
-        if (vendorSearchFieldAnalyzer != null) {
-            vendorSearchFieldAnalyzer.clear();
-        }
-    }
-
-    /**
-     * Saves a CPE IndexEntry into the Lucene index.
-     *
-     * @param entry a CPE entry.
-     * @throws CorruptIndexException is thrown if the index is corrupt.
-     * @throws IOException is thrown if an IOException occurs.
-     */
-    public void saveEntry(IndexEntry entry) throws CorruptIndexException, IOException {
-        final Document doc = convertEntryToDoc(entry);
-        final Term term = new Term(Fields.DOCUMENT_KEY, entry.getDocumentId());
-        getIndexWriter().updateDocument(term, doc);
-    }
-
-    /**
-     * Converts a CPE entry into a Lucene Document.
-     *
-     * @param entry a CPE IndexEntry.
-     * @return a Lucene Document containing a CPE IndexEntry.
-     */
-    protected Document convertEntryToDoc(IndexEntry entry) {
-        final Document doc = new Document();
-
-        final Field vendor = new TextField(Fields.VENDOR, entry.getVendor(), Field.Store.YES);
-        doc.add(vendor);
-
-        final Field product = new TextField(Fields.PRODUCT, entry.getProduct(), Field.Store.YES);
-        doc.add(product);
-        return doc;
-    }
-}

dependency-check-core/src/main/java/org/owasp/dependencycheck/data/lucene/AbstractIndex.java

@@ -1,341 +0,0 @@
-/*
- * This file is part of dependency-check-core.
- *
- * Dependency-check-core is free software: you can redistribute it and/or modify it
- * under the terms of the GNU General Public License as published by the Free
- * Software Foundation, either version 3 of the License, or (at your option) any
- * later version.
- *
- * Dependency-check-core is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
- * details.
- *
- * You should have received a copy of the GNU General Public License along with
- * dependency-check-core. If not, see http://www.gnu.org/licenses/.
- *
- * Copyright (c) 2012 Jeremy Long. All Rights Reserved.
- */
-package org.owasp.dependencycheck.data.lucene;
-
-import java.io.IOException;
-import java.util.logging.Level;
-import java.util.logging.Logger;
-import org.apache.lucene.analysis.Analyzer;
-import org.apache.lucene.document.Document;
-import org.apache.lucene.index.CorruptIndexException;
-import org.apache.lucene.index.DirectoryReader;
-import org.apache.lucene.index.IndexReader;
-import org.apache.lucene.index.IndexWriter;
-import org.apache.lucene.index.IndexWriterConfig;
-import org.apache.lucene.queryparser.classic.ParseException;
-import org.apache.lucene.queryparser.classic.QueryParser;
-import org.apache.lucene.search.IndexSearcher;
-import org.apache.lucene.search.Query;
-import org.apache.lucene.search.TopDocs;
-import org.apache.lucene.store.Directory;
-import org.apache.lucene.store.LockObtainFailedException;
-import org.apache.lucene.util.Version;
-
-/**
- * The base Index for other index objects. Implements the open and close
- * methods.
- *
- * @author Jeremy Long (jeremy.long@owasp.org)
- */
-public abstract class AbstractIndex {
-
-    /**
-     * The Lucene directory containing the index.
-     */
-    private Directory directory;
-    /**
-     * The IndexWriter for the Lucene index.
-     */
-    private IndexWriter indexWriter;
-    /**
-     * The Lucene IndexReader.
-     */
-    private IndexReader indexReader;
-    /**
-     * The Lucene IndexSearcher.
-     */
-    private IndexSearcher indexSearcher;
-    /**
-     * The Lucene Analyzer used for Indexing.
-     */
-    private Analyzer indexingAnalyzer;
-    /**
-     * The Lucene Analyzer used for Searching.
-     */
-    private Analyzer searchingAnalyzer;
-    /**
-     * The Lucene QueryParser used for Searching.
-     */
-    private QueryParser queryParser;
-    /**
-     * Indicates whether or not the Lucene Index is open.
-     */
-    private boolean indexOpen = false;
-
-    /**
-     * Opens the CPE Index.
-     *
-     * @throws IOException is thrown if an IOException occurs opening the index.
-     */
-    public void open() throws IOException {
-        directory = this.getDirectory();
-        indexingAnalyzer = this.getIndexingAnalyzer();
-        searchingAnalyzer = this.getSearchingAnalyzer();
-        indexOpen = true;
-    }
-
-    /**
-     * Commits any pending changes.
-     */
-    public void commit() {
-        if (indexWriter != null) {
-            try {
-                indexWriter.commit();
-            } catch (CorruptIndexException ex) {
-                final String msg = "Unable to update database, there is a corrupt index.";
-                Logger.getLogger(AbstractIndex.class.getName()).log(Level.SEVERE, msg);
-                Logger.getLogger(AbstractIndex.class.getName()).log(Level.FINE, null, ex);
-            } catch (IOException ex) {
-                final String msg = "Unable to update database due to an IO error.";
-                Logger.getLogger(AbstractIndex.class.getName()).log(Level.SEVERE, msg);
-                Logger.getLogger(AbstractIndex.class.getName()).log(Level.FINE, null, ex);
-            }
-        }
-    }
-
-    /**
-     * Closes the CPE Index.
-     */
-    public void close() {
-        if (indexWriter != null) {
-            commit();
-            try {
-                indexWriter.close(true);
-            } catch (CorruptIndexException ex) {
-                final String msg = "Unable to update database, there is a corrupt index.";
-                Logger.getLogger(AbstractIndex.class.getName()).log(Level.SEVERE, msg);
-                Logger.getLogger(AbstractIndex.class.getName()).log(Level.FINE, null, ex);
-            } catch (IOException ex) {
-                final String msg = "Unable to update database due to an IO error.";
-                Logger.getLogger(AbstractIndex.class.getName()).log(Level.SEVERE, msg);
-                Logger.getLogger(AbstractIndex.class.getName()).log(Level.FINE, null, ex);
-            } finally {
-                indexWriter = null;
-            }
-        }
-        if (indexSearcher != null) {
-            indexSearcher = null;
-        }
-
-        if (indexingAnalyzer != null) {
-            indexingAnalyzer.close();
-            indexingAnalyzer = null;
-        }
-
-        if (searchingAnalyzer != null) {
-            searchingAnalyzer.close();
-            searchingAnalyzer = null;
-        }
-
-        try {
-            directory.close();
-        } catch (IOException ex) {
-            final String msg = "Unable to update database due to an IO error.";
-            Logger.getLogger(AbstractIndex.class.getName()).log(Level.SEVERE, msg);
-            Logger.getLogger(AbstractIndex.class.getName()).log(Level.FINE, null, ex);
-        } finally {
-            directory = null;
-        }
-        indexOpen = false;
-    }
-
-    /**
-     * Returns the status of the data source - is the index open.
-     *
-     * @return true or false.
-     */
-    public boolean isOpen() {
-        return indexOpen;
-    }
-
-    /**
-     * Opens the Lucene Index Writer.
-     *
-     * @throws CorruptIndexException is thrown if the Lucene index is corrupt.
-     * @throws IOException is thrown if an IOException occurs opening the index.
-     */
-    public void openIndexWriter() throws CorruptIndexException, IOException {
-        if (!isOpen()) {
-            open();
-        }
-        final IndexWriterConfig conf = new IndexWriterConfig(Version.LUCENE_43, indexingAnalyzer);
-        indexWriter = new IndexWriter(directory, conf);
-    }
-
-    /**
-     * Retrieves the IndexWriter for the Lucene Index.
-     *
-     * @return an IndexWriter.
-     * @throws CorruptIndexException is thrown if the Lucene Index is corrupt.
-     * @throws LockObtainFailedException is thrown if there is an exception
-     * obtaining a lock on the Lucene index.
-     * @throws IOException is thrown if an IOException occurs opening the index.
-     */
-    public IndexWriter getIndexWriter() throws CorruptIndexException, LockObtainFailedException, IOException {
-        if (indexWriter == null) {
-            openIndexWriter();
-        }
-        return indexWriter;
-    }
-
-    /**
-     * Opens the Lucene Index for reading.
-     *
-     * @throws CorruptIndexException is thrown if the index is corrupt.
-     * @throws IOException is thrown if there is an exception reading the index.
-     */
-    public void openIndexReader() throws CorruptIndexException, IOException {
-        if (!isOpen()) {
-            open();
-        }
-        //indexReader = IndexReader.open(directory, true);
-        indexReader = DirectoryReader.open(directory);
-    }
-
-    /**
-     * Returns an IndexSearcher for the Lucene Index.
-     *
-     * @return an IndexSearcher.
-     * @throws CorruptIndexException is thrown if the index is corrupt.
-     * @throws IOException is thrown if there is an exception reading the index.
-     */
-    protected IndexSearcher getIndexSearcher() throws CorruptIndexException, IOException {
-        if (indexReader == null) {
-            openIndexReader();
-        }
-        if (indexSearcher == null) {
-            indexSearcher = new IndexSearcher(indexReader);
-        }
-        return indexSearcher;
-    }
-
-    /**
-     * Returns an Analyzer to be used when indexing.
-     *
-     * @return an Analyzer.
-     */
-    public Analyzer getIndexingAnalyzer() {
-        if (indexingAnalyzer == null) {
-            indexingAnalyzer = createIndexingAnalyzer();
-        }
-        return indexingAnalyzer;
-    }
-
-    /**
-     * Returns an analyzer used for searching the index
-     *
-     * @return a lucene analyzer
-     */
-    protected Analyzer getSearchingAnalyzer() {
-        if (searchingAnalyzer == null) {
-            searchingAnalyzer = createSearchingAnalyzer();
-        }
-        return searchingAnalyzer;
-    }
-
-    /**
-     * Gets a query parser
-     *
-     * @return a query parser
-     */
-    protected QueryParser getQueryParser() {
-        if (queryParser == null) {
-            queryParser = createQueryParser();
-        }
-        return queryParser;
-    }
-
-    /**
-     * Searches the index using the given search string.
-     *
-     * @param searchString the query text
-     * @param maxQueryResults the maximum number of documents to return
-     * @return the TopDocs found by the search
-     * @throws ParseException thrown when the searchString is invalid
-     * @throws IOException is thrown if there is an issue with the underlying
-     * Index
-     */
-    public TopDocs search(String searchString, int maxQueryResults) throws ParseException, IOException {
-        final QueryParser parser = getQueryParser();
-        final Query query = parser.parse(searchString);
-        resetSearchingAnalyzer();
-        final IndexSearcher is = getIndexSearcher();
-        return is.search(query, maxQueryResults);
-    }
-
-    /**
-     * Searches the index using the given query.
-     *
-     * @param query the query used to search the index
-     * @param maxQueryResults the max number of results to return
-     * @return the TopDocs found be the query
-     * @throws CorruptIndexException thrown if the Index is corrupt
-     * @throws IOException thrown if there is an IOException
-     */
-    public TopDocs search(Query query, int maxQueryResults) throws CorruptIndexException, IOException {
-        final IndexSearcher is = getIndexSearcher();
-        return is.search(query, maxQueryResults);
-    }
-
-    /**
-     * Retrieves a document from the Index.
-     *
-     * @param documentId the id of the document to retrieve
-     * @return the Document
-     * @throws IOException thrown if there is an IOException
-     */
-    public Document getDocument(int documentId) throws IOException {
-        final IndexSearcher is = getIndexSearcher();
-        return is.doc(documentId);
-    }
-
-    /**
-     * Gets the directory that contains the Lucene Index.
-     *
-     * @return a Lucene Directory
-     * @throws IOException is thrown when an IOException occurs
-     */
-    public abstract Directory getDirectory() throws IOException;
-
-    /**
-     * Creates the Lucene Analyzer used when indexing.
-     *
-     * @return a Lucene Analyzer
-     */
-    public abstract Analyzer createIndexingAnalyzer();
-
-    /**
-     * Creates the Lucene Analyzer used when querying the index.
-     *
-     * @return a Lucene Analyzer
-     */
-    public abstract Analyzer createSearchingAnalyzer();
-
-    /**
-     * Creates the Lucene QueryParser used when querying the index.
-     *
-     * @return a QueryParser
-     */
-    public abstract QueryParser createQueryParser();
-
-    /**
-     * Resets the searching analyzers
-     */
-    protected abstract void resetSearchingAnalyzer();
-}

dependency-check-core/src/main/java/org/owasp/dependencycheck/data/nvdcve/CveDB.java

@@ -58,7 +58,7 @@ public class CveDB {
     /**
      * The version of the current DB Schema.
      */
-    public static final String DB_SCHEMA_VERSION = "2.5";
+    public static final String DB_SCHEMA_VERSION = "2.6";
     /**
      * Database connection
      */
@@ -162,27 +162,16 @@ public class CveDB {
             value = "DMI_EMPTY_DB_PASSWORD",
             justification = "Yes, I know... Blank password.")
     public void open() throws IOException, SQLException, DatabaseException, ClassNotFoundException {
-        /*
-         * TODO - make it so we can exteralize the database (lucene index is a problem), could I store it as a blob
-         *        and just download it when needed?
-         */
-//        String dbDriver = Settings.getString(Settings.KEYS.DB_DRIVER);
-//        String dbConnStr = Settings.getString(Settings.KEYS.DB_CONNECTION_STRING);
-//        if (dbDriver != null && dbConnStr != null) {
-//            Class.forName(dbDriver);
-//            conn = DriverManager.getConnection(dbConnStr);
-//        } else { //use the embeded version
         final String fileName = CveDB.getDataDirectory().getCanonicalPath();
         final File f = new File(fileName, "cve." + DB_SCHEMA_VERSION);
         final File check = new File(f.getAbsolutePath() + ".h2.db");
         final boolean createTables = !check.exists();
-        final String connStr = "jdbc:h2:file:" + f.getAbsolutePath();
+        final String connStr = String.format("jdbc:h2:file:%s;AUTO_SERVER=TRUE", f.getAbsolutePath());
         Class.forName("org.h2.Driver");
         conn = DriverManager.getConnection(connStr, "sa", "");
         if (createTables) {
             createTables();
         }
-//        }
     }
 
     /**

dependency-check-core/src/main/java/org/owasp/dependencycheck/data/nvdcve/InvalidDataException.java

@@ -16,7 +16,7 @@
  *
  * Copyright (c) 2012 Jeremy Long. All Rights Reserved.
  */
-package org.owasp.dependencycheck.data.nvdcve.xml;
+package org.owasp.dependencycheck.data.nvdcve;
 
 /**
  * An InvalidDataDataException is a generic exception used when trying to load

dependency-check-core/src/main/java/org/owasp/dependencycheck/data/nvdcve/NvdCve12Handler.java

@@ -16,7 +16,7 @@
  *
  * Copyright (c) 2012 Jeremy Long. All Rights Reserved.
  */
-package org.owasp.dependencycheck.data.nvdcve.xml;
+package org.owasp.dependencycheck.data.nvdcve;
 
 import java.util.ArrayList;
 import java.util.HashMap;

dependency-check-core/src/main/java/org/owasp/dependencycheck/data/nvdcve/NvdCve20Handler.java

@@ -16,7 +16,7 @@
  *
  * Copyright (c) 2012 Jeremy Long. All Rights Reserved.
  */
-package org.owasp.dependencycheck.data.nvdcve.xml;
+package org.owasp.dependencycheck.data.nvdcve;
 
 import java.io.IOException;
 import java.util.List;
@@ -24,9 +24,7 @@ import java.util.Map;
 import java.util.logging.Level;
 import java.util.logging.Logger;
 import org.apache.lucene.index.CorruptIndexException;
-import org.owasp.dependencycheck.data.cpe.Index;
-import org.owasp.dependencycheck.data.nvdcve.CveDB;
-import org.owasp.dependencycheck.data.nvdcve.DatabaseException;
+import org.owasp.dependencycheck.data.cpe.CpeIndexWriter;
 import org.owasp.dependencycheck.dependency.Reference;
 import org.owasp.dependencycheck.dependency.Vulnerability;
 import org.owasp.dependencycheck.dependency.VulnerableSoftware;
@@ -274,14 +272,14 @@ public class NvdCve20Handler extends DefaultHandler {
     /**
      * the cpe index.
      */
-    private Index cpeIndex;
+    private CpeIndexWriter cpeIndex;
 
     /**
-     * Sets the cpe index.
+     * Sets the cpe index writer.
      *
      * @param index the CPE Lucene Index
      */
-    void setCpeIndex(Index index) {
+    public void setCpeIndex(CpeIndexWriter index) {
         cpeIndex = index;
     }
 

dependency-check-core/src/main/java/org/owasp/dependencycheck/data/nvdcve/xml/DatabaseUpdater.java

@@ -1,671 +0,0 @@
-/*
- * This file is part of dependency-check-core.
- *
- * Dependency-check-core is free software: you can redistribute it and/or modify it
- * under the terms of the GNU General Public License as published by the Free
- * Software Foundation, either version 3 of the License, or (at your option) any
- * later version.
- *
- * Dependency-check-core is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
- * details.
- *
- * You should have received a copy of the GNU General Public License along with
- * dependency-check-core. If not, see http://www.gnu.org/licenses/.
- *
- * Copyright (c) 2012 Jeremy Long. All Rights Reserved.
- */
-package org.owasp.dependencycheck.data.nvdcve.xml;
-
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileNotFoundException;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.io.OutputStreamWriter;
-import javax.xml.parsers.ParserConfigurationException;
-import org.owasp.dependencycheck.data.CachedWebDataSource;
-import java.net.MalformedURLException;
-import java.net.URL;
-import java.sql.SQLException;
-import java.util.Calendar;
-import java.util.Date;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Properties;
-import java.util.logging.Level;
-import java.util.logging.Logger;
-import javax.xml.parsers.SAXParser;
-import javax.xml.parsers.SAXParserFactory;
-import org.owasp.dependencycheck.data.UpdateException;
-import org.owasp.dependencycheck.data.cpe.Index;
-import org.owasp.dependencycheck.data.nvdcve.CveDB;
-import org.owasp.dependencycheck.dependency.VulnerableSoftware;
-import org.owasp.dependencycheck.utils.DownloadFailedException;
-import org.owasp.dependencycheck.utils.Downloader;
-import org.owasp.dependencycheck.utils.FileUtils;
-import org.owasp.dependencycheck.utils.InvalidSettingException;
-import org.owasp.dependencycheck.utils.Settings;
-import org.xml.sax.SAXException;
-import org.owasp.dependencycheck.data.nvdcve.DatabaseException;
-
-/**
- *
- * @author Jeremy Long (jeremy.long@owasp.org)
- */
-public class DatabaseUpdater implements CachedWebDataSource {
-
-    /**
-     * The name of the properties file containing the timestamp of the last
-     * update.
-     */
-    private static final String UPDATE_PROPERTIES_FILE = "lastupdated.prop";
-    /**
-     * The properties file key for the last updated field - used to store the
-     * last updated time of the Modified NVD CVE xml file.
-     */
-    private static final String LAST_UPDATED_MODIFIED = "lastupdated.modified";
-    /**
-     * Stores the last updated time for each of the NVD CVE files. These
-     * timestamps should be updated if we process the modified file within 7
-     * days of the last update.
-     */
-    private static final String LAST_UPDATED_BASE = "lastupdated.";
-    /**
-     * Modified key word.
-     */
-    public static final String MODIFIED = "modified";
-    /**
-     * Reference to the Cve Database.
-     */
-    private CveDB cveDB = null;
-    /**
-     * Reference to the Cpe Index.
-     */
-    private Index cpeIndex = null;
-
-    /**
-     * <p>Downloads the latest NVD CVE XML file from the web and imports it into
-     * the current CVE Database.</p>
-     *
-     * @throws UpdateException is thrown if there is an error updating the
-     * database
-     */
-    public void update() throws UpdateException {
-        try {
-            final Map<String, NvdCveUrl> update = updateNeeded();
-            int maxUpdates = 0;
-            for (NvdCveUrl cve : update.values()) {
-                if (cve.getNeedsUpdate()) {
-                    maxUpdates += 1;
-                }
-            }
-            if (maxUpdates > 3) {
-                Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.INFO,
-                        "NVD CVE requires several updates; this could take a couple of minutes.");
-            }
-            if (maxUpdates > 0) {
-                openDataStores();
-            }
-            int count = 0;
-
-            for (NvdCveUrl cve : update.values()) {
-                if (cve.getNeedsUpdate()) {
-                    count += 1;
-                    Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.INFO,
-                            "Updating NVD CVE ({0} of {1})", new Object[]{count, maxUpdates});
-                    URL url = new URL(cve.getUrl());
-                    File outputPath = null;
-                    File outputPath12 = null;
-                    try {
-                        Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.INFO,
-                                "Downloading {0}", cve.getUrl());
-
-                        outputPath = File.createTempFile("cve" + cve.getId() + "_", ".xml");
-                        Downloader.fetchFile(url, outputPath, false);
-
-                        url = new URL(cve.getOldSchemaVersionUrl());
-                        outputPath12 = File.createTempFile("cve_1_2_" + cve.getId() + "_", ".xml");
-                        Downloader.fetchFile(url, outputPath12, false);
-
-                        Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.INFO,
-                                "Processing {0}", cve.getUrl());
-
-                        importXML(outputPath, outputPath12);
-
-                        cveDB.commit();
-                        cpeIndex.commit();
-
-                        writeLastUpdatedPropertyFile(cve);
-
-                        Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.INFO,
-                                "Completed update {0} of {1}", new Object[]{count, maxUpdates});
-                    } catch (FileNotFoundException ex) {
-                        throw new UpdateException(ex);
-                    } catch (ParserConfigurationException ex) {
-                        throw new UpdateException(ex);
-                    } catch (SAXException ex) {
-                        throw new UpdateException(ex);
-                    } catch (IOException ex) {
-                        throw new UpdateException(ex);
-                    } catch (SQLException ex) {
-                        throw new UpdateException(ex);
-                    } catch (DatabaseException ex) {
-                        throw new UpdateException(ex);
-                    } catch (ClassNotFoundException ex) {
-                        throw new UpdateException(ex);
-                    } finally {
-                        boolean deleted = false;
-                        try {
-                            if (outputPath != null && outputPath.exists()) {
-                                deleted = outputPath.delete();
-                            }
-                        } finally {
-                            if (outputPath != null && (outputPath.exists() || !deleted)) {
-                                outputPath.deleteOnExit();
-                            }
-                        }
-                        try {
-                            deleted = false;
-                            if (outputPath12 != null && outputPath12.exists()) {
-                                deleted = outputPath12.delete();
-                            }
-                        } finally {
-                            if (outputPath12 != null && (outputPath12.exists() || !deleted)) {
-                                outputPath12.deleteOnExit();
-                            }
-                        }
-                    }
-                }
-            }
-            if (maxUpdates >= 1) {
-                ensureModifiedIsInLastUpdatedProperties(update);
-                cveDB.cleanupDatabase();
-            }
-        } catch (MalformedURLException ex) {
-            throw new UpdateException(ex);
-        } catch (DownloadFailedException ex) {
-            throw new UpdateException(ex);
-        } finally {
-            closeDataStores();
-        }
-    }
-
-    /**
-     * Imports the NVD CVE XML File into the Lucene Index.
-     *
-     * @param file the file containing the NVD CVE XML
-     * @param oldVersion contains the file containing the NVD CVE XML 1.2
-     * @throws ParserConfigurationException is thrown if there is a parser
-     * configuration exception
-     * @throws SAXException is thrown if there is a SAXException
-     * @throws IOException is thrown if there is a ioexception
-     * @throws SQLException is thrown if there is a sql exception
-     * @throws DatabaseException is thrown if there is a database exception
-     * @throws ClassNotFoundException thrown if the h2 database driver cannot be
-     * loaded
-     */
-    private void importXML(File file, File oldVersion)
-            throws ParserConfigurationException, SAXException, IOException, SQLException, DatabaseException, ClassNotFoundException {
-
-        final SAXParserFactory factory = SAXParserFactory.newInstance();
-        final SAXParser saxParser = factory.newSAXParser();
-
-        final NvdCve12Handler cve12Handler = new NvdCve12Handler();
-        saxParser.parse(oldVersion, cve12Handler);
-        final Map<String, List<VulnerableSoftware>> prevVersionVulnMap = cve12Handler.getVulnerabilities();
-
-        final NvdCve20Handler cve20Handler = new NvdCve20Handler();
-        cve20Handler.setCveDB(cveDB);
-        cve20Handler.setPrevVersionVulnMap(prevVersionVulnMap);
-        cve20Handler.setCpeIndex(cpeIndex);
-        saxParser.parse(file, cve20Handler);
-    }
-
-    /**
-     * Closes the CVE and CPE data stores.
-     */
-    private void closeDataStores() {
-        if (cveDB != null) {
-            try {
-                cveDB.close();
-            } catch (Exception ignore) {
-                Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.FINEST, "Error closing the cveDB", ignore);
-            }
-        }
-        if (cpeIndex != null) {
-            try {
-                cpeIndex.close();
-            } catch (Exception ignore) {
-                Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.FINEST, "Error closing the cpeIndex", ignore);
-            }
-        }
-    }
-
-    /**
-     * Opens the CVE and CPE data stores.
-     *
-     * @throws UpdateException thrown if a data store cannot be opened
-     */
-    private void openDataStores() throws UpdateException {
-        //open the cve and cpe data stores
-        try {
-            cveDB = new CveDB();
-            cveDB.open();
-            cpeIndex = new Index();
-            cpeIndex.openIndexWriter();
-        } catch (IOException ex) {
-            closeDataStores();
-            Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.FINE, "IO Error opening databases", ex);
-            throw new UpdateException("Error updating the CPE/CVE data, please see the log file for more details.");
-        } catch (SQLException ex) {
-            closeDataStores();
-            Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.FINE, "SQL Exception opening databases", ex);
-            throw new UpdateException("Error updating the CPE/CVE data, please see the log file for more details.");
-        } catch (DatabaseException ex) {
-            closeDataStores();
-            Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.FINE, "Database Exception opening databases", ex);
-            throw new UpdateException("Error updating the CPE/CVE data, please see the log file for more details.");
-        } catch (ClassNotFoundException ex) {
-            closeDataStores();
-            Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.FINE, "Class not found exception opening databases", ex);
-            throw new UpdateException("Error updating the CPE/CVE data, please see the log file for more details.");
-        }
-    }
-
-    //<editor-fold defaultstate="collapsed" desc="Code to read/write properties files regarding the last update dates">
-    /**
-     * Writes a properties file containing the last updated date to the
-     * VULNERABLE_CPE directory.
-     *
-     * @param updatedValue the updated nvdcve entry
-     * @throws UpdateException is thrown if there is an update exception
-     */
-    private void writeLastUpdatedPropertyFile(NvdCveUrl updatedValue) throws UpdateException {
-        if (updatedValue == null) {
-            return;
-        }
-        String dir;
-        try {
-            dir = CveDB.getDataDirectory().getCanonicalPath();
-        } catch (IOException ex) {
-            Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.FINE, "Error updating the databases propterty file.", ex);
-            throw new UpdateException("Unable to locate last updated properties file.", ex);
-        }
-        final File cveProp = new File(dir, UPDATE_PROPERTIES_FILE);
-        final Properties prop = new Properties();
-        if (cveProp.exists()) {
-            FileInputStream in = null;
-            try {
-                in = new FileInputStream(cveProp);
-                prop.load(in);
-            } catch (Exception ignoreMe) {
-                Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.FINEST, null, ignoreMe);
-            } finally {
-                if (in != null) {
-                    try {
-                        in.close();
-                    } catch (Exception ignoreMeToo) {
-                        Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.FINEST, null, ignoreMeToo);
-                    }
-                }
-            }
-
-        }
-        prop.put("version", CveDB.DB_SCHEMA_VERSION);
-        prop.put(LAST_UPDATED_BASE + updatedValue.getId(), String.valueOf(updatedValue.getTimestamp()));
-
-        OutputStream os = null;
-        OutputStreamWriter out = null;
-        try {
-            os = new FileOutputStream(cveProp);
-            out = new OutputStreamWriter(os, "UTF-8");
-            prop.store(out, dir);
-        } catch (FileNotFoundException ex) {
-            Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.FINE, null, ex);
-            throw new UpdateException("Unable to find last updated properties file.", ex);
-        } catch (IOException ex) {
-            Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.FINE, null, ex);
-            throw new UpdateException("Unable to update last updated properties file.", ex);
-        } finally {
-            if (out != null) {
-                try {
-                    out.close();
-                } catch (IOException ex) {
-                    Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.FINEST, null, ex);
-                }
-            }
-            if (os != null) {
-                try {
-                    os.close();
-                } catch (IOException ex) {
-                    Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.FINEST, null, ex);
-                }
-            }
-        }
-    }
-
-    /**
-     * Determines if the index needs to be updated. This is done by fetching the
-     * nvd cve meta data and checking the last update date. If the data needs to
-     * be refreshed this method will return the NvdCveUrl for the files that
-     * need to be updated.
-     *
-     * @return the NvdCveUrl of the files that need to be updated.
-     * @throws MalformedURLException is thrown if the URL for the NVD CVE Meta
-     * data is incorrect.
-     * @throws DownloadFailedException is thrown if there is an error.
-     * downloading the nvd cve download data file.
-     * @throws UpdateException Is thrown if there is an issue with the last
-     * updated properties file.
-     */
-    public Map<String, NvdCveUrl> updateNeeded() throws MalformedURLException, DownloadFailedException, UpdateException {
-
-        Map<String, NvdCveUrl> currentlyPublished;
-        try {
-            currentlyPublished = retrieveCurrentTimestampsFromWeb();
-        } catch (InvalidDataException ex) {
-            final String msg = "Unable to retrieve valid timestamp from nvd cve downloads page";
-            Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.FINE, msg, ex);
-            throw new DownloadFailedException(msg, ex);
-
-        } catch (InvalidSettingException ex) {
-            Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.FINE, "Invalid setting found when retrieving timestamps", ex);
-            throw new DownloadFailedException("Invalid settings", ex);
-        }
-
-        if (currentlyPublished == null) {
-            throw new DownloadFailedException("Unable to retrieve valid timestamp from nvd cve downloads page");
-        }
-        String dir;
-        try {
-            dir = CveDB.getDataDirectory().getCanonicalPath();
-        } catch (IOException ex) {
-            Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.FINE, "CveDB data directory doesn't exist?", ex);
-            throw new UpdateException("Unable to locate last updated properties file.", ex);
-        }
-
-        final File f = new File(dir);
-        if (f.exists()) {
-            final File cveProp = new File(dir, UPDATE_PROPERTIES_FILE);
-            if (cveProp.exists()) {
-                final Properties prop = new Properties();
-                InputStream is = null;
-                try {
-                    is = new FileInputStream(cveProp);
-                    prop.load(is);
-
-                    boolean deleteAndRecreate = false;
-                    float version;
-
-                    if (prop.getProperty("version") == null) {
-                        deleteAndRecreate = true;
-                    } else {
-                        try {
-                            version = Float.parseFloat(prop.getProperty("version"));
-                            final float currentVersion = Float.parseFloat(CveDB.DB_SCHEMA_VERSION);
-                            if (currentVersion > version) {
-                                deleteAndRecreate = true;
-                            }
-                        } catch (NumberFormatException ex) {
-                            deleteAndRecreate = true;
-                        }
-                    }
-                    if (deleteAndRecreate) {
-                        Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.INFO, "The database version is old. Rebuilding the database.");
-                        is.close();
-                        //this is an old version of the lucene index - just delete it
-                        FileUtils.delete(f);
-
-                        //this importer also updates the CPE index and it is also using an old version
-                        final Index cpeId = new Index();
-                        final File cpeDir = cpeId.getDataDirectory();
-                        FileUtils.delete(cpeDir);
-                        return currentlyPublished;
-                    }
-
-                    final long lastUpdated = Long.parseLong(prop.getProperty(LAST_UPDATED_MODIFIED, "0"));
-                    final Date now = new Date();
-                    final int days = Settings.getInt(Settings.KEYS.CVE_MODIFIED_VALID_FOR_DAYS, 7);
-                    final int start = Settings.getInt(Settings.KEYS.CVE_START_YEAR, 2002);
-                    final int end = Calendar.getInstance().get(Calendar.YEAR);
-                    if (lastUpdated == currentlyPublished.get(MODIFIED).timestamp) {
-                        currentlyPublished.clear(); //we don't need to update anything.
-                    } else if (withinRange(lastUpdated, now.getTime(), days)) {
-                        currentlyPublished.get(MODIFIED).setNeedsUpdate(true);
-                        for (int i = start; i <= end; i++) {
-                            currentlyPublished.get(String.valueOf(i)).setNeedsUpdate(false);
-                        }
-                    } else { //we figure out which of the several XML files need to be downloaded.
-                        currentlyPublished.get(MODIFIED).setNeedsUpdate(false);
-                        for (int i = start; i <= end; i++) {
-                            final NvdCveUrl cve = currentlyPublished.get(String.valueOf(i));
-                            long currentTimestamp = 0;
-                            try {
-                                currentTimestamp = Long.parseLong(prop.getProperty(LAST_UPDATED_BASE + String.valueOf(i), "0"));
-                            } catch (NumberFormatException ex) {
-                                final String msg = String.format("Error parsing '%s' '%s' from nvdcve.lastupdated",
-                                        LAST_UPDATED_BASE, String.valueOf(i));
-                                Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.FINE, msg, ex);
-                            }
-                            if (currentTimestamp == cve.getTimestamp()) {
-                                cve.setNeedsUpdate(false); //they default to true.
-                            }
-                        }
-                    }
-                } catch (FileNotFoundException ex) {
-                    Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.FINEST, null, ex);
-                } catch (IOException ex) {
-                    Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.FINEST, null, ex);
-                } catch (NumberFormatException ex) {
-                    Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.FINEST, null, ex);
-                } finally {
-                    if (is != null) {
-                        try {
-                            is.close();
-                        } catch (IOException ex) {
-                            Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.FINEST, null, ex);
-                        }
-                    }
-                }
-            }
-        }
-        return currentlyPublished;
-    }
-
-    /**
-     * Determines if the epoch date is within the range specified of the
-     * compareTo epoch time. This takes the (compareTo-date)/1000/60/60/24 to
-     * get the number of days. If the calculated days is less then the range the
-     * date is considered valid.
-     *
-     * @param date the date to be checked.
-     * @param compareTo the date to compare to.
-     * @param range the range in days to be considered valid.
-     * @return whether or not the date is within the range.
-     */
-    private boolean withinRange(long date, long compareTo, int range) {
-        final double differenceInDays = (compareTo - date) / 1000.0 / 60.0 / 60.0 / 24.0;
-        return differenceInDays < range;
-    }
-
-    /**
-     * Retrieves the timestamps from the NVD CVE meta data file.
-     *
-     * @return the timestamp from the currently published nvdcve downloads page
-     * @throws MalformedURLException thrown if the URL for the NVD CCE Meta data
-     * is incorrect.
-     * @throws DownloadFailedException thrown if there is an error downloading
-     * the nvd cve meta data file
-     * @throws InvalidDataException thrown if there is an exception parsing the
-     * timestamps
-     * @throws InvalidSettingException thrown if the settings are invalid
-     */
-    protected Map<String, NvdCveUrl> retrieveCurrentTimestampsFromWeb()
-            throws MalformedURLException, DownloadFailedException, InvalidDataException, InvalidSettingException {
-
-        final Map<String, NvdCveUrl> map = new HashMap<String, NvdCveUrl>();
-        String retrieveUrl = Settings.getString(Settings.KEYS.CVE_MODIFIED_20_URL);
-
-        NvdCveUrl item = new NvdCveUrl();
-        item.setNeedsUpdate(false); //the others default to true, to make life easier later this should default to false.
-        item.setId(MODIFIED);
-        item.setUrl(retrieveUrl);
-        item.setOldSchemaVersionUrl(Settings.getString(Settings.KEYS.CVE_MODIFIED_12_URL));
-
-        item.timestamp = Downloader.getLastModified(new URL(retrieveUrl));
-        map.put(MODIFIED, item);
-
-        final int start = Settings.getInt(Settings.KEYS.CVE_START_YEAR);
-        final int end = Calendar.getInstance().get(Calendar.YEAR);
-        final String baseUrl20 = Settings.getString(Settings.KEYS.CVE_SCHEMA_2_0);
-        final String baseUrl12 = Settings.getString(Settings.KEYS.CVE_SCHEMA_1_2);
-        for (int i = start; i <= end; i++) {
-            retrieveUrl = String.format(baseUrl20, i);
-            item = new NvdCveUrl();
-            item.setId(Integer.toString(i));
-            item.setUrl(retrieveUrl);
-            item.setOldSchemaVersionUrl(String.format(baseUrl12, i));
-            item.setTimestamp(Downloader.getLastModified(new URL(retrieveUrl)));
-            map.put(item.id, item);
-        }
-        return map;
-    }
-
-    /**
-     * Method to double check that the "modified" nvdcve file is listed and has
-     * a timestamp in the last updated properties file.
-     *
-     * @param update a set of updated NvdCveUrl objects
-     */
-    private void ensureModifiedIsInLastUpdatedProperties(Map<String, NvdCveUrl> update) {
-        try {
-            writeLastUpdatedPropertyFile(update.get(MODIFIED));
-        } catch (UpdateException ex) {
-            Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.FINE, null, ex);
-        }
-    }
-
-    /**
-     * A pojo that contains the Url and timestamp of the current NvdCve XML
-     * files.
-     */
-    protected static class NvdCveUrl {
-
-        /**
-         * an id.
-         */
-        private String id;
-
-        /**
-         * Get the value of id.
-         *
-         * @return the value of id
-         */
-        public String getId() {
-            return id;
-        }
-
-        /**
-         * Set the value of id.
-         *
-         * @param id new value of id
-         */
-        public void setId(String id) {
-            this.id = id;
-        }
-        /**
-         * a url.
-         */
-        private String url;
-
-        /**
-         * Get the value of url.
-         *
-         * @return the value of url
-         */
-        public String getUrl() {
-            return url;
-        }
-
-        /**
-         * Set the value of url.
-         *
-         * @param url new value of url
-         */
-        public void setUrl(String url) {
-            this.url = url;
-        }
-        /**
-         * The 1.2 schema URL.
-         */
-        private String oldSchemaVersionUrl;
-
-        /**
-         * Get the value of oldSchemaVersionUrl.
-         *
-         * @return the value of oldSchemaVersionUrl
-         */
-        public String getOldSchemaVersionUrl() {
-            return oldSchemaVersionUrl;
-        }
-
-        /**
-         * Set the value of oldSchemaVersionUrl.
-         *
-         * @param oldSchemaVersionUrl new value of oldSchemaVersionUrl
-         */
-        public void setOldSchemaVersionUrl(String oldSchemaVersionUrl) {
-            this.oldSchemaVersionUrl = oldSchemaVersionUrl;
-        }
-        /**
-         * a timestamp - epoch time.
-         */
-        private long timestamp;
-
-        /**
-         * Get the value of timestamp - epoch time.
-         *
-         * @return the value of timestamp - epoch time
-         */
-        public long getTimestamp() {
-            return timestamp;
-        }
-
-        /**
-         * Set the value of timestamp - epoch time.
-         *
-         * @param timestamp new value of timestamp - epoch time
-         */
-        public void setTimestamp(long timestamp) {
-            this.timestamp = timestamp;
-        }
-        /**
-         * indicates whether or not this item should be updated.
-         */
-        private boolean needsUpdate = true;
-
-        /**
-         * Get the value of needsUpdate.
-         *
-         * @return the value of needsUpdate
-         */
-        public boolean getNeedsUpdate() {
-            return needsUpdate;
-        }
-
-        /**
-         * Set the value of needsUpdate.
-         *
-         * @param needsUpdate new value of needsUpdate
-         */
-        public void setNeedsUpdate(boolean needsUpdate) {
-            this.needsUpdate = needsUpdate;
-        }
-    }
-    //</editor-fold>
-}

dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/DataStoreMetaInfo.java

@@ -0,0 +1,241 @@
+/*
+ * This file is part of dependency-check-core.
+ *
+ * Dependency-check-core is free software: you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation, either version 3 of the License, or (at your option) any
+ * later version.
+ *
+ * Dependency-check-core is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+ * details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * dependency-check-core. If not, see http://www.gnu.org/licenses/.
+ *
+ * Copyright (c) 2013 Jeremy Long. All Rights Reserved.
+ */
+package org.owasp.dependencycheck.data.update;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.io.OutputStreamWriter;
+import java.util.Properties;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+import org.owasp.dependencycheck.data.UpdateException;
+import org.owasp.dependencycheck.data.nvdcve.CveDB;
+import org.owasp.dependencycheck.utils.Settings;
+
+/**
+ *
+ * @author Jeremy Long (jeremy.long@owasp.org)
+ */
+public class DataStoreMetaInfo {
+
+    /**
+     * Batch key word, used as key to store information about batch mode.
+     */
+    public static final String BATCH = "batch";
+    /**
+     * Modified key word, used as a key to store information about the modified
+     * file (i.e. the containing the last 8 days of updates)..
+     */
+    public static final String MODIFIED = "modified";
+    /**
+     * The name of the properties file containing the timestamp of the last
+     * update.
+     */
+    private static final String UPDATE_PROPERTIES_FILE = "data.properties";
+    /**
+     * The properties file key for the last updated field - used to store the
+     * last updated time of the Modified NVD CVE xml file.
+     */
+    public static final String LAST_UPDATED = "lastupdated.modified";
+    /**
+     * Stores the last updated time for each of the NVD CVE files. These
+     * timestamps should be updated if we process the modified file within 7
+     * days of the last update.
+     */
+    public static final String LAST_UPDATED_BASE = "lastupdated.";
+    /**
+     * A collection of properties about the data.
+     */
+    private Properties properties = new Properties();
+    /**
+     * Indicates whether or not the updates are using a batch update mode or
+     * not.
+     */
+    private boolean batchUpdateMode;
+
+    /**
+     * Get the value of batchUpdateMode.
+     *
+     * @return the value of batchUpdateMode
+     */
+    protected boolean isBatchUpdateMode() {
+        return batchUpdateMode;
+    }
+
+    /**
+     * Set the value of batchUpdateMode.
+     *
+     * @param batchUpdateMode new value of batchUpdateMode
+     */
+    protected void setBatchUpdateMode(boolean batchUpdateMode) {
+        this.batchUpdateMode = batchUpdateMode;
+    }
+
+    /**
+     * Constructs a new data properties object.
+     */
+    public DataStoreMetaInfo() {
+        batchUpdateMode = !Settings.getString(Settings.KEYS.BATCH_UPDATE_URL, "").isEmpty();
+        loadProperties();
+    }
+
+    /**
+     * Loads the data's meta properties.
+     */
+    private void loadProperties() {
+        final File file = getPropertiesFile();
+        if (file.exists()) {
+            InputStream is = null;
+            try {
+                is = new FileInputStream(file);
+            } catch (FileNotFoundException ignore) {
+                //we will never get here as we check for existence above.
+                Logger.getLogger(DataStoreMetaInfo.class.getName()).log(Level.FINEST, null, ignore);
+            }
+            try {
+                properties.load(is);
+            } catch (IOException ex) {
+                final String msg = String.format("Unable to load properties file '%s'", file.getPath());
+                Logger.getLogger(DataStoreMetaInfo.class.getName()).log(Level.WARNING, msg);
+                Logger.getLogger(DataStoreMetaInfo.class.getName()).log(Level.FINE, null, ex);
+            } finally {
+                if (is != null) {
+                    try {
+                        is.close();
+                    } catch (IOException ex) {
+                        final String msg = String.format("Unable to close properties file '%s'", file.getPath());
+                        Logger.getLogger(DataStoreMetaInfo.class.getName()).log(Level.WARNING, msg);
+                        Logger.getLogger(DataStoreMetaInfo.class.getName()).log(Level.FINE, null, ex);
+                    }
+                }
+            }
+        }
+    }
+
+    /**
+     * Returns whether or not any properties are set.
+     *
+     * @return whether or not any properties are set
+     */
+    public boolean isEmpty() {
+        return properties.isEmpty();
+    }
+
+    /**
+     * Writes a properties file containing the last updated date to the
+     * VULNERABLE_CPE directory.
+     *
+     * @param updatedValue the updated nvdcve entry
+     * @throws UpdateException is thrown if there is an update exception
+     */
+    public void save(NvdCveInfo updatedValue) throws UpdateException {
+        if (updatedValue == null) {
+            return;
+        }
+        final File cveProp = getPropertiesFile();
+        final Properties prop = new Properties();
+        if (cveProp.exists()) {
+            FileInputStream in = null;
+            try {
+                in = new FileInputStream(cveProp);
+                prop.load(in);
+            } catch (Exception ignoreMe) {
+                Logger.getLogger(DataStoreMetaInfo.class.getName()).log(Level.FINEST, null, ignoreMe);
+            } finally {
+                if (in != null) {
+                    try {
+                        in.close();
+                    } catch (Exception ignoreMeToo) {
+                        Logger.getLogger(DataStoreMetaInfo.class.getName()).log(Level.FINEST, null, ignoreMeToo);
+                    }
+                }
+            }
+        }
+        prop.put("version", CveDB.DB_SCHEMA_VERSION);
+        prop.put(LAST_UPDATED_BASE + updatedValue.getId(), String.valueOf(updatedValue.getTimestamp()));
+
+        OutputStream os = null;
+        OutputStreamWriter out = null;
+        try {
+            os = new FileOutputStream(cveProp);
+            out = new OutputStreamWriter(os, "UTF-8");
+            prop.store(out, "Meta data about data and data sources used by dependency-check");
+        } catch (FileNotFoundException ex) {
+            Logger.getLogger(DataStoreMetaInfo.class.getName()).log(Level.FINE, null, ex);
+            throw new UpdateException("Unable to find last updated properties file.", ex);
+        } catch (IOException ex) {
+            Logger.getLogger(DataStoreMetaInfo.class.getName()).log(Level.FINE, null, ex);
+            throw new UpdateException("Unable to update last updated properties file.", ex);
+        } finally {
+            if (out != null) {
+                try {
+                    out.close();
+                } catch (IOException ex) {
+                    Logger.getLogger(DataStoreMetaInfo.class.getName()).log(Level.FINEST, null, ex);
+                }
+            }
+            if (os != null) {
+                try {
+                    os.close();
+                } catch (IOException ex) {
+                    Logger.getLogger(DataStoreMetaInfo.class.getName()).log(Level.FINEST, null, ex);
+                }
+            }
+        }
+    }
+
+    /**
+     * Returns the property value for the given key. If the key is not contained
+     * in the underlying properties null is returned.
+     *
+     * @param key the property key
+     * @return the value of the property
+     */
+    public String getProperty(String key) {
+        return properties.getProperty(key);
+    }
+
+    /**
+     * Returns the property value for the given key. If the key is not contained
+     * in the underlying properties the default value is returned.
+     *
+     * @param key the property key
+     * @param defaultValue the default value
+     * @return the value of the property
+     */
+    public String getProperty(String key, String defaultValue) {
+        return properties.getProperty(key, defaultValue);
+    }
+
+    /**
+     * Retrieves the properties file.
+     *
+     * @return the properties file
+     */
+    public static File getPropertiesFile() {
+        final File dataDirectory = Settings.getFile(Settings.KEYS.DATA_DIRECTORY);
+        final File file = new File(dataDirectory, UPDATE_PROPERTIES_FILE);
+        return file;
+    }
+}

dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/DatabaseUpdater.java

@@ -0,0 +1,572 @@
+/*
+ * This file is part of dependency-check-core.
+ *
+ * Dependency-check-core is free software: you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation, either version 3 of the License, or (at your option) any
+ * later version.
+ *
+ * Dependency-check-core is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+ * details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * dependency-check-core. If not, see http://www.gnu.org/licenses/.
+ *
+ * Copyright (c) 2012 Jeremy Long. All Rights Reserved.
+ */
+package org.owasp.dependencycheck.data.update;
+
+import org.owasp.dependencycheck.data.nvdcve.NvdCve12Handler;
+import org.owasp.dependencycheck.data.nvdcve.NvdCve20Handler;
+import org.owasp.dependencycheck.data.nvdcve.InvalidDataException;
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import javax.xml.parsers.ParserConfigurationException;
+import org.xml.sax.SAXException;
+import org.owasp.dependencycheck.data.CachedWebDataSource;
+import java.net.MalformedURLException;
+import java.net.URISyntaxException;
+import java.net.URL;
+import java.sql.SQLException;
+import java.util.Calendar;
+import java.util.Date;
+import java.util.List;
+import java.util.Map;
+import java.util.TreeMap;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+import javax.xml.parsers.SAXParser;
+import javax.xml.parsers.SAXParserFactory;
+import org.owasp.dependencycheck.data.UpdateException;
+import org.owasp.dependencycheck.data.cpe.CpeIndexWriter;
+import org.owasp.dependencycheck.data.nvdcve.CveDB;
+import org.owasp.dependencycheck.dependency.VulnerableSoftware;
+import org.owasp.dependencycheck.utils.DownloadFailedException;
+import org.owasp.dependencycheck.utils.Downloader;
+import org.owasp.dependencycheck.utils.FileUtils;
+import org.owasp.dependencycheck.utils.Settings;
+import org.owasp.dependencycheck.data.nvdcve.DatabaseException;
+import org.owasp.dependencycheck.utils.InvalidSettingException;
+import static org.owasp.dependencycheck.data.update.DataStoreMetaInfo.BATCH;
+import static org.owasp.dependencycheck.data.update.DataStoreMetaInfo.MODIFIED;
+
+/**
+ * Class responsible for updating the CPE and NVDCVE data stores.
+ *
+ * @author Jeremy Long (jeremy.long@owasp.org)
+ */
+public class DatabaseUpdater implements CachedWebDataSource {
+
+    /**
+     * Utility to read and write meta-data about the data.
+     */
+    private DataStoreMetaInfo properties = null;
+    /**
+     * Reference to the Cve Database.
+     */
+    private CveDB cveDB = null;
+    /**
+     * Reference to the Cpe Index.
+     */
+    private CpeIndexWriter cpeIndex = null;
+    /**
+     * A flag indicating whether or not the batch update should be performed.
+     */
+    private boolean doBatchUpdate;
+
+    /**
+     * Get the value of doBatchUpdate
+     *
+     * @return the value of doBatchUpdate
+     */
+    protected boolean isDoBatchUpdate() {
+        return doBatchUpdate;
+    }
+
+    /**
+     * Set the value of doBatchUpdate
+     *
+     * @param doBatchUpdate new value of doBatchUpdate
+     */
+    protected void setDoBatchUpdate(boolean doBatchUpdate) {
+        this.doBatchUpdate = doBatchUpdate;
+    }
+
+    /**
+     * <p>Downloads the latest NVD CVE XML file from the web and imports it into
+     * the current CVE Database.</p>
+     *
+     * @throws UpdateException is thrown if there is an error updating the
+     * database
+     */
+    @Override
+    public void update() throws UpdateException {
+        doBatchUpdate = false;
+        properties = new DataStoreMetaInfo();
+        try {
+            final Map<String, NvdCveInfo> update = updateNeeded();
+            int maxUpdates = 0;
+            for (NvdCveInfo cve : update.values()) {
+                if (cve.getNeedsUpdate()) {
+                    maxUpdates += 1;
+                }
+            }
+            if (maxUpdates > 3 && !properties.isBatchUpdateMode()) {
+                Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.INFO,
+                        "NVD CVE requires several updates; this could take a couple of minutes.");
+            }
+            if (maxUpdates > 0 && !isDoBatchUpdate()) {
+                openDataStores();
+            }
+
+            if (properties.isBatchUpdateMode() && isDoBatchUpdate()) {
+                try {
+                    performBatchUpdate();
+                    openDataStores();
+                } catch (IOException ex) {
+                    throw new UpdateException("Unable to perform batch update", ex);
+                }
+            }
+
+            int count = 0;
+            for (NvdCveInfo cve : update.values()) {
+                if (cve.getNeedsUpdate()) {
+                    count += 1;
+                    Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.INFO,
+                            "Updating NVD CVE ({0} of {1})", new Object[]{count, maxUpdates});
+                    URL url = new URL(cve.getUrl());
+                    File outputPath = null;
+                    File outputPath12 = null;
+                    try {
+                        Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.INFO,
+                                "Downloading {0}", cve.getUrl());
+                        outputPath = File.createTempFile("cve" + cve.getId() + "_", ".xml");
+                        Downloader.fetchFile(url, outputPath);
+
+                        url = new URL(cve.getOldSchemaVersionUrl());
+                        outputPath12 = File.createTempFile("cve_1_2_" + cve.getId() + "_", ".xml");
+                        Downloader.fetchFile(url, outputPath12);
+
+                        Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.INFO,
+                                "Processing {0}", cve.getUrl());
+
+                        importXML(outputPath, outputPath12);
+
+                        cveDB.commit();
+                        cpeIndex.commit();
+
+                        properties.save(cve);
+
+                        Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.INFO,
+                                "Completed update {0} of {1}", new Object[]{count, maxUpdates});
+                    } catch (FileNotFoundException ex) {
+                        throw new UpdateException(ex);
+                    } catch (ParserConfigurationException ex) {
+                        throw new UpdateException(ex);
+                    } catch (SAXException ex) {
+                        throw new UpdateException(ex);
+                    } catch (IOException ex) {
+                        throw new UpdateException(ex);
+                    } catch (SQLException ex) {
+                        throw new UpdateException(ex);
+                    } catch (DatabaseException ex) {
+                        throw new UpdateException(ex);
+                    } catch (ClassNotFoundException ex) {
+                        throw new UpdateException(ex);
+                    } finally {
+                        boolean deleted = false;
+                        try {
+                            if (outputPath != null && outputPath.exists()) {
+                                deleted = outputPath.delete();
+                            }
+                        } finally {
+                            if (outputPath != null && (outputPath.exists() || !deleted)) {
+                                outputPath.deleteOnExit();
+                            }
+                        }
+                        try {
+                            deleted = false;
+                            if (outputPath12 != null && outputPath12.exists()) {
+                                deleted = outputPath12.delete();
+                            }
+                        } finally {
+                            if (outputPath12 != null && (outputPath12.exists() || !deleted)) {
+                                outputPath12.deleteOnExit();
+                            }
+                        }
+                    }
+                }
+            }
+            if (maxUpdates >= 1) { //ensure the modified file date gets written
+                properties.save(update.get(MODIFIED));
+                cveDB.cleanupDatabase();
+            }
+            if (update.get(BATCH) != null) {
+                properties.save(update.get(BATCH));
+            }
+        } catch (MalformedURLException ex) {
+            throw new UpdateException(ex);
+        } catch (DownloadFailedException ex) {
+            throw new UpdateException(ex);
+        } finally {
+            closeDataStores();
+        }
+    }
+
+    /**
+     * Imports the NVD CVE XML File into the Lucene Index.
+     *
+     * @param file the file containing the NVD CVE XML
+     * @param oldVersion contains the file containing the NVD CVE XML 1.2
+     * @throws ParserConfigurationException is thrown if there is a parser
+     * configuration exception
+     * @throws SAXException is thrown if there is a SAXException
+     * @throws IOException is thrown if there is a IO Exception
+     * @throws SQLException is thrown if there is a SQL exception
+     * @throws DatabaseException is thrown if there is a database exception
+     * @throws ClassNotFoundException thrown if the h2 database driver cannot be
+     * loaded
+     */
+    private void importXML(File file, File oldVersion)
+            throws ParserConfigurationException, SAXException, IOException, SQLException, DatabaseException, ClassNotFoundException {
+
+        final SAXParserFactory factory = SAXParserFactory.newInstance();
+        final SAXParser saxParser = factory.newSAXParser();
+
+        final NvdCve12Handler cve12Handler = new NvdCve12Handler();
+        saxParser.parse(oldVersion, cve12Handler);
+        final Map<String, List<VulnerableSoftware>> prevVersionVulnMap = cve12Handler.getVulnerabilities();
+
+        final NvdCve20Handler cve20Handler = new NvdCve20Handler();
+        cve20Handler.setCveDB(cveDB);
+        cve20Handler.setPrevVersionVulnMap(prevVersionVulnMap);
+        cve20Handler.setCpeIndex(cpeIndex);
+        saxParser.parse(file, cve20Handler);
+    }
+
+    /**
+     * Deletes the existing data directories.
+     *
+     * @throws IOException thrown if the directory cannot be deleted
+     */
+    protected void deleteExistingData() throws IOException {
+        File data = Settings.getFile(Settings.KEYS.CVE_DATA_DIRECTORY);
+        if (data.exists()) {
+            FileUtils.delete(data);
+        }
+        data = Settings.getFile(Settings.KEYS.CPE_DATA_DIRECTORY);
+        if (data.exists()) {
+            FileUtils.delete(data);
+        }
+        data = DataStoreMetaInfo.getPropertiesFile();
+        if (data.exists()) {
+            FileUtils.delete(data);
+        }
+    }
+
+    /**
+     * Performs the batch update based on the configured batch update URL.
+     *
+     * @throws UpdateException thrown if there is an exception during the update
+     * process
+     */
+    private void performBatchUpdate() throws UpdateException {
+        if (properties.isBatchUpdateMode() && doBatchUpdate) {
+            final String batchSrc = Settings.getString(Settings.KEYS.BATCH_UPDATE_URL);
+            File tmp = null;
+            try {
+                deleteExistingData();
+                final File dataDirectory = CveDB.getDataDirectory().getParentFile();
+                final URL batchUrl = new URL(batchSrc);
+                if ("file".equals(batchUrl.getProtocol())) {
+                    try {
+                        tmp = new File(batchUrl.toURI());
+                    } catch (URISyntaxException ex) {
+                        final String msg = String.format("Invalid batch update URI: %s", batchSrc);
+                        throw new UpdateException(msg, ex);
+                    }
+                } else if ("http".equals(batchUrl.getProtocol())
+                        || "https".equals(batchUrl.getProtocol())) {
+                    tmp = File.createTempFile("batch_", ".zip");
+                    Downloader.fetchFile(batchUrl, tmp);
+                }
+                //TODO add FTP?
+                FileUtils.extractFiles(tmp, dataDirectory);
+
+            } catch (IOException ex) {
+                final String msg = String.format("IO Exception Occured performing batch update using: %s", batchSrc);
+                throw new UpdateException(msg, ex);
+            } finally {
+                if (tmp != null && !tmp.delete()) {
+                    tmp.deleteOnExit();
+                }
+            }
+        }
+    }
+
+    /**
+     * Closes the CVE and CPE data stores.
+     */
+    private void closeDataStores() {
+        if (cveDB != null) {
+            try {
+                cveDB.close();
+            } catch (Exception ignore) {
+                Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.FINEST, "Error closing the cveDB", ignore);
+            }
+        }
+        if (cpeIndex != null) {
+            try {
+                cpeIndex.close();
+            } catch (Exception ignore) {
+                Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.FINEST, "Error closing the cpeIndex", ignore);
+            }
+        }
+    }
+
+    /**
+     * Opens the CVE and CPE data stores.
+     *
+     * @throws UpdateException thrown if a data store cannot be opened
+     */
+    private void openDataStores() throws UpdateException {
+        //open the cve and cpe data stores
+        try {
+            cveDB = new CveDB();
+            cveDB.open();
+            cpeIndex = new CpeIndexWriter();
+            cpeIndex.open();
+        } catch (IOException ex) {
+            closeDataStores();
+            Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.FINE, "IO Error opening databases", ex);
+            throw new UpdateException("Error updating the CPE/CVE data, please see the log file for more details.");
+        } catch (SQLException ex) {
+            closeDataStores();
+            Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.FINE, "SQL Exception opening databases", ex);
+            throw new UpdateException("Error updating the CPE/CVE data, please see the log file for more details.");
+        } catch (DatabaseException ex) {
+            closeDataStores();
+            Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.FINE, "Database Exception opening databases", ex);
+            throw new UpdateException("Error updating the CPE/CVE data, please see the log file for more details.");
+        } catch (ClassNotFoundException ex) {
+            closeDataStores();
+            Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.FINE, "Class not found exception opening databases", ex);
+            throw new UpdateException("Error updating the CPE/CVE data, please see the log file for more details.");
+        }
+    }
+
+    /**
+     * Determines if the index needs to be updated. This is done by fetching the
+     * NVD CVE meta data and checking the last update date. If the data needs to
+     * be refreshed this method will return the NvdCveUrl for the files that
+     * need to be updated.
+     *
+     * @return the NvdCveUrl of the files that need to be updated.
+     * @throws MalformedURLException is thrown if the URL for the NVD CVE Meta
+     * data is incorrect.
+     * @throws DownloadFailedException is thrown if there is an error.
+     * downloading the NVD CVE download data file.
+     * @throws UpdateException Is thrown if there is an issue with the last
+     * updated properties file.
+     */
+    private Map<String, NvdCveInfo> updateNeeded() throws MalformedURLException, DownloadFailedException, UpdateException {
+
+        Map<String, NvdCveInfo> currentlyPublished;
+        try {
+            currentlyPublished = retrieveCurrentTimestampsFromWeb();
+        } catch (InvalidDataException ex) {
+            final String msg = "Unable to retrieve valid timestamp from nvd cve downloads page";
+            Logger.getLogger(DataStoreMetaInfo.class.getName()).log(Level.FINE, msg, ex);
+            throw new DownloadFailedException(msg, ex);
+        } catch (InvalidSettingException ex) {
+            Logger.getLogger(DataStoreMetaInfo.class.getName()).log(Level.FINE, "Invalid setting found when retrieving timestamps", ex);
+            throw new DownloadFailedException("Invalid settings", ex);
+        }
+
+        if (currentlyPublished == null) {
+            throw new DownloadFailedException("Unable to retrieve the timestamps of the currently published NVD CVE data");
+        }
+
+//        final File cpeDataDirectory;
+//        try {
+//            cpeDataDirectory = CveDB.getDataDirectory();
+//        } catch (IOException ex) {
+//            String msg;
+//            try {
+//                msg = String.format("Unable to create the CVE Data Directory '%s'",
+//                        Settings.getFile(Settings.KEYS.CVE_DATA_DIRECTORY).getCanonicalPath());
+//            } catch (IOException ex1) {
+//                msg = String.format("Unable to create the CVE Data Directory, this is likely a configuration issue: '%s%s%s'",
+//                        Settings.getString(Settings.KEYS.DATA_DIRECTORY, ""),
+//                        File.separator,
+//                        Settings.getString(Settings.KEYS.CVE_DATA_DIRECTORY, ""));
+//            }
+//            throw new UpdateException(msg, ex);
+//        }
+
+        if (!properties.isEmpty()) {
+            try {
+                boolean deleteAndRecreate = false;
+                float version;
+
+                if (properties.getProperty("version") == null) {
+                    deleteAndRecreate = true;
+                } else {
+                    try {
+                        version = Float.parseFloat(properties.getProperty("version"));
+                        final float currentVersion = Float.parseFloat(CveDB.DB_SCHEMA_VERSION);
+                        if (currentVersion > version) {
+                            deleteAndRecreate = true;
+                        }
+                    } catch (NumberFormatException ex) {
+                        deleteAndRecreate = true;
+                    }
+                }
+
+                final NvdCveInfo batchInfo = currentlyPublished.get(BATCH);
+                if (properties.isBatchUpdateMode() && batchInfo != null) {
+                    final long lastUpdated = Long.parseLong(properties.getProperty(DataStoreMetaInfo.BATCH, "0"));
+                    if (lastUpdated != batchInfo.getTimestamp()) {
+                        deleteAndRecreate = true;
+                    }
+                }
+
+                if (deleteAndRecreate) {
+                    setDoBatchUpdate(properties.isBatchUpdateMode());
+                    try {
+                        deleteExistingData();
+                    } catch (IOException ex) {
+                        final String msg = "Unable to delete existing data";
+                        Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.WARNING, msg);
+                        Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.FINE, null, ex);
+                    }
+                    return currentlyPublished;
+                }
+
+                final long lastUpdated = Long.parseLong(properties.getProperty(DataStoreMetaInfo.LAST_UPDATED, "0"));
+                final Date now = new Date();
+                final int days = Settings.getInt(Settings.KEYS.CVE_MODIFIED_VALID_FOR_DAYS, 7);
+                final int start = Settings.getInt(Settings.KEYS.CVE_START_YEAR, 2002);
+                final int end = Calendar.getInstance().get(Calendar.YEAR);
+                if (lastUpdated == currentlyPublished.get(MODIFIED).getTimestamp()) {
+                    currentlyPublished.clear(); //we don't need to update anything.
+                    setDoBatchUpdate(properties.isBatchUpdateMode());
+                } else if (withinRange(lastUpdated, now.getTime(), days)) {
+                    currentlyPublished.get(MODIFIED).setNeedsUpdate(true);
+                    if (properties.isBatchUpdateMode()) {
+                        setDoBatchUpdate(false);
+                    } else {
+                        for (int i = start; i <= end; i++) {
+                            currentlyPublished.get(String.valueOf(i)).setNeedsUpdate(false);
+                        }
+                    }
+                } else if (properties.isBatchUpdateMode()) {
+                    currentlyPublished.get(MODIFIED).setNeedsUpdate(true);
+                    setDoBatchUpdate(true);
+                } else { //we figure out which of the several XML files need to be downloaded.
+                    currentlyPublished.get(MODIFIED).setNeedsUpdate(false);
+                    for (int i = start; i <= end; i++) {
+                        final NvdCveInfo cve = currentlyPublished.get(String.valueOf(i));
+                        long currentTimestamp = 0;
+                        try {
+                            currentTimestamp = Long.parseLong(properties.getProperty(DataStoreMetaInfo.LAST_UPDATED_BASE + String.valueOf(i), "0"));
+                        } catch (NumberFormatException ex) {
+                            final String msg = String.format("Error parsing '%s' '%s' from nvdcve.lastupdated",
+                                    DataStoreMetaInfo.LAST_UPDATED_BASE, String.valueOf(i));
+                            Logger.getLogger(DataStoreMetaInfo.class.getName()).log(Level.FINE, msg, ex);
+                        }
+                        if (currentTimestamp == cve.getTimestamp()) {
+                            cve.setNeedsUpdate(false); //they default to true.
+                        }
+                    }
+                }
+            } catch (NumberFormatException ex) {
+                final String msg = "An invalid schema version or timestamp exists in the data.properties file.";
+                Logger.getLogger(DataStoreMetaInfo.class.getName()).log(Level.WARNING, msg);
+                Logger.getLogger(DataStoreMetaInfo.class.getName()).log(Level.FINE, null, ex);
+                setDoBatchUpdate(properties.isBatchUpdateMode());
+            }
+        } else {
+            setDoBatchUpdate(properties.isBatchUpdateMode());
+        }
+        return currentlyPublished;
+    }
+
+    /**
+     * Determines if the epoch date is within the range specified of the
+     * compareTo epoch time. This takes the (compareTo-date)/1000/60/60/24 to
+     * get the number of days. If the calculated days is less then the range the
+     * date is considered valid.
+     *
+     * @param date the date to be checked.
+     * @param compareTo the date to compare to.
+     * @param range the range in days to be considered valid.
+     * @return whether or not the date is within the range.
+     */
+    private boolean withinRange(long date, long compareTo, int range) {
+        final double differenceInDays = (compareTo - date) / 1000.0 / 60.0 / 60.0 / 24.0;
+        return differenceInDays < range;
+    }
+
+    /**
+     * Retrieves the timestamps from the NVD CVE meta data file.
+     *
+     * @return the timestamp from the currently published nvdcve downloads page
+     * @throws MalformedURLException thrown if the URL for the NVD CCE Meta data
+     * is incorrect.
+     * @throws DownloadFailedException thrown if there is an error downloading
+     * the nvd cve meta data file
+     * @throws InvalidDataException thrown if there is an exception parsing the
+     * timestamps
+     * @throws InvalidSettingException thrown if the settings are invalid
+     */
+    private Map<String, NvdCveInfo> retrieveCurrentTimestampsFromWeb()
+            throws MalformedURLException, DownloadFailedException, InvalidDataException, InvalidSettingException {
+
+        final Map<String, NvdCveInfo> map = new TreeMap<String, NvdCveInfo>();
+        String retrieveUrl = Settings.getString(Settings.KEYS.CVE_MODIFIED_20_URL);
+        if (retrieveUrl == null && properties.isBatchUpdateMode()) {
+            final NvdCveInfo item = new NvdCveInfo();
+            retrieveUrl = Settings.getString(Settings.KEYS.BATCH_UPDATE_URL);
+            if (retrieveUrl == null) {
+                final String msg = "Invalid configuration - neither the modified or batch update URLs are specified in the configuration.";
+                Logger.getLogger(DataStoreMetaInfo.class.getName()).log(Level.SEVERE, msg);
+                throw new InvalidSettingException(msg);
+            }
+            item.setTimestamp(Downloader.getLastModified(new URL(retrieveUrl)));
+            item.setId(BATCH);
+            item.setNeedsUpdate(false);
+            map.put(BATCH, item);
+        } else {
+            NvdCveInfo item = new NvdCveInfo();
+            item.setNeedsUpdate(false); //the others default to true, to make life easier later this should default to false.
+            item.setId(MODIFIED);
+            item.setUrl(retrieveUrl);
+            item.setOldSchemaVersionUrl(Settings.getString(Settings.KEYS.CVE_MODIFIED_12_URL));
+
+            item.setTimestamp(Downloader.getLastModified(new URL(retrieveUrl)));
+            map.put(MODIFIED, item);
+
+            //only add these urls if we are not in batch mode
+            if (!properties.isBatchUpdateMode()) {
+                final int start = Settings.getInt(Settings.KEYS.CVE_START_YEAR);
+                final int end = Calendar.getInstance().get(Calendar.YEAR);
+                final String baseUrl20 = Settings.getString(Settings.KEYS.CVE_SCHEMA_2_0);
+                final String baseUrl12 = Settings.getString(Settings.KEYS.CVE_SCHEMA_1_2);
+                for (int i = start; i <= end; i++) {
+                    retrieveUrl = String.format(baseUrl20, i);
+                    item = new NvdCveInfo();
+                    item.setId(Integer.toString(i));
+                    item.setUrl(retrieveUrl);
+                    item.setOldSchemaVersionUrl(String.format(baseUrl12, i));
+                    item.setTimestamp(Downloader.getLastModified(new URL(retrieveUrl)));
+                    map.put(item.getId(), item);
+                }
+            }
+        }
+        return map;
+    }
+}

dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/NvdCveInfo.java

@@ -0,0 +1,138 @@
+/*
+ * This file is part of dependency-check-core.
+ *
+ * Dependency-check-core is free software: you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation, either version 3 of the License, or (at your option) any
+ * later version.
+ *
+ * Dependency-check-core is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+ * details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * dependency-check-core. If not, see http://www.gnu.org/licenses/.
+ *
+ * Copyright (c) 2013 Jeremy Long. All Rights Reserved.
+ */
+package org.owasp.dependencycheck.data.update;
+
+/**
+ * A pojo that contains the Url and timestamp of the current NvdCve XML files.
+ *
+ * @author Jeremy Long (jeremy.long@owasp.org)
+ */
+public class NvdCveInfo {
+
+    /**
+     * an id.
+     */
+    private String id;
+
+    /**
+     * Get the value of id.
+     *
+     * @return the value of id
+     */
+    public String getId() {
+        return id;
+    }
+
+    /**
+     * Set the value of id.
+     *
+     * @param id new value of id
+     */
+    public void setId(String id) {
+        this.id = id;
+    }
+    /**
+     * a url.
+     */
+    private String url;
+
+    /**
+     * Get the value of url.
+     *
+     * @return the value of url
+     */
+    public String getUrl() {
+        return url;
+    }
+
+    /**
+     * Set the value of url.
+     *
+     * @param url new value of url
+     */
+    public void setUrl(String url) {
+        this.url = url;
+    }
+    /**
+     * The 1.2 schema URL.
+     */
+    private String oldSchemaVersionUrl;
+
+    /**
+     * Get the value of oldSchemaVersionUrl.
+     *
+     * @return the value of oldSchemaVersionUrl
+     */
+    public String getOldSchemaVersionUrl() {
+        return oldSchemaVersionUrl;
+    }
+
+    /**
+     * Set the value of oldSchemaVersionUrl.
+     *
+     * @param oldSchemaVersionUrl new value of oldSchemaVersionUrl
+     */
+    public void setOldSchemaVersionUrl(String oldSchemaVersionUrl) {
+        this.oldSchemaVersionUrl = oldSchemaVersionUrl;
+    }
+    /**
+     * a timestamp - epoch time.
+     */
+    private long timestamp;
+
+    /**
+     * Get the value of timestamp - epoch time.
+     *
+     * @return the value of timestamp - epoch time
+     */
+    public long getTimestamp() {
+        return timestamp;
+    }
+
+    /**
+     * Set the value of timestamp - epoch time.
+     *
+     * @param timestamp new value of timestamp - epoch time
+     */
+    public void setTimestamp(long timestamp) {
+        this.timestamp = timestamp;
+    }
+    /**
+     * indicates whether or not this item should be updated.
+     */
+    private boolean needsUpdate = true;
+
+    /**
+     * Get the value of needsUpdate.
+     *
+     * @return the value of needsUpdate
+     */
+    public boolean getNeedsUpdate() {
+        return needsUpdate;
+    }
+
+    /**
+     * Set the value of needsUpdate.
+     *
+     * @param needsUpdate new value of needsUpdate
+     */
+    public void setNeedsUpdate(boolean needsUpdate) {
+        this.needsUpdate = needsUpdate;
+    }
+}

dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/package-info.java

@@ -15,4 +15,4 @@
  * </html>
 */
 
-package org.owasp.dependencycheck.data.nvdcve.xml;
+package org.owasp.dependencycheck.data.update;

dependency-check-core/src/main/java/org/owasp/dependencycheck/utils/Downloader.java

@@ -27,6 +27,7 @@ import java.net.HttpURLConnection;
 import java.net.InetSocketAddress;
 import java.net.Proxy;
 import java.net.SocketAddress;
+import java.net.URISyntaxException;
 import java.net.URL;
 import java.util.logging.Level;
 import java.util.logging.Logger;
@@ -46,33 +47,6 @@ public final class Downloader {
     private Downloader() {
     }
 
-    /**
-     * Retrieves a file from a given URL and saves it to the outputPath.
-     *
-     * @param url the URL of the file to download.
-     * @param outputPath the path to the save the file to.
-     * @throws DownloadFailedException is thrown if there is an error
-     * downloading the file.
-     */
-    public static void fetchFile(URL url, String outputPath) throws DownloadFailedException {
-        fetchFile(url, outputPath, false);
-    }
-
-    /**
-     * Retrieves a file from a given URL and saves it to the outputPath.
-     *
-     * @param url the URL of the file to download.
-     * @param outputPath the path to the save the file to.
-     * @param unzip true/false indicating that the file being retrieved is
-     * gzipped and if true, should be uncompressed before writing to the file.
-     * @throws DownloadFailedException is thrown if there is an error
-     * downloading the file.
-     */
-    public static void fetchFile(URL url, String outputPath, boolean unzip) throws DownloadFailedException {
-        final File f = new File(outputPath);
-        fetchFile(url, f, unzip);
-    }
-
     /**
      * Retrieves a file from a given URL and saves it to the outputPath.
      *
@@ -82,20 +56,6 @@ public final class Downloader {
      * downloading the file.
      */
     public static void fetchFile(URL url, File outputPath) throws DownloadFailedException {
-        fetchFile(url, outputPath, false);
-    }
-
-    /**
-     * Retrieves a file from a given URL and saves it to the outputPath.
-     *
-     * @param url the URL of the file to download.
-     * @param outputPath the path to the save the file to.
-     * @param unzip true/false indicating that the file being retrieved is
-     * gzipped and if true, should be uncompressed before writing to the file.
-     * @throws DownloadFailedException is thrown if there is an error
-     * downloading the file.
-     */
-    public static void fetchFile(URL url, File outputPath, boolean unzip) throws DownloadFailedException {
         HttpURLConnection conn = null;
         try {
             conn = Downloader.getConnection(url);
@@ -116,7 +76,7 @@ public final class Downloader {
         BufferedOutputStream writer = null;
         InputStream reader = null;
         try {
-            if (unzip || (encoding != null && "gzip".equalsIgnoreCase(encoding))) {
+            if (encoding != null && "gzip".equalsIgnoreCase(encoding)) {
                 reader = new GZIPInputStream(conn.getInputStream());
             } else if (encoding != null && "deflate".equalsIgnoreCase(encoding)) {
                 reader = new InflaterInputStream(conn.getInputStream());
@@ -136,7 +96,6 @@ public final class Downloader {
             if (writer != null) {
                 try {
                     writer.close();
-                    writer = null;
                 } catch (Exception ex) {
                     Logger.getLogger(Downloader.class.getName()).log(Level.FINEST,
                             "Error closing the writer in Downloader.", ex);
@@ -145,9 +104,7 @@ public final class Downloader {
             if (reader != null) {
                 try {
                     reader.close();
-                    reader = null;
                 } catch (Exception ex) {
-
                     Logger.getLogger(Downloader.class.getName()).log(Level.FINEST,
                             "Error closing the reader in Downloader.", ex);
                 }
@@ -162,7 +119,8 @@ public final class Downloader {
 
     /**
      * Makes an HTTP Head request to retrieve the last modified date of the
-     * given URL.
+     * given URL. If the file:// protocol is specified, then the lastTimestamp
+     * of the file is returned.
      *
      * @param url the URL to retrieve the timestamp from
      * @return an epoch timestamp
@@ -170,21 +128,42 @@ public final class Downloader {
      * the HTTP request
      */
     public static long getLastModified(URL url) throws DownloadFailedException {
-        HttpURLConnection conn = null;
         long timestamp = 0;
-        try {
-            conn = Downloader.getConnection(url);
-            conn.setRequestMethod("HEAD");
-            conn.connect();
-            timestamp = conn.getLastModified();
-        } catch (Exception ex) {
-            throw new DownloadFailedException("Error making HTTP HEAD request.", ex);
-        } finally {
-            if (conn != null) {
-                try {
-                    conn.disconnect();
-                } finally {
-                    conn = null;
+        //TODO add the FPR protocol?
+        if ("file".equalsIgnoreCase(url.getProtocol())) {
+            File lastModifiedFile;
+            try {
+//                if (System.getProperty("os.name").toLowerCase().startsWith("windows")) {
+//                    String filePath = url.toString();
+//                    if (filePath.matches("file://[a-zA-Z]:.*")) {
+//                        f = new File(filePath.substring(7));
+//                    } else {
+//                        f = new File(url.toURI());
+//                    }
+//                } else {
+                lastModifiedFile = new File(url.toURI());
+//                }
+            } catch (URISyntaxException ex) {
+                final String msg = String.format("Unable to locate '%s'; is the cve.url-2.0.modified property set correctly?", url.toString());
+                throw new DownloadFailedException(msg);
+            }
+            timestamp = lastModifiedFile.lastModified();
+        } else {
+            HttpURLConnection conn = null;
+            try {
+                conn = Downloader.getConnection(url);
+                conn.setRequestMethod("HEAD");
+                conn.connect();
+                timestamp = conn.getLastModified();
+            } catch (Exception ex) {
+                throw new DownloadFailedException("Error making HTTP HEAD request.", ex);
+            } finally {
+                if (conn != null) {
+                    try {
+                        conn.disconnect();
+                    } finally {
+                        conn = null;
+                    }
                 }
             }
         }
@@ -213,11 +192,8 @@ public final class Downloader {
             } else {
                 conn = (HttpURLConnection) url.openConnection();
             }
-            //added a default timeout of 20000
-            //if (Settings.getString(Settings.KEYS.CONNECTION_TIMEOUT) != null) {
             final int timeout = Settings.getInt(Settings.KEYS.CONNECTION_TIMEOUT, 60000);
             conn.setConnectTimeout(timeout);
-            //}
         } catch (IOException ex) {
             if (conn != null) {
                 try {

dependency-check-core/src/main/java/org/owasp/dependencycheck/utils/ExtractionException.java

@@ -0,0 +1,69 @@
+/*
+ * This file is part of dependency-check-core.
+ *
+ * Dependency-check-core is free software: you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation, either version 3 of the License, or (at your option) any
+ * later version.
+ *
+ * Dependency-check-core is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+ * details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * dependency-check-core. If not, see http://www.gnu.org/licenses/.
+ *
+ * Copyright (c) 2013 Jeremy Long. All Rights Reserved.
+ */
+package org.owasp.dependencycheck.utils;
+
+import java.io.IOException;
+
+/**
+ * An exception used when a file is unable to be un-zipped.
+ *
+ * @author Jeremy Long (jeremy.long@owasp.org)
+ */
+public class ExtractionException extends IOException {
+
+    /**
+     * The serial version UID.
+     */
+    private static final long serialVersionUID = 1L;
+
+    /**
+     * Creates a new ExtractionException.
+     */
+    public ExtractionException() {
+        super();
+    }
+
+    /**
+     * Creates a new ExtractionException.
+     *
+     * @param msg a message for the exception.
+     */
+    public ExtractionException(String msg) {
+        super(msg);
+    }
+
+    /**
+     * Creates a new ExtractionException.
+     *
+     * @param ex the cause of the download failure.
+     */
+    public ExtractionException(Throwable ex) {
+        super(ex);
+    }
+
+    /**
+     * Creates a new ExtractionException.
+     *
+     * @param msg a message for the exception.
+     * @param ex the cause of the download failure.
+     */
+    public ExtractionException(String msg, Throwable ex) {
+        super(msg, ex);
+    }
+}

dependency-check-core/src/main/java/org/owasp/dependencycheck/utils/FileUtils.java

@@ -18,11 +18,20 @@
  */
 package org.owasp.dependencycheck.utils;
 
+import java.io.BufferedInputStream;
+import java.io.BufferedOutputStream;
 import java.io.File;
+import java.io.FileInputStream;
 import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
 import java.io.IOException;
 import java.io.UnsupportedEncodingException;
 import java.net.URLDecoder;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipInputStream;
+import org.owasp.dependencycheck.Engine;
 
 /**
  * A collection of utilities for processing information about files.
@@ -31,6 +40,11 @@ import java.net.URLDecoder;
  */
 public final class FileUtils {
 
+    /**
+     * The buffer size to use when extracting files from the archive.
+     */
+    private static final int BUFFER_SIZE = 4096;
+
     /**
      * Private constructor for a utility class.
      */
@@ -65,8 +79,37 @@ public final class FileUtils {
                 delete(c);
             }
         }
-        if (!file.delete()) {
+        if (!org.apache.commons.io.FileUtils.deleteQuietly(file)) {
+            //if (!file.delete()) {
             throw new FileNotFoundException("Failed to delete file: " + file);
+        } else {
+            file.deleteOnExit();
+        }
+    }
+
+    /**
+     * Deletes a file. If the File is a directory it will recursively delete the
+     * contents.
+     *
+     * @param file the File to delete
+     * @param deleteOnExit setting this to true will cause errors to be ignored
+     * and if there is an error deleting the file it will be setup to be deleted
+     * when the JVM exits.
+     * @throws IOException is thrown if the file could not be deleted
+     */
+    public static void delete(File file, boolean deleteOnExit) throws IOException {
+        if (file.isDirectory()) {
+            for (File c : file.listFiles()) {
+                delete(c);
+            }
+        }
+        if (!org.apache.commons.io.FileUtils.deleteQuietly(file)) {
+            //if (!file.delete()) {
+            if (deleteOnExit) {
+                file.deleteOnExit();
+            } else {
+                throw new FileNotFoundException("Failed to delete file: " + file);
+            }
         }
     }
 
@@ -114,4 +157,100 @@ public final class FileUtils {
         final File jarPath = new File(decodedPath);
         return jarPath.getParentFile();
     }
+
+    /**
+     * Extracts the contents of an archive into the specified directory.
+     *
+     * @param archive an archive file such as a WAR or EAR
+     * @param extractTo a directory to extract the contents to
+     * @throws ExtractionException thrown if an exception occurs while
+     * extracting the files
+     */
+    public static void extractFiles(File archive, File extractTo) throws ExtractionException {
+        extractFiles(archive, extractTo, null);
+    }
+
+    /**
+     * Extracts the contents of an archive into the specified directory. The
+     * files are only extracted if they are supported by the analyzers loaded
+     * into the specified engine. If the engine is specified as null then all
+     * files are extracted.
+     *
+     * @param archive an archive file such as a WAR or EAR
+     * @param extractTo a directory to extract the contents to
+     * @param engine the scanning engine
+     * @throws ExtractionException thrown if there is an error extracting the
+     * files
+     */
+    public static void extractFiles(File archive, File extractTo, Engine engine) throws ExtractionException {
+        if (archive == null || extractTo == null) {
+            return;
+        }
+
+        FileInputStream fis = null;
+        ZipInputStream zis = null;
+
+        try {
+            fis = new FileInputStream(archive);
+        } catch (FileNotFoundException ex) {
+            Logger.getLogger(FileUtils.class.getName()).log(Level.INFO, null, ex);
+            throw new ExtractionException("Archive file was not found.", ex);
+        }
+        zis = new ZipInputStream(new BufferedInputStream(fis));
+        ZipEntry entry;
+        try {
+            while ((entry = zis.getNextEntry()) != null) {
+                if (entry.isDirectory()) {
+                    final File d = new File(extractTo, entry.getName());
+                    if (!d.exists() && !d.mkdirs()) {
+                        final String msg = String.format("Unable to create '%s'.", d.getAbsolutePath());
+                        throw new ExtractionException(msg);
+                    }
+                } else {
+                    final File file = new File(extractTo, entry.getName());
+                    final String ext = getFileExtension(file.getName());
+                    if (engine == null || engine.supportsExtension(ext)) {
+                        BufferedOutputStream bos = null;
+                        FileOutputStream fos;
+                        try {
+                            fos = new FileOutputStream(file);
+                            bos = new BufferedOutputStream(fos, BUFFER_SIZE);
+                            int count;
+                            final byte data[] = new byte[BUFFER_SIZE];
+                            while ((count = zis.read(data, 0, BUFFER_SIZE)) != -1) {
+                                bos.write(data, 0, count);
+                            }
+                            bos.flush();
+                        } catch (FileNotFoundException ex) {
+                            Logger.getLogger(FileUtils.class.getName()).log(Level.FINE, null, ex);
+                            final String msg = String.format("Unable to find file '%s'.", file.getName());
+                            throw new ExtractionException(msg, ex);
+                        } catch (IOException ex) {
+                            Logger.getLogger(FileUtils.class.getName()).log(Level.FINE, null, ex);
+                            final String msg = String.format("IO Exception while parsing file '%s'.", file.getName());
+                            throw new ExtractionException(msg, ex);
+                        } finally {
+                            if (bos != null) {
+                                try {
+                                    bos.close();
+                                } catch (IOException ex) {
+                                    Logger.getLogger(FileUtils.class.getName()).log(Level.FINEST, null, ex);
+                                }
+                            }
+                        }
+                    }
+                }
+            }
+        } catch (IOException ex) {
+            final String msg = String.format("Exception reading archive '%s'.", archive.getName());
+            Logger.getLogger(FileUtils.class.getName()).log(Level.FINE, msg, ex);
+            throw new ExtractionException(msg, ex);
+        } finally {
+            try {
+                zis.close();
+            } catch (IOException ex) {
+                Logger.getLogger(FileUtils.class.getName()).log(Level.FINEST, null, ex);
+            }
+        }
+    }
 }

dependency-check-core/src/main/java/org/owasp/dependencycheck/utils/Settings.java

@@ -67,6 +67,11 @@ public final class Settings {
          * The base path to use for the data directory.
          */
         public static final String DATA_DIRECTORY = "data.directory";
+        /**
+         * The location of the batch update URL. This is a zip file that
+         * contains the contents of the data directory.
+         */
+        public static final String BATCH_UPDATE_URL = "batch.update.url";
         /**
          * The properties key for the path where the CPE Lucene Index will be
          * stored.
@@ -77,14 +82,6 @@ public final class Settings {
          * stored.
          */
         public static final String CVE_DATA_DIRECTORY = "data.cve";
-        /**
-         * The properties key for the URL to the CPE.
-         */
-        public static final String CPE_URL = "cpe.url";
-        /**
-         * The properties key for the URL to the CPE.
-         */
-        public static final String CPE_META_URL = "cpe.meta.url";
         /**
          * The properties key for the URL to retrieve the "meta" data from about
          * the CVE entries.
@@ -229,25 +226,6 @@ public final class Settings {
         INSTANCE.props.load(stream);
     }
 
-    /**
-     * Returns a value from the properties file as a File object. If the value
-     * was specified as a system property or passed in via the -Dprop=value
-     * argument - this method will return the value from the system properties
-     * before the values in the contained configuration file.
-     *
-     * @param key the key to lookup within the properties file
-     * @param defaultValue the default value for the requested property
-     * @return the property from the properties file as a File object
-     */
-    public static File getFile(String key, String defaultValue) {
-        final String baseDir = getString(Settings.KEYS.DATA_DIRECTORY);
-        final String str = getString(key, defaultValue);
-        if (baseDir != null) {
-            return new File(baseDir, str);
-        }
-        return new File(str);
-    }
-
     /**
      * Returns a value from the properties file as a File object. If the value
      * was specified as a system property or passed in via the -Dprop=value
@@ -259,17 +237,22 @@ public final class Settings {
      *
      * @param key the key to lookup within the properties file
      * @return the property from the properties file converted to a File object
-     * @throws IOException thrown if the file path to the JAR cannot be found
      */
-    public static File getFile(String key) throws IOException {
+    public static File getFile(String key) {
         final String file = getString(key);
         final String baseDir = getString(Settings.KEYS.DATA_DIRECTORY);
         if (baseDir != null) {
             if (baseDir.startsWith("[JAR]/")) {
                 final File jarPath = getJarPath();
-                final File newBase = new File(jarPath.getCanonicalPath(), baseDir.substring(6));
+                final File newBase = new File(jarPath, baseDir.substring(6));
+                if (Settings.KEYS.DATA_DIRECTORY.equals(key)) {
+                    return newBase;
+                }
                 return new File(newBase, file);
             }
+            if (Settings.KEYS.DATA_DIRECTORY.equals(key)) {
+                return new File(baseDir);
+            }
             return new File(baseDir, file);
         }
         return new File(file);
@@ -313,6 +296,15 @@ public final class Settings {
         return str;
     }
 
+    /**
+     * Returns the temporary directory.
+     *
+     * @return the temporary directory
+     */
+    public static File getTempDirectory() {
+        return new File(Settings.getString(Settings.KEYS.TEMP_DIRECTORY, System.getProperty("java.io.tmpdir")));
+    }
+
     /**
      * Returns a value from the properties file. If the value was specified as a
      * system property or passed in via the -Dprop=value argument - this method
@@ -326,6 +318,16 @@ public final class Settings {
         return System.getProperty(key, INSTANCE.props.getProperty(key));
     }
 
+    /**
+     * Removes a property from the local properties collection. This is mainly
+     * used in test cases.
+     *
+     * @param key the property key to remove
+     */
+    public static void removeProperty(String key) {
+        INSTANCE.props.remove(key);
+    }
+
     /**
      * Returns an int value from the properties file. If the value was specified
      * as a system property or passed in via the -Dprop=value argument - this

dependency-check-core/src/main/java/org/owasp/dependencycheck/utils/UrlStringUtils.java

@@ -21,6 +21,8 @@ package org.owasp.dependencycheck.utils;
 import java.net.MalformedURLException;
 import java.net.URL;
 import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashSet;
 import java.util.List;
 import java.util.regex.Pattern;
 
@@ -64,6 +66,11 @@ public final class UrlStringUtils {
     public static boolean isUrl(String text) {
         return IS_URL_TEST.matcher(text).matches();
     }
+    /**
+     * A listing of domain parts that shold not be used as evidence. Yes, this
+     * is an incomplete list.
+     */
+    private static final HashSet<String> IGNORE_LIST = new HashSet<String>(Arrays.asList("www", "com", "org", "gov", "info", "name", "net", "pro", "tel", "mobi", "xxx"));
 
     /**
      * <p>Takes a URL, in String format, and adds the important parts of the URL
@@ -84,7 +91,7 @@ public final class UrlStringUtils {
         //add the domain except www and the tld.
         for (int i = 0; i < domain.length - 1; i++) {
             final String sub = domain[i];
-            if (!"www".equalsIgnoreCase(sub)) {
+            if (!IGNORE_LIST.contains(sub.toLowerCase())) {
                 importantParts.add(sub);
             }
         }

dependency-check-core/src/main/resources/META-INF/services/org.owasp.dependencycheck.analyzer.Analyzer

@@ -4,5 +4,5 @@ org.owasp.dependencycheck.analyzer.FileNameAnalyzer
 org.owasp.dependencycheck.analyzer.HintAnalyzer
 org.owasp.dependencycheck.analyzer.DependencyBundlingAnalyzer
 org.owasp.dependencycheck.analyzer.FalsePositiveAnalyzer
-org.owasp.dependencycheck.data.cpe.CPEAnalyzer
-org.owasp.dependencycheck.data.nvdcve.NvdCveAnalyzer
+org.owasp.dependencycheck.analyzer.CPEAnalyzer
+org.owasp.dependencycheck.analyzer.NvdCveAnalyzer

dependency-check-core/src/main/resources/META-INF/services/org.owasp.dependencycheck.data.CachedWebDataSource

@@ -1 +1 @@
-org.owasp.dependencycheck.data.nvdcve.xml.DatabaseUpdater
+org.owasp.dependencycheck.data.update.DatabaseUpdater

dependency-check-core/src/main/resources/dependencycheck.properties

@@ -23,6 +23,15 @@ cpe.meta.url=http://static.nvd.nist.gov/feeds/xml/cpe/dictionary/official-cpe-di
 # holds 8 days of updates, we are using 7 just to be safe.
 cve.url.modified.validfordays=7
 
+# The location of the zipped CVE H2 database and CPE Lucene index. If specified and
+# a full download of data is required this URL will be used and the data extracted
+# into the specified "data" directory. Additionally, after pulling the data the
+# system will attempt to update the modified. Thus, if one were maintaining an
+# internal copy of the data one would not need to update it nightly.
+# If the 'modified' URLs below for the CVE are removed and a batch url is provided
+# then if an update is required, the entre zip file will be downloaded.
+#batch.update.url=file:///C:/path/to/data.zip
+
 # the path to the modified nvd cve xml file.
 cve.url-1.2.modified=http://nvd.nist.gov/download/nvdcve-modified.xml
 cve.url-2.0.modified=http://static.nvd.nist.gov/feeds/xml/cve/nvdcve-2.0-modified.xml

dependency-check-core/src/main/resources/templates/HtmlReport.vsl

@@ -52,6 +52,11 @@ Copyright (c) 2012 Jeremy Long. All Rights Reserved.
                 });
             });
         </script>
+        <script type="text/javascript">
+            function toggleVuln() {
+                $(".notvulnerable").toggle();
+            }
+        </script>
         <style type="text/css">
 
             .rounded-corners {
@@ -275,6 +280,15 @@ Copyright (c) 2012 Jeremy Long. All Rights Reserved.
                 margin-top:3px;
                 margin-bottom:3px;
             }
+            .vulnerable {
+            	color: #f00;
+            }
+            .vulnerable li {
+            	color: #000;
+            }
+            .notvulnerable {
+                display:none;
+            }
         </style>
     </head>
     <body>
@@ -292,13 +306,14 @@ Copyright (c) 2012 Jeremy Long. All Rights Reserved.
                         #set($vulnCount=$vulnCount+1)
                     #end
                 #end
-                Dependencies Scanned:&nbsp;$depCount<br/>
+                Dependencies Scanned:&nbsp;$depCount&nbsp;(<a href="#" onclick="toggleVuln()">show all</a>)<br/>
                 Vulnerable Dependencies:&nbsp;$vulnCount<br/><br/>
-                <div class="indent">
+                <ul class="indent">
                 #set($lnkcnt=0)
                 #foreach($dependency in $dependencies)
                     #set($lnkcnt=$lnkcnt+1)
-                    <a href="#l${lnkcnt}_$esc.html($esc.url($dependency.Sha1sum))">$esc.html($dependency.FileName)</a>#if($dependency.getVulnerabilities().size()>0)&nbsp;<b style="color:#ff0000;">&#8226;</b>#end<br/>
+                    <li class="#if($dependency.getVulnerabilities().size()==0)notvulnerable#else vulnerable#end">
+                    <a href="#l${lnkcnt}_$esc.html($esc.url($dependency.Sha1sum))">$esc.html($dependency.FileName)</a>
                     #if($dependency.getRelatedDependencies().size()>0)
                         <ul>
                         #foreach($related in $dependency.getRelatedDependencies())
@@ -306,15 +321,16 @@ Copyright (c) 2012 Jeremy Long. All Rights Reserved.
                         #end
                         </ul>
                     #end
+                    </li>
                 #end
-                </div>
+                </ul>
                 <h2>Dependencies</h2>
                 #set($lnkcnt=0)
                 #set($cnt=0)
                 #foreach($dependency in $dependencies)
                     #set($lnkcnt=$lnkcnt+1)
-                    <h3 class="subsectionheader standardsubsection"><a name="l${lnkcnt}_$esc.html($dependency.Sha1sum)"></a>$esc.html($dependency.FileName)</h3>
-                    <div class="subsectioncontent">
+                    <h3 class="subsectionheader standardsubsection#if($dependency.getVulnerabilities().size()==0) notvulnerable#end"><a name="l${lnkcnt}_$esc.html($dependency.Sha1sum)"></a>$esc.html($dependency.FileName)</h3>
+                    <div class="subsectioncontent#if($dependency.getVulnerabilities().size()==0) notvulnerable#end">
                         #if ($dependency.description)
                             <p><b>Description:</b>&nbsp;$esc.html($dependency.description)<br/></p>
                         #end

dependency-check-core/src/site/markdown/index.md

@@ -13,4 +13,4 @@ The engine is currently exposed via:
 - [Command Line Tool](../dependency-check-cli/installation.html)
 - [Maven Plugin](../dependency-check-maven/usage.html)
 - [Ant Task](../dependency-check-ant/installation.html)
-- Jenkins Plugin
+- [Jenkins Plugin](../dependency-check-jenkins/index.html)

dependency-check-core/src/test/java/org/owasp/dependencycheck/EngineIntegrationTest.java

@@ -44,7 +44,6 @@ public class EngineIntegrationTest {
     @Before
     public void setUp() throws Exception {
         org.owasp.dependencycheck.data.nvdcve.BaseDBTestCase.ensureDBExists();
-        org.owasp.dependencycheck.data.cpe.BaseIndexTestCase.ensureIndexExists();
     }
 
     @After
@@ -58,9 +57,9 @@ public class EngineIntegrationTest {
      */
     @Test
     public void testScan() throws Exception {
-        String path = "target/test-classes";
+        String testClasses = "target/test-classes";
         Engine instance = new Engine();
-        instance.scan(path);
+        instance.scan(testClasses);
         assertTrue(instance.getDependencies().size() > 0);
         instance.analyzeDependencies();
         ReportGenerator rg = new ReportGenerator("DependencyCheck",

dependency-check-core/src/test/java/org/owasp/dependencycheck/analyzer/ArchiveAnalyzerTest.java

@@ -139,7 +139,7 @@ public class ArchiveAnalyzerTest {
         try {
             instance.initialize();
 
-            File file = new File(this.getClass().getClassLoader().getResource("opensso.war").getPath());
+            File file = new File(this.getClass().getClassLoader().getResource("daytrader-ear-2.1.7.ear").getPath());
             Dependency dependency = new Dependency(file);
             Settings.setBoolean(Settings.KEYS.AUTO_UPDATE, false);
             Engine engine = new Engine();
@@ -154,4 +154,29 @@ public class ArchiveAnalyzerTest {
             instance.close();
         }
     }
+
+    /**
+     * Test of analyze method, of class ArchiveAnalyzer.
+     */
+    @Test
+    public void testAnalyze_badZip() throws Exception {
+        ArchiveAnalyzer instance = new ArchiveAnalyzer();
+        try {
+            instance.initialize();
+
+            File file = new File(this.getClass().getClassLoader().getResource("test.zip").getPath());
+            Dependency dependency = new Dependency(file);
+            Settings.setBoolean(Settings.KEYS.AUTO_UPDATE, false);
+            Engine engine = new Engine();
+
+            int initial_size = engine.getDependencies().size();
+            instance.analyze(dependency, engine);
+            int ending_size = engine.getDependencies().size();
+
+            assertTrue(initial_size == ending_size);
+
+        } finally {
+            instance.close();
+        }
+    }
 }

dependency-check-core/src/test/java/org/owasp/dependencycheck/analyzer/CPEAnalyzerTest.java

@@ -16,10 +16,10 @@
  *
  * Copyright (c) 2012 Jeremy Long. All Rights Reserved.
  */
-package org.owasp.dependencycheck.data.cpe;
+package org.owasp.dependencycheck.analyzer;
 
 import org.owasp.dependencycheck.data.cpe.IndexEntry;
-import org.owasp.dependencycheck.data.cpe.CPEAnalyzer;
+import org.owasp.dependencycheck.analyzer.CPEAnalyzer;
 import java.io.File;
 import java.io.IOException;
 import java.util.HashSet;
@@ -30,15 +30,12 @@ import org.apache.lucene.queryparser.classic.ParseException;
 import org.junit.After;
 import org.junit.AfterClass;
 import org.owasp.dependencycheck.dependency.Dependency;
-import org.owasp.dependencycheck.analyzer.JarAnalyzer;
 import org.junit.Assert;
 import org.junit.Before;
 import org.junit.BeforeClass;
 import org.junit.Test;
-import org.owasp.dependencycheck.analyzer.FalsePositiveAnalyzer;
-import org.owasp.dependencycheck.analyzer.FileNameAnalyzer;
-import org.owasp.dependencycheck.analyzer.HintAnalyzer;
-import static org.owasp.dependencycheck.data.cpe.BaseIndexTestCase.ensureIndexExists;
+import org.owasp.dependencycheck.data.cpe.BaseIndexTestCase;
+import org.owasp.dependencycheck.data.cpe.IndexEntry;
 import org.owasp.dependencycheck.dependency.Identifier;
 
 /**
@@ -56,11 +53,13 @@ public class CPEAnalyzerTest extends BaseIndexTestCase {
     }
 
     @Before
+    @Override
     public void setUp() throws Exception {
         super.setUp();
     }
 
     @After
+    @Override
     public void tearDown() throws Exception {
         super.tearDown();
     }
@@ -124,6 +123,7 @@ public class CPEAnalyzerTest extends BaseIndexTestCase {
      */
     @Test
     public void testDetermineCPE_full() throws Exception {
+        callDetermineCPE_full("hazelcast-2.5.jar", null);
         callDetermineCPE_full("spring-context-support-2.5.5.jar", "cpe:/a:vmware:springsource_spring_framework:2.5.5");
         callDetermineCPE_full("spring-core-3.0.0.RELEASE.jar", "cpe:/a:vmware:springsource_spring_framework:3.0.0");
         callDetermineCPE_full("org.mortbay.jetty.jar", "cpe:/a:mortbay_jetty:jetty:4.2");
@@ -139,6 +139,7 @@ public class CPEAnalyzerTest extends BaseIndexTestCase {
     public void callDetermineCPE_full(String depName, String expResult) throws Exception {
 
         File file = new File(this.getClass().getClassLoader().getResource(depName).getPath());
+
         Dependency dep = new Dependency(file);
 
         FileNameAnalyzer fnAnalyzer = new FileNameAnalyzer();
@@ -163,8 +164,10 @@ public class CPEAnalyzerTest extends BaseIndexTestCase {
         if (expResult != null) {
             Identifier expIdentifier = new Identifier("cpe", expResult, expResult);
             Assert.assertTrue("Incorrect match: { dep:'" + dep.getFileName() + "' }", dep.getIdentifiers().contains(expIdentifier));
-        } else {
+        } else if (dep.getIdentifiers().isEmpty()) {
             Assert.assertTrue("Match found when an Identifier should not have been found: { dep:'" + dep.getFileName() + "' }", dep.getIdentifiers().isEmpty());
+        } else {
+            Assert.assertTrue("Match found when an Identifier should not have been found: { dep:'" + dep.getFileName() + "', identifier:'" + dep.getIdentifiers().iterator().next().getValue() + "' }", dep.getIdentifiers().isEmpty());
         }
     }
 

dependency-check-core/src/test/java/org/owasp/dependencycheck/analyzer/JarAnalyzerTest.java

@@ -118,6 +118,7 @@ public class JarAnalyzerTest {
         JarAnalyzer instance = new JarAnalyzer();
         Set expResult = new HashSet();
         expResult.add("jar");
+        expResult.add("war");
         Set result = instance.getSupportedExtensions();
         assertEquals(expResult, result);
     }

dependency-check-core/src/test/java/org/owasp/dependencycheck/concurrency/DirectorySpinLockTest.java

@@ -0,0 +1,116 @@
+/*
+ * This file is part of dependency-check-core.
+ *
+ * Dependency-check-core is free software: you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation, either version 3 of the License, or (at your option) any
+ * later version.
+ *
+ * Dependency-check-core is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+ * details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * dependency-check-core. If not, see http://www.gnu.org/licenses/.
+ *
+ * Copyright (c) 2013 Jeremy Long. All Rights Reserved.
+ */
+package org.owasp.dependencycheck.concurrency;
+
+import java.io.File;
+import java.net.URL;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import static org.junit.Assert.*;
+
+/**
+ *
+ * @author Jeremy Long (jeremy.long@owasp.org)
+ */
+public class DirectorySpinLockTest {
+
+    public DirectorySpinLockTest() {
+    }
+
+    @BeforeClass
+    public static void setUpClass() {
+    }
+
+    @AfterClass
+    public static void tearDownClass() {
+    }
+
+    @Before
+    public void setUp() {
+    }
+
+    @After
+    public void tearDown() {
+    }
+
+    /**
+     * Test of obtainSharedLock method, of class DirectorySpinLock.
+     * Specifically, this test uses the SpinLockTask to obtain an exclusive lock
+     * that is held for 5 seconds. We then try to obtain a shared lock while
+     * that task is running. It should take longer then 5 seconds to obtain the
+     * shared lock.
+     */
+    @Test
+    public void testObtainSharedLock_withContention() throws Exception {
+        URL location = this.getClass().getProtectionDomain().getCodeSource().getLocation();
+        File directory = new File(location.getFile());
+        DirectorySpinLock instance = new DirectorySpinLock(directory);
+        SpinLockTask task = new SpinLockTask(directory, 5000, false, 2);
+        long start = System.currentTimeMillis();
+        task.run();
+        instance.obtainSharedLock();
+        long end = System.currentTimeMillis();
+        instance.close();
+        if (task.getException() != null) {
+            throw task.getException();
+        }
+        long timeElapsed = end - start;
+        assertTrue("no lock contention occured?", timeElapsed >= 5000);
+        //no exceptions means everything worked.
+    }
+
+    /**
+     * Test of obtainSharedLock method, of class DirectorySpinLock. This method
+     * obtains two shared locks by using the SpinLockTask to obtain a lock in
+     * another thread.
+     */
+    @Test
+    public void testObtainSharedLock() throws Exception {
+        URL location = this.getClass().getProtectionDomain().getCodeSource().getLocation();
+        File directory = new File(location.getFile());
+        DirectorySpinLock instance = new DirectorySpinLock(directory);
+        SpinLockTask task = new SpinLockTask(directory, 1000, true, 2);
+        task.run();
+        instance.obtainSharedLock();
+        instance.close();
+        if (task.getException() != null) {
+            throw task.getException();
+        }
+        //no exceptions means everything worked.
+    }
+
+    /**
+     * Test of obtainExclusiveLock method, of class DirectorySpinLock.
+     */
+    @Test
+    public void testObtainExclusiveLock() throws Exception {
+        URL location = this.getClass().getProtectionDomain().getCodeSource().getLocation();
+        File directory = new File(location.getFile());
+        DirectorySpinLock instance = new DirectorySpinLock(directory);
+        SpinLockTask task = new SpinLockTask(directory, 1000, true, 1);
+        instance.obtainExclusiveLock();
+        task.run();
+        instance.close();
+        assertNotNull("No exception thrown due to exclusive lock failure?", task.getException());
+        assertEquals("Incorrect exception when obtaining exclusive lock", "Unable to obtain lock", task.getException().getMessage());
+    }
+}

dependency-check-core/src/test/java/org/owasp/dependencycheck/concurrency/SpinLockTask.java

@@ -0,0 +1,84 @@
+/*
+ * This file is part of dependency-check-core.
+ *
+ * Dependency-check-core is free software: you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation, either version 3 of the License, or (at your option) any
+ * later version.
+ *
+ * Dependency-check-core is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+ * details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * dependency-check-core. If not, see http://www.gnu.org/licenses/.
+ *
+ * Copyright (c) 2013 Jeremy Long. All Rights Reserved.
+ */
+package org.owasp.dependencycheck.concurrency;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+/**
+ * A simple task that obtains a lock on a directory. This is used in testing of
+ * the shared and exclusive locks.
+ *
+ * @author Jeremy Long (jeremy.long@owasp.org)
+ */
+public class SpinLockTask implements Runnable {
+
+    DirectorySpinLock lock = null;
+    int holdLockFor;
+    long maxWait;
+    boolean shared;
+    private Exception exception = null;
+
+    /**
+     * Get the value of exception
+     *
+     * @return the value of exception
+     */
+    public Exception getException() {
+        return exception;
+    }
+
+    /**
+     * Set the value of exception
+     *
+     * @param exception new value of exception
+     */
+    public void setException(Exception exception) {
+        this.exception = exception;
+    }
+
+    public SpinLockTask(File directory, int holdLockFor, boolean shared, long maxWait) throws InvalidDirectoryException, DirectoryLockException {
+        this.holdLockFor = holdLockFor;
+        this.shared = shared;
+        this.maxWait = maxWait;
+        lock = new DirectorySpinLock(directory);
+    }
+
+    @Override
+    public void run() {
+        try {
+            lock.obtainLock(shared, maxWait);
+            Thread.sleep(holdLockFor);
+        } catch (DirectoryLockException ex) {
+            exception = ex;
+        } catch (InterruptedException ex) {
+            exception = ex;
+        } finally {
+            if (lock != null) {
+                try {
+                    lock.close();
+                } catch (IOException ex) {
+                    exception = ex;
+                }
+            }
+        }
+    }
+}

dependency-check-core/src/test/java/org/owasp/dependencycheck/data/cpe/BaseIndexTest.java

@@ -0,0 +1,63 @@
+/*
+ * This file is part of dependency-check-core.
+ *
+ * Dependency-check-core is free software: you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation, either version 3 of the License, or (at your option) any
+ * later version.
+ *
+ * Dependency-check-core is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+ * details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * dependency-check-core. If not, see http://www.gnu.org/licenses/.
+ *
+ * Copyright (c) 2012 Jeremy Long. All Rights Reserved.
+ */
+package org.owasp.dependencycheck.data.cpe;
+
+import org.owasp.dependencycheck.data.cpe.BaseIndex;
+import java.io.File;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import static org.junit.Assert.*;
+
+/**
+ *
+ * @author Jeremy Long (jeremy.long@owasp.org)
+ */
+public class BaseIndexTest {
+
+    @BeforeClass
+    public static void setUpClass() throws Exception {
+    }
+
+    @AfterClass
+    public static void tearDownClass() throws Exception {
+    }
+
+    @Before
+    public void setUp() {
+    }
+
+    @After
+    public void tearDown() {
+    }
+
+    /**
+     * Test of getDataDirectory method, of class BaseIndex.
+     *
+     * @throws Exception
+     */
+    @Test
+    public void testGetDataDirectory() throws Exception {
+        String file = BaseIndex.getDataDirectory().getPath();
+        String exp = File.separatorChar + "target" + File.separatorChar + "data" + File.separatorChar + "cpe";
+        assertTrue(file.contains(exp));
+    }
+}

dependency-check-core/src/test/java/org/owasp/dependencycheck/data/cpe/BaseIndexTestCase.java

@@ -18,30 +18,18 @@
  */
 package org.owasp.dependencycheck.data.cpe;
 
-import java.io.BufferedInputStream;
-import java.io.BufferedOutputStream;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.util.logging.Level;
-import java.util.logging.Logger;
-import java.util.zip.ZipEntry;
-import java.util.zip.ZipInputStream;
+import junit.framework.TestCase;
 import org.junit.After;
 import org.junit.AfterClass;
 import org.junit.Before;
 import org.junit.BeforeClass;
 import org.owasp.dependencycheck.data.nvdcve.BaseDBTestCase;
-import org.owasp.dependencycheck.utils.Settings;
 
 /**
  *
  * @author Jeremy Long (jeremy.long@owasp.org)
  */
-public abstract class BaseIndexTestCase {
-
-    protected static final int BUFFER_SIZE = 2048;
+public abstract class BaseIndexTestCase extends TestCase {
 
     @BeforeClass
     public static void setUpClass() throws Exception {
@@ -52,88 +40,15 @@ public abstract class BaseIndexTestCase {
     }
 
     @Before
+    @Override
     public void setUp() throws Exception {
-        ensureIndexExists();
         BaseDBTestCase.ensureDBExists();
+        super.setUp();
     }
 
     @After
+    @Override
     public void tearDown() throws Exception {
-    }
-
-    protected static File getDataDirectory() throws IOException {
-        final String fileName = Settings.getString(Settings.KEYS.CPE_DATA_DIRECTORY);
-        final String dataDirectory = Settings.getString(Settings.KEYS.DATA_DIRECTORY);
-        return new File(dataDirectory, fileName);
-        //return FileUtils.getDataDirectory(fileName, Index.class);
-    }
-
-    public static void ensureIndexExists() throws Exception {
-        //String indexPath = Settings.getString(Settings.KEYS.CPE_DATA_DIRECTORY);
-        String indexPath = getDataDirectory().getCanonicalPath();
-        java.io.File f = new File(indexPath);
-
-        if (!f.exists() || (f.isDirectory() && f.listFiles().length == 0)) {
-            f.mkdirs();
-            FileInputStream fis = null;
-            ZipInputStream zin = null;
-            try {
-                File path = new File(BaseIndexTestCase.class.getClassLoader().getResource("index.cpe.zip").getPath());
-                fis = new FileInputStream(path);
-                zin = new ZipInputStream(new BufferedInputStream(fis));
-                ZipEntry entry;
-                while ((entry = zin.getNextEntry()) != null) {
-                    if (entry.isDirectory()) {
-                        continue;
-                    }
-                    FileOutputStream fos = null;
-                    BufferedOutputStream dest = null;
-                    try {
-                        File o = new File(indexPath, entry.getName());
-                        o.createNewFile();
-                        fos = new FileOutputStream(o, false);
-                        dest = new BufferedOutputStream(fos, BUFFER_SIZE);
-                        byte data[] = new byte[BUFFER_SIZE];
-                        int count;
-                        while ((count = zin.read(data, 0, BUFFER_SIZE)) != -1) {
-                            dest.write(data, 0, count);
-                        }
-                    } catch (Exception ex) {
-                        Logger.getLogger(BaseIndexTestCase.class.getName()).log(Level.FINEST, null, ex);
-                    } finally {
-                        if (dest != null) {
-                            try {
-                                dest.flush();
-                                dest.close();
-                            } catch (Throwable ex) {
-                                Logger.getLogger(BaseIndexTestCase.class.getName()).log(Level.FINEST, null, ex);
-                            }
-                        }
-                        if (fos != null) {
-                            try {
-                                fos.close();
-                            } catch (Throwable ex) {
-                                Logger.getLogger(BaseIndexTestCase.class.getName()).log(Level.FINEST, null, ex);
-                            }
-                        }
-                    }
-                }
-            } finally {
-                try {
-                    if (zin != null) {
-                        zin.close();
-                    }
-                } catch (Throwable ex) {
-                    Logger.getLogger(BaseIndexTestCase.class.getName()).log(Level.FINEST, null, ex);
-                }
-                try {
-                    if (fis != null) {
-                        fis.close();
-                    }
-                } catch (Throwable ex) {
-                    Logger.getLogger(BaseIndexTestCase.class.getName()).log(Level.FINEST, null, ex);
-                }
-            }
-        }
+        super.tearDown();
     }
 }

dependency-check-core/src/test/java/org/owasp/dependencycheck/data/cpe/IndexEntryTest.java

@@ -19,6 +19,7 @@
 package org.owasp.dependencycheck.data.cpe;
 
 import org.owasp.dependencycheck.data.cpe.IndexEntry;
+import junit.framework.TestCase;
 import org.junit.After;
 import org.junit.AfterClass;
 import org.junit.Before;
@@ -30,7 +31,7 @@ import org.junit.Assert;
  *
  * @author Jeremy Long (jeremy.long@owasp.org)
  */
-public class IndexEntryTest {
+public class IndexEntryTest extends TestCase {
 
     @BeforeClass
     public static void setUpClass() throws Exception {
@@ -41,11 +42,15 @@ public class IndexEntryTest {
     }
 
     @Before
-    public void setUp() {
+    @Override
+    public void setUp() throws Exception {
+        super.setUp();
     }
 
     @After
-    public void tearDown() {
+    @Override
+    public void tearDown() throws Exception {
+        super.tearDown();
     }
 
     /**

dependency-check-core/src/test/java/org/owasp/dependencycheck/data/cpe/IndexTest.java

@@ -1,103 +0,0 @@
-/*
- * This file is part of dependency-check-core.
- *
- * Dependency-check-core is free software: you can redistribute it and/or modify it
- * under the terms of the GNU General Public License as published by the Free
- * Software Foundation, either version 3 of the License, or (at your option) any
- * later version.
- *
- * Dependency-check-core is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
- * details.
- *
- * You should have received a copy of the GNU General Public License along with
- * dependency-check-core. If not, see http://www.gnu.org/licenses/.
- *
- * Copyright (c) 2012 Jeremy Long. All Rights Reserved.
- */
-package org.owasp.dependencycheck.data.cpe;
-
-import org.owasp.dependencycheck.data.cpe.Index;
-import java.io.File;
-import java.io.IOException;
-import java.util.logging.Level;
-import java.util.logging.Logger;
-import org.apache.lucene.document.Document;
-import org.apache.lucene.queryparser.classic.ParseException;
-import org.apache.lucene.search.ScoreDoc;
-import org.apache.lucene.search.TopDocs;
-import org.apache.lucene.store.Directory;
-import org.junit.After;
-import org.junit.AfterClass;
-import org.junit.Before;
-import org.junit.BeforeClass;
-import org.junit.Test;
-import static org.junit.Assert.*;
-
-/**
- *
- * @author Jeremy Long (jeremy.long@owasp.org)
- */
-public class IndexTest {
-
-    @BeforeClass
-    public static void setUpClass() throws Exception {
-    }
-
-    @AfterClass
-    public static void tearDownClass() throws Exception {
-    }
-
-    @Before
-    public void setUp() {
-    }
-
-    @After
-    public void tearDown() {
-    }
-
-    /**
-     * Test of open method, of class Index.
-     */
-    @Test
-    public void testOpen() {
-        Index instance = new Index();
-        try {
-            instance.open();
-            //TODO research why are we getting multiple documents for the same documentId. is the update method not working?
-//            try {
-//                instance.createSearchingAnalyzer();
-//                TopDocs docs = instance.search("product:( project\\-open )", 20);
-//                for (ScoreDoc d : docs.scoreDocs) {
-//                    final Document doc = instance.getDocument(d.doc);
-//                    String vendor = doc.getField(Fields.VENDOR).stringValue();
-//                    String product = doc.getField(Fields.PRODUCT).stringValue();
-//                    System.out.print(d.doc);
-//                    System.out.print(" : ");
-//                    System.out.print(vendor + ":");
-//                    System.out.println(product);
-//                }
-//            } catch (ParseException ex) {
-//                Logger.getLogger(IndexTest.class.getName()).log(Level.SEVERE, null, ex);
-//            }
-        } catch (IOException ex) {
-            assertNull(ex.getMessage(), ex);
-        }
-        instance.close();
-    }
-
-    /**
-     * Test of getDirectory method, of class Index.
-     *
-     * @throws Exception
-     */
-    @Test
-    public void testGetDirectory() throws Exception {
-        Index index = new Index();
-        Directory result = index.getDirectory();
-
-        String exp = File.separatorChar + "target" + File.separatorChar + "data" + File.separatorChar + "cpe";
-        assertTrue(result.toString().contains(exp));
-    }
-}

dependency-check-core/src/test/java/org/owasp/dependencycheck/data/nvdcve/BaseDBTestCase.java

@@ -30,6 +30,7 @@ import java.util.logging.Logger;
 import java.util.zip.ZipEntry;
 import java.util.zip.ZipInputStream;
 import junit.framework.TestCase;
+import org.owasp.dependencycheck.data.update.DataStoreMetaInfo;
 import org.owasp.dependencycheck.utils.Settings;
 
 /**
@@ -46,32 +47,28 @@ public abstract class BaseDBTestCase extends TestCase {
         ensureDBExists();
     }
 
-    protected static File getDataDirectory() throws IOException {
-        final String fileName = Settings.getString(Settings.KEYS.CVE_DATA_DIRECTORY);
-        final String dataDirectory = Settings.getString(Settings.KEYS.DATA_DIRECTORY);
-        return new File(dataDirectory, fileName);
-    }
-
     public static void ensureDBExists() throws Exception {
-        String indexPath = getDataDirectory().getCanonicalPath();
-        java.io.File f = new File(indexPath);
-        if (!f.exists() || (f.isDirectory() && f.listFiles().length == 0)) {
-            f.mkdirs();
+
+        java.io.File dataPath = Settings.getFile(Settings.KEYS.DATA_DIRECTORY);
+        if (!dataPath.exists() || (dataPath.isDirectory() && dataPath.listFiles().length < 3)) {
+            dataPath.mkdirs();
             FileInputStream fis = null;
             ZipInputStream zin = null;
             try {
-                File path = new File(BaseDBTestCase.class.getClassLoader().getResource("db.cve.zip").getPath());
+                File path = new File(BaseDBTestCase.class.getClassLoader().getResource("data.zip").getPath());
                 fis = new FileInputStream(path);
                 zin = new ZipInputStream(new BufferedInputStream(fis));
                 ZipEntry entry;
                 while ((entry = zin.getNextEntry()) != null) {
                     if (entry.isDirectory()) {
+                        final File d = new File(dataPath, entry.getName());
+                        d.mkdir();
                         continue;
                     }
                     FileOutputStream fos = null;
                     BufferedOutputStream dest = null;
                     try {
-                        File o = new File(indexPath, entry.getName());
+                        File o = new File(dataPath, entry.getName());
                         o.createNewFile();
                         fos = new FileOutputStream(o, false);
                         dest = new BufferedOutputStream(fos, BUFFER_SIZE);
@@ -81,7 +78,7 @@ public abstract class BaseDBTestCase extends TestCase {
                             dest.write(data, 0, count);
                         }
                     } catch (Exception ex) {
-                        Logger.getLogger(BaseDBTestCase.class.getName()).log(Level.FINEST, null, ex);
+                        Logger.getLogger(BaseDBTestCase.class.getName()).log(Level.SEVERE, null, ex);
                     } finally {
                         try {
                             if (dest != null) {

dependency-check-core/src/test/java/org/owasp/dependencycheck/data/nvdcve/NvdCve_1_2_HandlerTest.java

@@ -16,9 +16,9 @@
  *
  * Copyright (c) 2012 Jeremy Long. All Rights Reserved.
  */
-package org.owasp.dependencycheck.data.nvdcve.xml;
+package org.owasp.dependencycheck.data.nvdcve;
 
-import org.owasp.dependencycheck.data.nvdcve.xml.NvdCve12Handler;
+import org.owasp.dependencycheck.data.nvdcve.NvdCve12Handler;
 import java.io.File;
 import java.util.List;
 import java.util.Map;

dependency-check-core/src/test/java/org/owasp/dependencycheck/data/nvdcve/NvdCve_2_0_HandlerTest.java

@@ -16,9 +16,9 @@
  *
  * Copyright (c) 2012 Jeremy Long. All Rights Reserved.
  */
-package org.owasp.dependencycheck.data.nvdcve.xml;
+package org.owasp.dependencycheck.data.nvdcve;
 
-import org.owasp.dependencycheck.data.nvdcve.xml.NvdCve20Handler;
+import org.owasp.dependencycheck.data.nvdcve.NvdCve20Handler;
 import java.io.File;
 import javax.xml.parsers.SAXParser;
 import javax.xml.parsers.SAXParserFactory;

dependency-check-core/src/test/java/org/owasp/dependencycheck/data/update/DatabaseUpdaterIntegrationTest.java

@@ -16,14 +16,17 @@
  *
  * Copyright (c) 2012 Jeremy Long. All Rights Reserved.
  */
-package org.owasp.dependencycheck.data.nvdcve.xml;
+package org.owasp.dependencycheck.data.update;
 
-import org.owasp.dependencycheck.data.nvdcve.xml.DatabaseUpdater;
+import java.io.File;
+import java.net.URL;
+import org.owasp.dependencycheck.data.update.DatabaseUpdater;
 import org.junit.After;
 import org.junit.AfterClass;
 import org.junit.Before;
 import org.junit.BeforeClass;
 import org.junit.Test;
+import org.owasp.dependencycheck.utils.Settings;
 
 /**
  *

dependency-check-core/src/test/java/org/owasp/dependencycheck/data/update/DatabaseUpdater_1_Test.java

@@ -0,0 +1,97 @@
+/*
+ * This file is part of dependency-check-core.
+ *
+ * Dependency-check-core is free software: you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation, either version 3 of the License, or (at your option) any
+ * later version.
+ *
+ * Dependency-check-core is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+ * details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * dependency-check-core. If not, see http://www.gnu.org/licenses/.
+ *
+ * Copyright (c) 2012 Jeremy Long. All Rights Reserved.
+ */
+package org.owasp.dependencycheck.data.update;
+
+import org.owasp.dependencycheck.data.update.DatabaseUpdater;
+import java.io.File;
+import org.apache.commons.io.FileUtils;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.owasp.dependencycheck.utils.Settings;
+
+/**
+ *
+ * @author Jeremy Long (jeremy.long@owasp.org)
+ */
+public class DatabaseUpdater_1_Test {
+
+    public DatabaseUpdater_1_Test() {
+    }
+
+    @BeforeClass
+    public static void setUpClass() throws Exception {
+    }
+
+    @AfterClass
+    public static void tearDownClass() throws Exception {
+    }
+    private String old12;
+    private String old20;
+
+    @Before
+    public void setUp() throws Exception {
+        old12 = Settings.getString(Settings.KEYS.CVE_MODIFIED_12_URL);
+        old20 = Settings.getString(Settings.KEYS.CVE_MODIFIED_20_URL);
+
+        File tmp = Settings.getTempDirectory();
+        if (!tmp.exists()) {
+            tmp.mkdirs();
+        }
+
+        File dest = new File(tmp, "data.zip");
+        File file = new File(this.getClass().getClassLoader().getResource("data.zip").toURI());
+        FileUtils.copyFile(file, dest);
+        String path = "file:///" + dest.getCanonicalPath();
+        Settings.setString(Settings.KEYS.BATCH_UPDATE_URL, path);
+
+        dest = new File(tmp, "nvdcve-2012.xml");
+        file = new File(this.getClass().getClassLoader().getResource("nvdcve-2012.xml").toURI());
+        FileUtils.copyFile(file, dest);
+        path = "file:///" + dest.getCanonicalPath();
+        Settings.setString(Settings.KEYS.CVE_MODIFIED_12_URL, path);
+
+        dest = new File(tmp, "nvdcve-2.0-2012.xml");
+        file = new File(this.getClass().getClassLoader().getResource("nvdcve-2.0-2012.xml").toURI());
+        FileUtils.copyFile(file, dest);
+        path = "file:///" + dest.getCanonicalPath();
+        Settings.setString(Settings.KEYS.CVE_MODIFIED_20_URL, path);
+    }
+
+    @After
+    public void tearDown() {
+        Settings.setString(Settings.KEYS.CVE_MODIFIED_12_URL, old12);
+        Settings.setString(Settings.KEYS.CVE_MODIFIED_20_URL, old20);
+        Settings.setString(Settings.KEYS.BATCH_UPDATE_URL, "");
+    }
+
+    /**
+     * Test of update method (when in batch mode), of class DatabaseUpdater.
+     *
+     * @throws Exception
+     */
+    @Test
+    public void testBatchUpdate() throws Exception {
+        DatabaseUpdater instance = new DatabaseUpdater();
+        instance.deleteExistingData();
+        instance.update();
+    }
+}

dependency-check-core/src/test/java/org/owasp/dependencycheck/data/update/DatabaseUpdater_2_Test.java

@@ -0,0 +1,87 @@
+/*
+ * This file is part of dependency-check-core.
+ *
+ * Dependency-check-core is free software: you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation, either version 3 of the License, or (at your option) any
+ * later version.
+ *
+ * Dependency-check-core is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+ * details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * dependency-check-core. If not, see http://www.gnu.org/licenses/.
+ *
+ * Copyright (c) 2012 Jeremy Long. All Rights Reserved.
+ */
+package org.owasp.dependencycheck.data.update;
+
+import org.owasp.dependencycheck.data.update.DatabaseUpdater;
+import java.io.File;
+import org.apache.commons.io.FileUtils;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.owasp.dependencycheck.utils.Settings;
+
+/**
+ *
+ * @author Jeremy Long (jeremy.long@owasp.org)
+ */
+public class DatabaseUpdater_2_Test {
+
+    public DatabaseUpdater_2_Test() {
+    }
+
+    @BeforeClass
+    public static void setUpClass() throws Exception {
+    }
+
+    @AfterClass
+    public static void tearDownClass() throws Exception {
+    }
+    private String old12;
+    private String old20;
+
+    @Before
+    public void setUp() throws Exception {
+        old12 = Settings.getString(Settings.KEYS.CVE_MODIFIED_12_URL);
+        old20 = Settings.getString(Settings.KEYS.CVE_MODIFIED_20_URL);
+        Settings.removeProperty(Settings.KEYS.CVE_MODIFIED_12_URL);
+        Settings.removeProperty(Settings.KEYS.CVE_MODIFIED_20_URL);
+
+        File tmp = Settings.getTempDirectory();
+        if (!tmp.exists()) {
+            tmp.mkdirs();
+        }
+
+        File dest = new File(tmp, "data.zip");
+        File file = new File(this.getClass().getClassLoader().getResource("data.zip").toURI());
+        FileUtils.copyFile(file, dest);
+        String path = "file:///" + dest.getCanonicalPath();
+        Settings.setString(Settings.KEYS.BATCH_UPDATE_URL, path);
+    }
+
+    @After
+    public void tearDown() {
+        Settings.setString(Settings.KEYS.CVE_MODIFIED_12_URL, old12);
+        Settings.setString(Settings.KEYS.CVE_MODIFIED_20_URL, old20);
+        Settings.removeProperty(Settings.KEYS.BATCH_UPDATE_URL);
+    }
+
+    /**
+     * Test of update method (when in batch mode), of class DatabaseUpdater.
+     *
+     * @throws Exception
+     */
+    @Test
+    public void testBatchUpdateWithoutModified() throws Exception {
+        DatabaseUpdater instance = new DatabaseUpdater();
+        instance.deleteExistingData();
+        instance.update();
+    }
+}

dependency-check-core/src/test/java/org/owasp/dependencycheck/utils/DownloaderIntegrationTest.java

@@ -18,6 +18,7 @@
  */
 package org.owasp.dependencycheck.utils;
 
+import java.io.File;
 import org.owasp.dependencycheck.utils.Settings;
 import org.owasp.dependencycheck.utils.Downloader;
 import java.net.URL;
@@ -59,17 +60,12 @@ public class DownloaderIntegrationTest {
     public void testFetchFile() throws Exception {
 
 //        Settings.setString(Settings.KEYS.CONNECTION_TIMEOUT, "1000");
-
 //        Settings.setString(Settings.KEYS.PROXY_PORT, "8080");
 //        Settings.setString(Settings.KEYS.PROXY_URL, "127.0.0.1");
 
-        URL url = new URL(Settings.getString(Settings.KEYS.CPE_URL));
-        String outputPath = "target/downloaded_cpe.xml";
-        Downloader.fetchFile(url, outputPath, true);
-
-        url = new URL(Settings.getString(Settings.KEYS.CVE_MODIFIED_20_URL));
-        outputPath = "target/downloaded_cve.xml";
-        Downloader.fetchFile(url, outputPath, false);
+        URL url = new URL(Settings.getString(Settings.KEYS.CVE_MODIFIED_20_URL));
+        File outputPath = new File("target/downloaded_cve.xml");
+        Downloader.fetchFile(url, outputPath);
 
     }
 

dependency-check-core/src/test/java/org/owasp/dependencycheck/utils/DownloaderTest.java

@@ -0,0 +1,60 @@
+/*
+ * This file is part of dependency-check-core.
+ *
+ * Dependency-check-core is free software: you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation, either version 3 of the License, or (at your option) any
+ * later version.
+ *
+ * Dependency-check-core is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+ * details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * dependency-check-core. If not, see http://www.gnu.org/licenses/.
+ *
+ * Copyright (c) 2012 Jeremy Long. All Rights Reserved.
+ */
+package org.owasp.dependencycheck.utils;
+
+import java.io.File;
+import org.owasp.dependencycheck.utils.Downloader;
+import java.net.URL;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import static org.junit.Assert.*;
+
+/**
+ *
+ * @author Jeremy Long (jeremy.long@owasp.org)
+ */
+public class DownloaderTest {
+
+    @BeforeClass
+    public static void setUpClass() throws Exception {
+    }
+
+    @AfterClass
+    public static void tearDownClass() throws Exception {
+    }
+
+    @Before
+    public void setUp() {
+    }
+
+    @After
+    public void tearDown() {
+    }
+
+    @Test
+    public void testGetLastModified_file() throws Exception {
+        File f = new File("target/test-classes/nvdcve-2.0-2012.xml");
+        URL url = new URL("file:///" + f.getCanonicalPath());
+        long timestamp = Downloader.getLastModified(url);
+        assertTrue("timestamp equal to zero?", timestamp > 0);
+    }
+}

dependency-check-core/src/test/java/org/owasp/dependencycheck/utils/SettingsTest.java

@@ -73,9 +73,9 @@ public class SettingsTest {
         File result = Settings.getFile(key);
         Assert.assertTrue(result.getAbsolutePath().endsWith(expResult));
 
-        key = "an invalid key!!!";
-        result = Settings.getFile(key, expResult);
-        Assert.assertTrue(result.getAbsolutePath().endsWith(expResult));
+        result = Settings.getFile(Settings.KEYS.DATA_DIRECTORY);
+        String path = result.getPath();
+        Assert.assertTrue(path.endsWith("data") || path.endsWith("data" + File.separator));
     }
 
     /**
@@ -163,4 +163,20 @@ public class SettingsTest {
         boolean result = Settings.getBoolean(key);
         Assert.assertEquals(expResult, result);
     }
+
+    /**
+     * Test of removeProperty method, of class Settings.
+     */
+    @Test
+    public void testRemoveProperty() {
+        String key = "SomeKey";
+        String value = "value";
+        String dfault = "default";
+        Settings.setString(key, value);
+        String ret = Settings.getString(key);
+        Assert.assertEquals(value, ret);
+        Settings.removeProperty(key);
+        ret = Settings.getString(key, dfault);
+        Assert.assertEquals(dfault, ret);
+    }
 }

dependency-check-jenkins/README.md

@@ -0,0 +1,33 @@
+Dependency-Check Jenkins Plugin
+==============================
+
+The Dependency-Check Jenkins Plugin features the ability to perform a dependency
+analysis build and later view results post build. The plugin is built using [analysis-core]
+and features many of the same features that Jenkins static analysis plugins offer,
+including thresholds, charts and the ability to view vulnerability information should
+a dependency have one identified.
+
+The main repository is located at [jenkins-cli/dependency-check-jenkins](https://github.com/jenkinsci/dependency-check-jenkins).
+
+The main site for documentation is located at [OWASP Dependency-Check-Jenkins](https://wiki.jenkins-ci.org/display/JENKINS/OWASP+Dependency-Check+Plugin).
+
+More information can be found on the [wiki].
+
+Mailing List
+------------
+
+Subscribe: [dependency-check+subscribe@googlegroups.com](mailto:dependency-check+subscribe@googlegroups.com)
+
+Post: [dependency-check@googlegroups.com](mailto:dependency-check@googlegroups.com)
+
+Copyright & License
+-------------------
+
+Dependency-Check Jenkins Plugin is Copyright (c) 2013 Steve Springett. All Rights Reserved.
+
+Permission to modify and redistribute is granted under the terms of the GPLv3 license. See the [LICENSE.txt] [GPLv3] file for the full license.
+
+  [wiki]: https://github.com/jenkinsci/dependency-check-jenkins/wiki
+  [analysis-core]: http://wiki.jenkins-ci.org/x/CwDgAQ
+  [GPLv3]: https://github.com/jenkinsci/dependency-check-jenkins/blob/master/LICENSE.txt
+  [notices]: https://github.com/jenkinsci/dependency-check-jenkins/blob/master/NOTICES.txt

dependency-check-jenkins/pom.xml

@@ -0,0 +1,94 @@
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+
+    <description>This plug-in can independently execute a Dependency-Check analysis and visualize the results.</description>
+    <url>http://wiki.jenkins-ci.org/display/JENKINS/OWASP+Dependency-Check+Plugin</url>
+    <parent>
+        <groupId>org.owasp</groupId>
+        <artifactId>dependency-check-parent</artifactId>
+        <version>1.0.2</version>
+    </parent>
+
+    <groupId>org.owasp</groupId>
+    <artifactId>dependency-check-jenkins</artifactId>
+    <name>Dependency-Check Jenkins Plugin</name>
+    <packaging>jar</packaging>
+    <inceptionYear>2012</inceptionYear>
+    <organization>
+        <name>OWASP</name>
+        <url>http://www.owasp.org</url>
+    </organization>
+
+    <developers>
+        <developer>
+            <name>Steve Springett</name>
+            <email>steve.springett@owasp.org</email>
+            <organization>OWASP</organization>
+            <organizationUrl>https://www.owasp.org/index.php/OWASP_Dependency_Check</organizationUrl>
+            <roles>
+                <role>architect</role>
+                <role>developer</role>
+            </roles>
+        </developer>
+    </developers>
+    <!-- begin copy from http://minds.coremedia.com/2012/09/11/problem-solved-deploy-multi-module-maven-project-site-as-github-pages/ -->
+    <distributionManagement>
+        <site>
+            <id>github-pages-site</id>
+            <name>Deployment through GitHub's site deployment plugin</name>
+            <url>${basedir}/../target/site/${project.version}/dependency-check-maven</url>
+        </site>
+    </distributionManagement>
+    <!-- end copy -->
+    <scm>
+        <connection>scm:git:git@github.com:jenkinsci/dependency-check-jenkins.git</connection>
+        <url>https://github.com/jenkinsci/dependency-check-jenkins</url>
+        <developerConnection>scm:git:git@github.com:jenkinsci/dependency-check-jenkins.git</developerConnection>
+    </scm>
+    <issueManagement>
+        <system>github</system>
+        <url>https://github.com/jenkinsci/dependency-check-jenkins/issues</url>
+    </issueManagement>
+    <licenses>
+        <license>
+            <name>GNU General Public License version 3</name>
+            <url>http://www.gnu.org/licenses/</url>
+        </license>
+    </licenses>
+    <build>
+        <plugins>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-site-plugin</artifactId>
+                <version>3.3</version>
+                <dependencies>
+                    <dependency>
+                        <groupId>org.apache.maven.doxia</groupId>
+                        <artifactId>doxia-module-markdown</artifactId>
+                        <version>1.4</version>
+                    </dependency>
+                </dependencies>
+                <configuration>
+                    <skipDeploy>true</skipDeploy>
+                    <reportPlugins>
+                        <plugin>
+                            <groupId>org.apache.maven.plugins</groupId>
+                            <artifactId>maven-project-info-reports-plugin</artifactId>
+                            <version>2.6</version>
+                            <reportSets>
+                                <reportSet>
+                                    <reports>
+                                        <report>index</report>
+                                        <report>summary</report>
+                                        <report>license</report>
+                                        <report>help</report>
+                                    </reports>
+                                </reportSet>
+                            </reportSets>
+                        </plugin>
+                    </reportPlugins>
+                </configuration>
+            </plugin>
+        </plugins>
+    </build>
+</project>

dependency-check-jenkins/src/site/markdown/index.md

@@ -0,0 +1,33 @@
+Dependency-Check Jenkins Plugin
+==============================
+
+The Dependency-Check Jenkins Plugin features the ability to perform a dependency
+analysis build and later view results post build. The plugin is built using [analysis-core]
+and features many of the same features that Jenkins static analysis plugins offer,
+including thresholds, charts and the ability to view vulnerability information should
+a dependency have one identified.
+
+The main repository is located at [jenkins-cli/dependency-check-jenkins](https://github.com/jenkinsci/dependency-check-jenkins).
+
+The main site for documentation is located at [OWASP Dependency-Check-Jenkins](https://wiki.jenkins-ci.org/display/JENKINS/OWASP+Dependency-Check+Plugin).
+
+More information can be found on the [wiki].
+
+Mailing List
+------------
+
+Subscribe: [dependency-check+subscribe@googlegroups.com](mailto:dependency-check+subscribe@googlegroups.com)
+
+Post: [dependency-check@googlegroups.com](mailto:dependency-check@googlegroups.com)
+
+Copyright & License
+-------------------
+
+Dependency-Check Jenkins Plugin is Copyright (c) 2013 Steve Springett. All Rights Reserved.
+
+Permission to modify and redistribute is granted under the terms of the GPLv3 license. See the [LICENSE.txt] [GPLv3] file for the full license.
+
+  [wiki]: https://github.com/jenkinsci/dependency-check-jenkins/wiki
+  [analysis-core]: http://wiki.jenkins-ci.org/x/CwDgAQ
+  [GPLv3]: https://github.com/jenkinsci/dependency-check-jenkins/blob/master/LICENSE.txt
+  [notices]: https://github.com/jenkinsci/dependency-check-jenkins/blob/master/NOTICES.txt

dependency-check-jenkins/src/site/site.xml

@@ -0,0 +1,31 @@
+<?xml version="1.0" encoding="ISO-8859-1"?>
+<!--
+This file is part of dependency-check-jenkins.
+
+Dependency-check-jenkins is free software: you can redistribute it and/or modify it
+under the terms of the GNU General Public License as published by the Free
+Software Foundation, either version 3 of the License, or (at your option) any
+later version.
+
+Dependency-check-jenkins is distributed in the hope that it will be useful, but
+WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+details.
+
+You should have received a copy of the GNU General Public License along with
+dependency-check-jenkins. If not, see http://www.gnu.org/licenses/.
+
+Copyright (c) 2013 Jeremy Long. All Rights Reserved.
+-->
+<project name="dependency-check-jenkins">
+    <bannerLeft>
+        <name>dependency-check-jenkins</name>
+    </bannerLeft>
+    <body>
+        <breadcrumbs>
+            <item name="dependency-check" href="../index.html"/>
+        </breadcrumbs>
+        <menu ref="Project Documentation" />
+        <menu ref="reports" />
+    </body>
+</project>

dependency-check-maven/pom.xml

@@ -24,13 +24,13 @@ Copyright (c) 2013 Jeremy Long. All Rights Reserved.
     <parent>
         <groupId>org.owasp</groupId>
         <artifactId>dependency-check-parent</artifactId>
-        <version>1.0.1</version>
+        <version>1.0.2</version>
     </parent>
 
     <artifactId>dependency-check-maven</artifactId>
     <packaging>maven-plugin</packaging>
 
-    <name>dependency-check-maven</name>
+    <name>Dependency-Check Maven Plugin</name>
     <description>Dependency-Check-Maven is a Maven Plugin that attempts to detect publicly disclosed vulnerabilities contained within project dependencies. It does this by determining if there is a Common Platform Enumeration (CPE) identifier for a given dependency. If found, it will generate a report linking to the associated CVE entries.</description>
     <inceptionYear>2013</inceptionYear>
     <!-- begin copy from http://minds.coremedia.com/2012/09/11/problem-solved-deploy-multi-module-maven-project-site-as-github-pages/ -->

pom.xml

@@ -22,7 +22,7 @@ along with Dependency-Check.  If not, see <http://www.gnu.org/licenses />.
 
     <groupId>org.owasp</groupId>
     <artifactId>dependency-check-parent</artifactId>
-    <version>1.0.1</version>
+    <version>1.0.2</version>
     <packaging>pom</packaging>
 
     <parent>
@@ -35,8 +35,9 @@ along with Dependency-Check.  If not, see <http://www.gnu.org/licenses />.
         <module>dependency-check-cli</module>
         <module>dependency-check-ant</module>
         <module>dependency-check-maven</module>
+        <module>dependency-check-jenkins</module>
     </modules>
-    <name>dependency-check-parent</name>
+    <name>Dependency-Check</name>
     <url>https://github.com/jeremylong/DependencyCheck.git</url>
     <description>Dependency-check is a utility that attempts to detect publicly disclosed vulnerabilities contained within project dependencies. It does this by determining if there is a Common Platform Enumeration (CPE) identifier for a given dependency. If found, it will generate a report linking to the associated CVE entries.</description>
     <inceptionYear>2012</inceptionYear>
@@ -77,7 +78,7 @@ along with Dependency-Check.  If not, see <http://www.gnu.org/licenses />.
     </contributors>
     <scm>
         <connection>scm:git:git@github.com:jeremylong/DependencyCheck.git</connection>
-        <url>https://github.com/jeremylong/DependencyCheck.git</url>
+        <url>https://github.com/jeremylong/DependencyCheck</url>
         <developerConnection>scm:git:git@github.com:jeremylong/DependencyCheck.git</developerConnection>
         <tag>HEAD</tag>
     </scm>

src/site/markdown/index.md

@@ -28,4 +28,4 @@ Dependency-check's core analysis library is exposed in various forms:
 -  [Command Line Tool](dependency-check-cli/index.html)
 -  [Maven Plugin](dependency-check-maven/usage.html)
 -  [Ant Task](dependency-check-ant/installation.html)
--  Jenkins Plugin
+-  [Jenkins Plugin](dependency-check-jenkins/index.html)

src/site/site.xml

@@ -61,15 +61,18 @@ Copyright (c) 2013 Jeremy Long. All Rights Reserved.
             <item name="dependency-check-core" href="./dependency-check-core/index.html">
                 <description>The core dependency-check library</description>
             </item>
-            <item name="dependency-check-cli" href="./dependency-check-cli/index.html">
+            <item name="dependency-check-cli" href="./dependency-check-cli/installation.html">
                 <description>The command line interface for dependency-check.</description>
             </item>
-            <item name="dependency-check-ant" href="./dependency-check-ant/index.html">
+            <item name="dependency-check-ant" href="./dependency-check-ant/installation.html">
                 <description>An Ant task to run dependency-check.</description>
             </item>
-            <item name="dependency-check-maven" href="./dependency-check-maven/index.html">
+            <item name="dependency-check-maven" href="./dependency-check-maven/usage.html">
                 <description>A Maven plugin for dependency-check.</description>
             </item>
+            <item name="dependency-check-jenkins" href="./dependency-check-jenkins/index.html">
+                <description>A Jenkins plugin for dependency-check.</description>
+            </item>
         </menu>
         <footer/>
     </body>