Merge branch 'master' of github.com:jeremylong/DependencyCheck

dependency-check-cli/src/site/markdown/index.md.vm

@@ -14,9 +14,16 @@ script executable:
 
     $ chmod +777 dependency-check.sh
 
-To scan a folder on the system you can run:
 #set( $H = '#' )
 
+$H$H$H Homebrew
+    $ brew install dependency-check
+
+This puts an executable `dependency-check` script in the `/bin` directory of
+your homebrew installation.
+
+To scan a folder on the system you can run:
+
 $H$H$H Windows
     dependency-check.bat --app "My App Name" --scan "c:\java\application\lib"
 

dependency-check-core/src/main/java/org/owasp/dependencycheck/analyzer/ArchiveAnalyzer.java

@@ -17,22 +17,6 @@
  */
 package org.owasp.dependencycheck.analyzer;
 
-import java.io.BufferedInputStream;
-import java.io.BufferedOutputStream;
-import java.io.File;
-import java.io.FileFilter;
-import java.io.FileInputStream;
-import java.io.FileNotFoundException;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.Enumeration;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-
 import org.apache.commons.compress.archivers.ArchiveEntry;
 import org.apache.commons.compress.archivers.ArchiveInputStream;
 import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;
@@ -40,6 +24,8 @@ import org.apache.commons.compress.archivers.zip.ZipArchiveEntry;
 import org.apache.commons.compress.archivers.zip.ZipArchiveInputStream;
 import org.apache.commons.compress.archivers.zip.ZipFile;
 import org.apache.commons.compress.compressors.CompressorInputStream;
+import org.apache.commons.compress.compressors.bzip2.BZip2CompressorInputStream;
+import org.apache.commons.compress.compressors.bzip2.BZip2Utils;
 import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream;
 import org.apache.commons.compress.compressors.gzip.GzipUtils;
 import org.owasp.dependencycheck.Engine;
@@ -52,6 +38,9 @@ import org.owasp.dependencycheck.utils.Settings;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import java.io.*;
+import java.util.*;
+
 /**
  * <p>
  * An analyzer that extracts files from archives and ensures any supported files contained within the archive are added to the
@@ -100,20 +89,21 @@ public class ArchiveAnalyzer extends AbstractFileTypeAnalyzer {
      */
     private static final Set<String> ZIPPABLES = newHashSet("zip", "ear", "war", "jar", "sar", "apk", "nupkg");
     /**
-     * The set of file extensions supported by this analyzer. Note for developers, any additions to this list will need to be
+     * The set of file extensions supported by this analyzer. Note for developers, any additions to this list will need
-     * explicitly handled in extractFiles().
+     * to be explicitly handled in {@link #extractFiles(File, File, Engine)}.
      */
-    private static final Set<String> EXTENSIONS = newHashSet("tar", "gz", "tgz");
+    private static final Set<String> EXTENSIONS = newHashSet("tar", "gz", "tgz", "bz2", "tbz2");
 
     /**
      * Detects files with extensions to remove from the engine's collection of dependencies.
      */
-    private static final FileFilter REMOVE_FROM_ANALYSIS = FileFilterBuilder.newInstance().addExtensions("zip", "tar", "gz", "tgz").build();
+    private static final FileFilter REMOVE_FROM_ANALYSIS =
+            FileFilterBuilder.newInstance().addExtensions("zip", "tar", "gz", "tgz", "bz2", "tbz2").build();
 
     static {
         final String additionalZipExt = Settings.getString(Settings.KEYS.ADDITIONAL_ZIP_EXTENSIONS);
         if (additionalZipExt != null) {
-            final Set<String> ext = new HashSet<String>(Arrays.asList(additionalZipExt));
+            final Set<String> ext = new HashSet<String>(Collections.singletonList(additionalZipExt));
             ZIPPABLES.addAll(ext);
         }
         EXTENSIONS.addAll(ZIPPABLES);
@@ -215,15 +205,8 @@ public class ArchiveAnalyzer extends AbstractFileTypeAnalyzer {
         extractFiles(f, tmpDir, engine);
 
         //make a copy
-        List<Dependency> dependencies = new ArrayList<Dependency>(engine.getDependencies());
+        final Set<Dependency> dependencySet = findMoreDependencies(engine, tmpDir);
-        engine.scan(tmpDir);
+        if (!dependencySet.isEmpty()) {
-        List<Dependency> newDependencies = engine.getDependencies();
-        if (dependencies.size() != newDependencies.size()) {
-            //get the new dependencies
-            final Set<Dependency> dependencySet = new HashSet<Dependency>();
-            dependencySet.addAll(newDependencies);
-            dependencySet.removeAll(dependencies);
-
             for (Dependency d : dependencySet) {
                 //fix the dependency's display name and path
                 final String displayPath = String.format("%s%s",
@@ -245,6 +228,13 @@ public class ArchiveAnalyzer extends AbstractFileTypeAnalyzer {
             }
         }
         if (REMOVE_FROM_ANALYSIS.accept(dependency.getActualFile())) {
+            addDisguisedJarsToDependencies(dependency, engine);
+            engine.getDependencies().remove(dependency);
+        }
+        Collections.sort(engine.getDependencies());
+    }
+
+    private void addDisguisedJarsToDependencies(Dependency dependency, Engine engine) throws AnalysisException {
         if (ZIP_FILTER.accept(dependency.getActualFile()) && isZipFileActuallyJarFile(dependency)) {
             final File tdir = getNextTempDirectory();
             final String fileName = dependency.getFileName();
@@ -254,16 +244,10 @@ public class ArchiveAnalyzer extends AbstractFileTypeAnalyzer {
             final File tmpLoc = new File(tdir, fileName.substring(0, fileName.length() - 3) + "jar");
             try {
                 org.apache.commons.io.FileUtils.copyFile(tdir, tmpLoc);
-                    dependencies = new ArrayList<Dependency>(engine.getDependencies());
+                final Set<Dependency> dependencySet = findMoreDependencies(engine, tmpLoc);
-                    engine.scan(tmpLoc);
+                if (!dependencySet.isEmpty()) {
-                    newDependencies = engine.getDependencies();
-                    if (dependencies.size() != newDependencies.size()) {
-                        //get the new dependencies
-                        final Set<Dependency> dependencySet = new HashSet<Dependency>();
-                        dependencySet.addAll(newDependencies);
-                        dependencySet.removeAll(dependencies);
                     if (dependencySet.size() != 1) {
-                            LOGGER.info("Deep copy of ZIP to JAR file resulted in more then one dependency?");
+                        LOGGER.info("Deep copy of ZIP to JAR file resulted in more than one dependency?");
                     }
                     for (Dependency d : dependencySet) {
                         //fix the dependency's display name and path
@@ -275,10 +259,34 @@ public class ArchiveAnalyzer extends AbstractFileTypeAnalyzer {
                 LOGGER.debug("Unable to perform deep copy on '{}'", dependency.getActualFile().getPath(), ex);
             }
         }
-            engine.getDependencies().remove(dependency);
     }
-        Collections.sort(engine.getDependencies());
+
+    private static final Set<Dependency> EMPTY_DEPENDENCY_SET = Collections.emptySet();
+
+    /**
+     * Scan the given file/folder, and return any new dependencies found.
+     *
+     * @param engine used to scan
+     * @param file target of scanning
+     * @return any dependencies that weren't known to the engine before
+     */
+    private static Set<Dependency> findMoreDependencies(Engine engine, File file) {
+        List<Dependency> before = new ArrayList<Dependency>(engine.getDependencies());
+        engine.scan(file);
+        List<Dependency> after = engine.getDependencies();
+        final boolean sizeChanged = before.size() != after.size();
+        final Set<Dependency> newDependencies;
+        if (sizeChanged) {
+            //get the new dependencies
+            newDependencies = new HashSet<Dependency>();
+            newDependencies.addAll(after);
+            newDependencies.removeAll(before);
+        } else {
+            newDependencies = EMPTY_DEPENDENCY_SET;
         }
+        return newDependencies;
+    }
+
 
     /**
      * Retrieves the next temporary directory to extract an archive too.
@@ -309,11 +317,8 @@ public class ArchiveAnalyzer extends AbstractFileTypeAnalyzer {
      * @throws AnalysisException thrown if the archive is not found
      */
     private void extractFiles(File archive, File destination, Engine engine) throws AnalysisException {
-        if (archive == null || destination == null) {
+        if (archive != null && destination != null) {
-            return;
+            FileInputStream fis;
-        }
-
-        FileInputStream fis = null;
             try {
                 fis = new FileInputStream(archive);
             } catch (FileNotFoundException ex) {
@@ -332,6 +337,12 @@ public class ArchiveAnalyzer extends AbstractFileTypeAnalyzer {
                     if (engine.accept(f)) {
                         decompressFile(new GzipCompressorInputStream(new BufferedInputStream(fis)), f);
                     }
+                } else if ("bz2".equals(archiveExt) || "tbz2".equals(archiveExt)) {
+                    final String uncompressedName = BZip2Utils.getUncompressedFilename(archive.getName());
+                    final File f = new File(destination, uncompressedName);
+                    if (engine.accept(f)) {
+                        decompressFile(new BZip2CompressorInputStream(new BufferedInputStream(fis)), f);
+                    }
                 }
             } catch (ArchiveExtractionException ex) {
                 LOGGER.warn("Exception extracting archive '{}'.", archive.getName());
@@ -340,10 +351,7 @@ public class ArchiveAnalyzer extends AbstractFileTypeAnalyzer {
                 LOGGER.warn("Exception reading archive '{}'.", archive.getName());
                 LOGGER.debug("", ex);
             } finally {
-            try {
+                close(fis);
-                fis.close();
-            } catch (IOException ex) {
-                LOGGER.debug("", ex);
             }
         }
     }
@@ -360,17 +368,24 @@ public class ArchiveAnalyzer extends AbstractFileTypeAnalyzer {
         ArchiveEntry entry;
         try {
             while ((entry = input.getNextEntry()) != null) {
+                final File file = new File(destination, entry.getName());
                 if (entry.isDirectory()) {
-                    final File d = new File(destination, entry.getName());
+                    if (!file.exists() && !file.mkdirs()) {
-                    if (!d.exists()) {
+                        final String msg = String.format("Unable to create directory '%s'.", file.getAbsolutePath());
-                        if (!d.mkdirs()) {
-                            final String msg = String.format("Unable to create directory '%s'.", d.getAbsolutePath());
                         throw new AnalysisException(msg);
                     }
+                } else if (engine.accept(file)) {
+                    extractAcceptedFile(input, file);
                 }
-                } else {
+            }
-                    final File file = new File(destination, entry.getName());
+        } catch (Throwable ex) {
-                    if (engine.accept(file)) {
+            throw new ArchiveExtractionException(ex);
+        } finally {
+            close(input);
+        }
+    }
+
+    private static void extractAcceptedFile(ArchiveInputStream input, File file) throws AnalysisException {
         LOGGER.debug("Extracting '{}'", file.getPath());
         BufferedOutputStream bos = null;
         FileOutputStream fos = null;
@@ -399,36 +414,8 @@ public class ArchiveAnalyzer extends AbstractFileTypeAnalyzer {
             final String msg = String.format("IO Exception while parsing file '%s'.", file.getName());
             throw new AnalysisException(msg, ex);
         } finally {
-                            if (bos != null) {
+            close(bos);
-                                try {
+            close(fos);
-                                    bos.close();
-                                } catch (IOException ex) {
-                                    LOGGER.trace("", ex);
-                                }
-                            }
-                            if (fos != null) {
-                                try {
-                                    fos.close();
-                                } catch (IOException ex) {
-                                    LOGGER.trace("", ex);
-                                }
-                            }
-                        }
-                    }
-                }
-            }
-        } catch (IOException ex) {
-            throw new ArchiveExtractionException(ex);
-        } catch (Throwable ex) {
-            throw new ArchiveExtractionException(ex);
-        } finally {
-            if (input != null) {
-                try {
-                    input.close();
-                } catch (IOException ex) {
-                    LOGGER.trace("", ex);
-                }
-            }
         }
     }
 
@@ -445,7 +432,7 @@ public class ArchiveAnalyzer extends AbstractFileTypeAnalyzer {
         try {
             out = new FileOutputStream(outputFile);
             final byte[] buffer = new byte[BUFFER_SIZE];
-            int n = 0;
+            int n; // = 0
             while (-1 != (n = inputStream.read(buffer))) {
                 out.write(buffer, 0, n);
             }
@@ -456,15 +443,24 @@ public class ArchiveAnalyzer extends AbstractFileTypeAnalyzer {
             LOGGER.debug("", ex);
             throw new ArchiveExtractionException(ex);
         } finally {
-            if (out != null) {
+            close(out);
+        }
+    }
+
+    /**
+     * Close the given {@link Closeable} instance, ignoring nulls, and logging any thrown {@link IOException}.
+     *
+     * @param closeable to be closed
+     */
+    private static void close(Closeable closeable){
+        if (null != closeable) {
             try {
-                    out.close();
+                closeable.close();
             } catch (IOException ex) {
                 LOGGER.trace("", ex);
             }
         }
     }
-    }
 
     /**
      * Attempts to determine if a zip file is actually a JAR file.

dependency-check-core/src/test/java/org/owasp/dependencycheck/analyzer/ArchiveAnalyzerIntegrationTest.java

@@ -51,6 +51,8 @@ public class ArchiveAnalyzerIntegrationTest extends AbstractDatabaseTestCase {
         expResult.add("tar");
         expResult.add("gz");
         expResult.add("tgz");
+        expResult.add("bz2");
+        expResult.add("tbz2");
         for (String ext : expResult) {
             assertTrue(ext, instance.accept(new File("test." + ext)));
         }
@@ -197,28 +199,31 @@ public class ArchiveAnalyzerIntegrationTest extends AbstractDatabaseTestCase {
         }
     }
 
-//    /**
+    /**
-//     * Test of analyze method, of class ArchiveAnalyzer.
+     * Test of analyze method, of class ArchiveAnalyzer.
-//     */
+     */
-//    @Test
+    @Test
-//    public void testNestedZipFolder() throws Exception {
+    public void testAnalyzeTarBz2() throws Exception {
-//        ArchiveAnalyzer instance = new ArchiveAnalyzer();
+        ArchiveAnalyzer instance = new ArchiveAnalyzer();
-//        try {
+        instance.accept(new File("zip")); //ensure analyzer is "enabled"
-//            instance.initialize();
+        try {
-//
+            instance.initialize();
-//            File file = new File(this.getClass().getClassLoader().getResource("nested.zip").getPath());
+            File file = BaseTest.getResourceAsFile(this, "file.tar.bz2");
-//            Settings.setBoolean(Settings.KEYS.AUTO_UPDATE, false);
+            Settings.setBoolean(Settings.KEYS.AUTO_UPDATE, false);
-//            Settings.setBoolean(Settings.KEYS.ANALYZER_NEXUS_ENABLED, false);
+            Settings.setBoolean(Settings.KEYS.ANALYZER_NEXUS_ENABLED, false);
-//            Settings.setBoolean(Settings.KEYS.ANALYZER_CENTRAL_ENABLED, false);
+            Settings.setBoolean(Settings.KEYS.ANALYZER_CENTRAL_ENABLED, false);
-//            Engine engine = new Engine();
+            Engine engine = new Engine();
-//
+            int initial_size = engine.getDependencies().size();
-//            engine.scan(file);
+            engine.scan(file);
-//            engine.analyzeDependencies();
+            engine.analyzeDependencies();
-//
+            int ending_size = engine.getDependencies().size();
-//        } finally {
+            engine.cleanup();
-//            instance.close();
+            assertTrue(initial_size < ending_size);
-//        }
+        } finally {
-//    }
+            instance.close();
+        }
+    }
+
     /**
      * Test of analyze method, of class ArchiveAnalyzer.
      */
@@ -248,6 +253,31 @@ public class ArchiveAnalyzerIntegrationTest extends AbstractDatabaseTestCase {
         }
     }
 
+    /**
+     * Test of analyze method, of class ArchiveAnalyzer.
+     */
+    @Test
+    public void testAnalyzeTbz2() throws Exception {
+        ArchiveAnalyzer instance = new ArchiveAnalyzer();
+        instance.accept(new File("zip")); //ensure analyzer is "enabled"
+        try {
+            instance.initialize();
+            File file = BaseTest.getResourceAsFile(this, "file.tbz2");
+            Settings.setBoolean(Settings.KEYS.AUTO_UPDATE, false);
+            Settings.setBoolean(Settings.KEYS.ANALYZER_NEXUS_ENABLED, false);
+            Settings.setBoolean(Settings.KEYS.ANALYZER_CENTRAL_ENABLED, false);
+            Engine engine = new Engine();
+            int initial_size = engine.getDependencies().size();
+            engine.scan(file);
+            engine.analyzeDependencies();
+            int ending_size = engine.getDependencies().size();
+            engine.cleanup();
+            assertTrue(initial_size < ending_size);
+        } finally {
+            instance.close();
+        }
+    }
+
     /**
      * Test of analyze method, of class ArchiveAnalyzer.
      */

dependency-check-gradle/.gitignore

@@ -1,5 +1,6 @@
 .idea/
 .gradle
+gradle/
 
 *.iml
 *.ipr

dependency-check-gradle/README.md

@@ -7,6 +7,8 @@ This is a DependencyCheck gradle plugin designed for project which use Gradle as
 
 Dependency-Check is a utility that attempts to detect publicly disclosed vulnerabilities contained within project dependencies. It does this by determining if there is a Common Platform Enumeration (CPE) identifier for a given dependency. If found, it will generate a report linking to the associated CVE entries.
 
+Current latest version is `0.0.6`
+
 =========
 
 ## Usage
@@ -15,7 +17,7 @@ Dependency-Check is a utility that attempts to detect publicly disclosed vulnera
 
 Please refer to either one of the solution
 
-#### Solution 1，Install from Maven Central
+#### Solution 1，Install from Maven Central (Recommended)
 
 ```groovy
 buildscript {
@@ -23,7 +25,7 @@ buildscript {
         mavenCentral()
     }
     dependencies {
-        classpath 'com.thoughtworks.tools:dependency-check:0.0.5'
+        classpath 'com.thoughtworks.tools:dependency-check:0.0.6'
     }
 }
 ```
@@ -38,7 +40,7 @@ apply plugin: 'dependency.check'
 
 ```groovy
 plugins {
-    id "dependency.check" version "0.0.5"
+    id "dependency.check" version "0.0.6"
 }
 ```
 
@@ -52,11 +54,11 @@ buildscript {
     }
   }
   dependencies {
-    classpath "gradle.plugin.com.tools.security:dependency-check:0.0.5"
+    classpath "gradle.plugin.com.tools.security:dependency-check:0.0.6"
   }
 }
 
-apply plugin: "dependency.check"
+apply plugin: "dependency-check"
 ```
 
 #### Solution 3，Install from Bintray
@@ -73,7 +75,7 @@ buildscript {
     }
     dependencies {
         classpath(
-                'com.tools.security:dependency-check:0.0.5'
+                'com.tools.security:dependency-check:0.0.6'
         )
     }
 }
@@ -115,6 +117,19 @@ dependencyCheck {
 }
 ```
 
+In addition, if the proxy only allow HTTP `GET` or `POST` methods, you will find that the update process will always fail,
+ the root cause is that every time you run `dependencyCheck` task, it will try to query the latest timestamp to determine whether need to perform an update action,
+ and for performance reason the HTTP method it uses by default is `HEAD`, which probably is disabled or not supported by the proxy. To avoid this problem, you can simply change the HTTP method by below configuration:
+
+```groovy
+dependencyCheck {
+    proxyServer = "127.0.0.1"      // required, the server name or IP address of the proxy
+    proxyPort = 3128               // required, the port number of the proxy
+
+    quickQueryTimestamp = false    // when set to false, it means use HTTP GET method to query timestamp. (default value is true)
+}
+```
+
 ### What if my project includes multiple sub-project? How can I use this plugin for each of them including the root project?
 
 Try put 'apply plugin: "dependency-check"' inside the 'allprojects' or 'subprojects' if you'd like to check all sub-projects only, see below:
@@ -127,7 +142,7 @@ buildscript {
     mavenCentral()
   }
   dependencies {
-    classpath "gradle.plugin.com.tools.security:dependency-check:0.0.5"
+    classpath "gradle.plugin.com.tools.security:dependency-check:0.0.6"
   }
 }
 
@@ -144,7 +159,7 @@ buildscript {
     mavenCentral()
   }
   dependencies {
-    classpath "gradle.plugin.com.tools.security:dependency-check:0.0.5"
+    classpath "gradle.plugin.com.tools.security:dependency-check:0.0.6"
   }
 }
 

dependency-check-gradle/build.gradle

@@ -46,23 +46,15 @@ dependencies {
     compile(
             localGroovy(),
             gradleApi(),
-            'org.owasp:dependency-check-core:1.2.11',
+            'org.owasp:dependency-check-core:1.3.0',
-            'org.owasp:dependency-check-utils:1.2.11'
+            'org.owasp:dependency-check-utils:1.3.0'
     )
 
-    testCompile ('com.netflix.nebula:nebula-test:2.2.+'){
+    testCompile ('com.netflix.nebula:nebula-test:2.2.2'){
         exclude group: 'org.codehaus.groovy'
     }
 }
 
-group = 'com.thoughtworks.tools'
-version = '0.0.5'
-
-apply from: 'conf/publish/local.gradle'
-//apply from: 'conf/publish/maven.gradle'
-apply from: 'conf/publish/gradlePluginsPortal.gradle'
-//apply from: 'conf/publish/bintray.gradle'        // according to the documentation of plugindev, this line has to be placed and the very end of the build file
-
 sourceSets {
     integTest {
         groovy.srcDir file('src/integTest/groovy')
@@ -79,3 +71,11 @@ task integTest(type: Test) {
     reports.html.destination = file("$buildDir/reports/integ")
     jvmArgs '-XX:MaxPermSize=256m'
 }
+
+group = 'com.thoughtworks.tools'
+version = '0.0.6'
+
+apply from: 'conf/publish/local.gradle'
+//apply from: 'conf/publish/maven.gradle'
+apply from: 'conf/publish/gradlePluginsPortal.gradle'
+//apply from: 'conf/publish/bintray.gradle'        // according to the documentation of plugindev, this line has to be placed and the very end of the build file

dependency-check-gradle/conf/publish/maven.gradle

@@ -66,6 +66,11 @@ task javadocJar(type: Jar) {
     from javadoc
 }
 
+task sourcesJar(type: Jar, dependsOn: classes) {
+    classifier = 'sources'
+    from sourceSets.main.allSource
+}
+
 artifacts {
     archives javadocJar, sourcesJar
 }

dependency-check-gradle/src/main/groovy/com/tools/security/extension/DependencyCheckConfigurationExtension.groovy

@@ -31,4 +31,6 @@ class DependencyCheckConfigurationExtension {
     String cveUrl20Base = "https://nvd.nist.gov/feeds/xml/cve/nvdcve-2.0-%d.xml.gz"
 
     String outputDirectory = "./reports"
+
+    Boolean quickQueryTimestamp = true;
 }

dependency-check-gradle/src/main/groovy/com/tools/security/plugin/DependencyCheckGradlePlugin.groovy

@@ -49,6 +49,7 @@ class DependencyCheckGradlePlugin implements Plugin<Project> {
             conventionMapping.cveUrl12Base = { extension.cveUrl12Base }
             conventionMapping.cveUrl20Base = { extension.cveUrl20Base }
             conventionMapping.outputDirectory = { extension.outputDirectory }
+            conventionMapping.quickQueryTimestamp = { extension.quickQueryTimestamp }
         }
     }
 }

dependency-check-gradle/src/main/groovy/com/tools/security/tasks/DependencyCheckTask.groovy

@@ -28,6 +28,7 @@ import org.owasp.dependencycheck.dependency.Dependency
 import org.owasp.dependencycheck.reporting.ReportGenerator
 import org.owasp.dependencycheck.utils.Settings
 
+import static org.owasp.dependencycheck.utils.Settings.setBoolean
 import static org.owasp.dependencycheck.utils.Settings.setString
 
 class DependencyCheckTask extends DefaultTask {
@@ -47,6 +48,8 @@ class DependencyCheckTask extends DefaultTask {
 
     String outputDirectory = "./reports"
 
+    Boolean quickQueryTimestamp = true;
+
     DependencyCheckTask() {
         group = 'Dependency Check'
         description = 'Produce dependency security report.'
@@ -73,6 +76,7 @@ class DependencyCheckTask extends DefaultTask {
         Settings.initialize()
         overrideProxySetting()
         overrideCveUrlSetting()
+        overrideDownloaderSetting()
     }
 
     def cleanup(engine) {
@@ -140,4 +144,8 @@ class DependencyCheckTask extends DefaultTask {
         setString(Settings.KEYS.CVE_SCHEMA_2_0, getCveUrl20Base())
         setString(Settings.KEYS.CVE_SCHEMA_1_2, getCveUrl12Base())
     }
+
+    def overrideDownloaderSetting() {
+        setBoolean(Settings.KEYS.DOWNLOADER_QUICK_QUERY_TIMESTAMP, getQuickQueryTimestamp())
+    }
 }

dependency-check-gradle/src/main/resources/META-INF/gradle-plugins/dependency.check.properties

@@ -0,0 +1,19 @@
+#
+# This file is part of dependency-check-gradle.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Copyright (c) 2015 Wei Ma. All Rights Reserved.
+#
+
+implementation-class=com.tools.security.plugin.DependencyCheckGradlePlugin

dependency-check-gradle/src/test/groovy/com/tools/security/plugin/DependencyCheckGradlePluginSpec.groovy

@@ -58,6 +58,7 @@ class DependencyCheckGradlePluginSpec extends PluginProjectSpec {
         task.cveUrl12Base == 'https://nvd.nist.gov/download/nvdcve-%d.xml.gz'
         task.cveUrl20Base == 'https://nvd.nist.gov/feeds/xml/cve/nvdcve-2.0-%d.xml.gz'
         task.outputDirectory == './reports'
+        task.quickQueryTimestamp == true
     }
 
     def 'tasks use correct values when extension is used'() {
@@ -73,6 +74,7 @@ class DependencyCheckGradlePluginSpec extends PluginProjectSpec {
             cveUrl12Base = 'cveUrl12Base'
             cveUrl20Base = 'cveUrl20Base'
             outputDirectory = 'outputDirectory'
+            quickQueryTimestamp = false
         }
 
         then:
@@ -87,5 +89,6 @@ class DependencyCheckGradlePluginSpec extends PluginProjectSpec {
         task.cveUrl12Base == 'cveUrl12Base'
         task.cveUrl20Base == 'cveUrl20Base'
         task.outputDirectory == 'outputDirectory'
+        task.quickQueryTimestamp == false
     }
 }

src/site/markdown/data/cachenvd.md

@@ -0,0 +1,93 @@
+Snapshotting the NVD
+====================
+
+The [Mirroring the NVD from NIST](./mirrornvd.html) topic describes briefly
+how to use the [Nist-Data-Mirror](https://github.com/stevespringett/nist-data-mirror/)
+project to cache the NVD locally and run Dependency Check (D-C) against the
+local cache.
+
+This topic goes into a bit more depth with the [cli](../dependency-check-cli/index.html)
+client, focusing on the following use case.
+
+1. You wish to have daily local snapshots of the NVD, so that
+2. in order to compare later runs of D-C with earlier runs, you can compare
+   "apples with apples".
+
+In other words: It is sometimes desirable to run a comparison D-C analysis
+against the same NVD snapshot that an earlier D-C report used.
+
+In the steps below, concrete examples will be given assuming an Ubuntu Linux
+system. Hopefully, enough explanation is provided that the steps can easily be
+translated to other systems.
+
+Build Nist-Data-Mirror
+----------------------
+
+1. Perform a "git clone" of [Nist-Data-Mirror](https://github.com/stevespringett/nist-data-mirror/)
+2. Install gradle, if necessary. See [here](http://gradle.org/gradle-download/)
+   or your Linux distributions package management system. (e.g.,
+   `sudo apt-get install gradle`).
+3. Follow the [build instructions](https://github.com/stevespringett/nist-data-mirror/blob/master/README.md#user-content-build).
+   You will be left with a build artifact called `nist-data-mirror-1.0.0.jar`.
+
+Set Up a Daily NVD Download Job
+-------------------------------
+
+On Linux, the way to do this using the [cron daemon](http://linux.die.net/man/8/cron).
+"Cron jobs" are configured by invoking [crontab](http://linux.die.net/man/5/crontab).
+For example, invoke `crontab -e` to add a line like the following to your
+crontab file:
+
+    4 5 * * * ~/.local/bin/nvd_download.sh ~/NVD ~/.local/jars
+
+This would run a job on your system at 4:05 AM daily to run the
+[nvd_download.sh](general/nvd_download.sh) shell script with the two given
+arguments. The script is simple:
+
+```sh
+#!/bin/sh
+NVD_ROOT=$1/`date -I`
+JAR_PATH=$2/nist-data-mirror-1.0.0.jar
+java -jar $JAR_PATH $NVD_ROOT
+rm $NVD_ROOT/*.xml # D-C works directly with .gz files anyway.
+```
+
+Nist-Data-Mirror will automatically create the directory, download the
+.xml.gz files, and extract the .xml files alongside them. Given the parameters
+in the cron example above, the new directory will be `~/NVD/2015-08-03` if
+executed on August 3<sup>rd</sup>, 2015. The download for 2015-08-03 pulled 47
+MiB, and took up a total of 668 MiB after extracting from the compressed
+archive format. It turns out that D-C works directly with the .xml.gz files,
+so the above script preserves disk space by deleting the .xml files.
+
+Invoke the Command-Line Using a Specific Daily Snapshot
+-------------------------------------------------------
+
+An example script named [dep-check-date.sh](general/dep-check-date.sh) is
+shown below, which facilitates a D-C scan against an arbitrary NVD snapshot:
+
+```sh
+#!/bin/sh
+CLI_LOCATION=~/.local/dependency-check-1.2.11
+CLI_SCRIPT=$CLI_LOCATION/bin/dependency-check.sh
+NVD_PATH=$1/`date -I -d $2`
+NVD=file://$NVD_PATH
+shift 2 # We've used the first two params. The rest go to CLI_SCRIPT.
+$CLI_SCRIPT --cveUrl20Base $NVD/nvdcve-2.0-%d.xml.gz \
+    --cveUrl12Base $NVD/nvdcve-%d.xml.gz \
+    --cveUrl20Modified $NVD/nvdcve-2.0-Modified.xml.gz \
+    --cveUrl12Modified $NVD/nvdcve-Modified.xml.gz \
+    --data $NVD_PATH $@
+```
+
+The script takes advantage of the `date` command's ability to parse a variety
+of date formats. The following invokation would successfully point to the
+`~/NVD/2015-08-03` folder.
+
+    $ ./dep-check-date.sh ~/NVD "08/03/2015" -app Foo -scan /path/to/Foo --out ~/DCreports/FooFollowup/
+
+If today happened to be August 4th, 2015, `"yesterday"` also would have
+worked. Also notice the usage of the `--data` parameter. This places the H2
+database file directly in the folder alongside the .xml.gz files. This is
+critical, so that D-C doesn't run against another version of the database,
+like the usual default in `$CLI_LOCATION/data`.

src/site/resources/general/dep-check-date.sh

@@ -0,0 +1,11 @@
+#!/bin/sh
+CLI_LOCATION=~/.local/dependency-check-1.2.11
+CLI_SCRIPT=$CLI_LOCATION/bin/dependency-check.sh
+NVD_PATH=$1/`date -I -d $2`
+NVD=file://$NVD_PATH
+shift 2 # We've used the first two params. The rest go to CLI_SCRIPT.
+$CLI_SCRIPT --cveUrl20Base $NVD/nvdcve-2.0-%d.xml.gz \
+    --cveUrl12Base $NVD/nvdcve-%d.xml.gz \
+    --cveUrl20Modified $NVD/nvdcve-2.0-Modified.xml.gz \
+    --cveUrl12Modified $NVD/nvdcve-Modified.xml.gz \
+    --data $NVD_PATH $@

src/site/resources/general/nvd_download.sh

@@ -0,0 +1,5 @@
+#!/bin/sh
+NVD_ROOT=$1/`date -I`
+JAR_PATH=$2/nist-data-mirror-1.0.0.jar
+java -jar $JAR_PATH $NVD_ROOT
+rm $NVD_ROOT/*.xml # D-C works directly with .gz files anyway.

src/site/site.xml

@@ -87,6 +87,7 @@ Copyright (c) 2013 Jeremy Long. All Rights Reserved.
                 <item collapse="true" name="Internet Access Required" href="./data/index.html">
                     <item name="Proxy" href="./data/proxy.html" />
                     <item name="Mirroring NVD" href="./data/mirrornvd.html" />
+                    <item name="Snapshotting the NVD" href="./data/cachenvd.html" />
                     <item name="Central DB" href="./data/database.html" />
                 </item>
                 <item name="Related Work" href="./related.html">