Merge branch 'awhitford-NvdCheck'

dependency-check-ant/src/main/java/org/owasp/dependencycheck/taskdefs/Check.java

@@ -245,14 +245,14 @@ public class Check extends Update {
      * Sets whether auto-updating of the NVD CVE/CPE data is enabled. It is not recommended that this be turned to false. Default
      * is true.
      */
-    private boolean autoUpdate = true;
+    private Boolean autoUpdate;
 
     /**
      * Get the value of autoUpdate.
      *
      * @return the value of autoUpdate
      */
-    public boolean isAutoUpdate() {
+    public Boolean isAutoUpdate() {
         return autoUpdate;
     }
 
@@ -261,7 +261,7 @@ public class Check extends Update {
      *
      * @param autoUpdate new value of autoUpdate
      */
-    public void setAutoUpdate(boolean autoUpdate) {
+    public void setAutoUpdate(Boolean autoUpdate) {
         this.autoUpdate = autoUpdate;
     }
     /**
@@ -839,12 +839,15 @@ public class Check extends Update {
     /**
      * Takes the properties supplied and updates the dependency-check settings. Additionally, this sets the system properties
      * required to change the proxy server, port, and connection timeout.
+     *
+     * @throws BuildException thrown when an invalid setting is configured.
      */
     @Override
-    protected void populateSettings() {
+    protected void populateSettings() throws BuildException {
         super.populateSettings();
-        Settings.setBoolean(Settings.KEYS.AUTO_UPDATE, autoUpdate);
+        if (autoUpdate != null) {
-
+            Settings.setBoolean(Settings.KEYS.AUTO_UPDATE, autoUpdate);
+        }
         if (suppressionFile != null && !suppressionFile.isEmpty()) {
             Settings.setString(Settings.KEYS.SUPPRESSION_FILE, suppressionFile);
         }

dependency-check-ant/src/main/java/org/owasp/dependencycheck/taskdefs/Update.java

@@ -357,6 +357,26 @@ public class Update extends Purge {
         this.cveUrl20Base = cveUrl20Base;
     }
 
+    private Integer cveValidForHours;
+
+    /**
+     * Get the value of cveValidForHours
+     *
+     * @return the value of cveValidForHours
+     */
+    public Integer getCveValidForHours() {
+        return cveValidForHours;
+    }
+
+    /**
+     * Set the value of cveValidForHours
+     *
+     * @param cveValidForHours new value of cveValidForHours
+     */
+    public void setCveValidForHours(Integer cveValidForHours) {
+        this.cveValidForHours = cveValidForHours;
+    }
+
     /**
      * Executes the update by initializing the settings, downloads the NVD XML data, and then processes the data storing it in the
      * local database.
@@ -383,9 +403,11 @@ public class Update extends Purge {
     /**
      * Takes the properties supplied and updates the dependency-check settings. Additionally, this sets the system properties
      * required to change the proxy server, port, and connection timeout.
+     *
+     * @throws BuildException thrown when an invalid setting is configured.
      */
     @Override
-    protected void populateSettings() {
+    protected void populateSettings() throws BuildException {
         super.populateSettings();
         if (proxyServer != null && !proxyServer.isEmpty()) {
             Settings.setString(Settings.KEYS.PROXY_SERVER, proxyServer);
@@ -429,5 +451,12 @@ public class Update extends Purge {
         if (cveUrl20Base != null && !cveUrl20Base.isEmpty()) {
             Settings.setString(Settings.KEYS.CVE_SCHEMA_2_0, cveUrl20Base);
         }
+        if (cveValidForHours != null) {
+            if (cveValidForHours >= 0) {
+                Settings.setInt(Settings.KEYS.CVE_CHECK_VALID_FOR_HOURS, cveValidForHours);
+            } else {
+                throw new BuildException("Invalid setting: `cpeValidForHours` must be 0 or greater");
+            }
+        }
     }
 }

dependency-check-ant/src/site/markdown/configuration.md

@@ -29,19 +29,20 @@ Configuration: dependency-check Task
 --------------------
 The following properties can be set on the dependency-check-update task.
 
-Property              | Description                        | Default Value
+Property              | Description                                                                                                                                                                                        | Default Value
-----------------------|------------------------------------|------------------
+----------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-----------------
-autoUpdate            | Sets whether auto-updating of the NVD CVE/CPE data is enabled. It is not recommended that this be turned to false. | true
+autoUpdate            | Sets whether auto-updating of the NVD CVE/CPE data is enabled. It is not recommended that this be turned to false.                                                                                 | true
-projectName           | The name of the project being scanned. | Dependency-Check
+cveValidForHours      | Sets the number of hours to wait before checking for new updates from the NVD                                                                                                                      | 4
-reportOutputDirectory | The location to write the report(s). Note, this is not used if generating the report as part of a `mvn site` build | 'target'
+failBuildOnCVSS       | Specifies if the build should be failed if a CVSS score above a specified level is identified. The default is 11 which means since the CVSS scores are 0-10, by default the build will never fail. | 11
-failBuildOnCVSS       | Specifies if the build should be failed if a CVSS score above a specified level is identified. The default is 11 which means since the CVSS scores are 0-10, by default the build will never fail.         | 11
+projectName           | The name of the project being scanned.                                                                                                                                                             | Dependency-Check
-reportFormat          | The report format to be generated (HTML, XML, VULN, ALL). This configuration option has no affect if using this within the Site plugin unless the externalReport is set to true. | HTML
+reportFormat          | The report format to be generated (HTML, XML, VULN, ALL). This configuration option has no affect if using this within the Site plugin unless the externalReport is set to true.                   | HTML
-suppressionFile       | The file path to the XML suppression file \- used to suppress [false positives](../general/suppression.html) |  
+reportOutputDirectory | The location to write the report(s). Note, this is not used if generating the report as part of a `mvn site` build                                                                                 | 'target'
-proxyServer           | The Proxy Server.                  |  
+suppressionFile       | The file path to the XML suppression file \- used to suppress [false positives](../general/suppression.html)                                                                                       |  
-proxyPort             | The Proxy Port.                    |  
+proxyServer           | The Proxy Server.                                                                                                                                                                                  |  
-proxyUsername         | Defines the proxy user name.       |  
+proxyPort             | The Proxy Port.                                                                                                                                                                                    |  
-proxyPassword         | Defines the proxy password.        |  
+proxyUsername         | Defines the proxy user name.                                                                                                                                                                       |  
-connectionTimeout     | The URL Connection Timeout.        |  
+proxyPassword         | Defines the proxy password.                                                                                                                                                                        |  
+connectionTimeout     | The URL Connection Timeout.                                                                                                                                                                        |  
 
 Analyzer Configuration
 ====================

dependency-check-cli/src/main/java/org/owasp/dependencycheck/App.java

@@ -279,6 +279,7 @@ public class App {
         final String cveMod20 = cli.getModifiedCve20Url();
         final String cveBase12 = cli.getBaseCve12Url();
         final String cveBase20 = cli.getBaseCve20Url();
+        final Integer cveValidForHours = cli.getCveValidForHours();
 
         if (propertiesFile != null) {
             try {
@@ -326,6 +327,9 @@ public class App {
         if (suppressionFile != null && !suppressionFile.isEmpty()) {
             Settings.setString(Settings.KEYS.SUPPRESSION_FILE, suppressionFile);
         }
+        if (cveValidForHours != null) {
+            Settings.setInt(Settings.KEYS.CVE_CHECK_VALID_FOR_HOURS, cveValidForHours);
+        }
 
         //File Type Analyzer Settings
         Settings.setBoolean(Settings.KEYS.ANALYZER_JAR_ENABLED, !cli.isJarDisabled());

dependency-check-cli/src/main/java/org/owasp/dependencycheck/CliParser.java

@@ -90,6 +90,19 @@ public final class CliParser {
      * @throws ParseException is thrown if there is an exception parsing the command line.
      */
     private void validateArgs() throws FileNotFoundException, ParseException {
+        if (isUpdateOnly() || isRunScan()) {
+            String value = line.getOptionValue(ARGUMENT.CVE_VALID_FOR_HOURS);
+            if (value != null) {
+                try {
+                    int i = Integer.parseInt(value);
+                    if (i < 0) {
+                        throw new ParseException("Invalid Setting: cveValidForHours must be a number greater than or equal to 0.");
+                    }
+                } catch (NumberFormatException ex) {
+                    throw new ParseException("Invalid Setting: cveValidForHours must be a number greater than or equal to 0.");
+                }
+            }
+        }
         if (isRunScan()) {
             validatePathExists(getScanFiles(), ARGUMENT.SCAN);
             validatePathExists(getReportDirectory(), ARGUMENT.OUT);
@@ -255,6 +268,10 @@ public final class CliParser {
                 .desc("The file path to the suppression XML file.")
                 .build();
 
+        final Option cveValidForHours = Option.builder().argName("hours").hasArg().longOpt(ARGUMENT.CVE_VALID_FOR_HOURS)
+                .desc("The number of hours to wait before checking for new updates from the NVD.")
+                .build();
+
         //This is an option group because it can be specified more then once.
         final OptionGroup og = new OptionGroup();
         og.addOption(path);
@@ -274,7 +291,8 @@ public final class CliParser {
                 .addOption(symLinkDepth)
                 .addOption(props)
                 .addOption(verboseLog)
-                .addOption(suppressionFile);
+                .addOption(suppressionFile)
+                .addOption(cveValidForHours);
     }
 
     /**
@@ -970,6 +988,15 @@ public final class CliParser {
         return line.getOptionValue(ARGUMENT.ADDITIONAL_ZIP_EXTENSIONS);
     }
 
+    /**
+     * Get the value of cveValidForHours
+     *
+     * @return the value of cveValidForHours
+     */
+    public Integer getCveValidForHours() {
+        return Integer.parseInt(line.getOptionValue(ARGUMENT.CVE_VALID_FOR_HOURS));
+    }
+
     /**
      * A collection of static final strings that represent the possible command line arguments.
      */
@@ -1133,6 +1160,10 @@ public final class CliParser {
          * The CLI argument name for setting the location of the suppression file.
          */
         public static final String SUPPRESSION_FILE = "suppression";
+        /**
+         * The CLI argument name for setting the location of the suppression file.
+         */
+        public static final String CVE_VALID_FOR_HOURS = "cveValidForHours";
         /**
          * Disables the Jar Analyzer.
          */

dependency-check-cli/src/site/markdown/arguments.md

@@ -17,6 +17,8 @@ Short  | Argument Name   | Parameter       | Description | Requir
  \-h   | \-\-help              |                 | Print the help message. | Optional
        | \-\-advancedHelp      |                 | Print the advanced help message. | Optional
  \-v   | \-\-version           |                 | Print the version information. | Optional
+       | \-\-cveValidForHours  | \<hours\>       | The number of hours to wait before checking for new updates from the NVD. The default is 4 hours. | Optional
+
 
 Advanced Options
 ================

dependency-check-core/src/main/java/org/owasp/dependencycheck/data/nvdcve/DatabaseProperties.java

@@ -45,6 +45,10 @@ public class DatabaseProperties {
      * updates)..
      */
     public static final String MODIFIED = "Modified";
+    /**
+     * The properties file key for the last checked field - used to store the last check time of the Modified NVD CVE xml file.
+     */
+    public static final String LAST_CHECKED = "NVD CVE Checked";
     /**
      * The properties file key for the last updated field - used to store the last updated time of the Modified NVD CVE xml file.
      */

dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/NvdCveUpdater.java

@@ -66,9 +66,11 @@ public class NvdCveUpdater extends BaseUpdater implements CachedWebDataSource {
     public void update() throws UpdateException {
         try {
             openDataStores();
-            final UpdateableNvdCve updateable = getUpdatesNeeded();
+            if (checkUpdate()) {
-            if (updateable.isUpdateNeeded()) {
+                final UpdateableNvdCve updateable = getUpdatesNeeded();
-                performUpdate(updateable);
+                if (updateable.isUpdateNeeded()) {
+                    performUpdate(updateable);
+                }
             }
         } catch (MalformedURLException ex) {
             LOGGER.warn(
@@ -87,6 +89,35 @@ public class NvdCveUpdater extends BaseUpdater implements CachedWebDataSource {
         }
     }
 
+    /**
+     * Checks if the NVD CVE XML files were last checked recently.
+     * As an optimization, we can avoid repetitive checks against the NVD.
+     * Setting CVE_CHECK_VALID_FOR_HOURS determines the duration since last check before checking again.
+     * A database property stores the timestamp of the last check. 
+     *
+     * @return true to proceed with the check, or false to skip.
+     */
+    private boolean checkUpdate () throws UpdateException {
+        boolean proceed = true;
+        // If the valid setting has not been specified, then we proceed to check...
+        final int validForHours = Settings.getInt(Settings.KEYS.CVE_CHECK_VALID_FOR_HOURS, 0);
+        if (0 < validForHours) {
+            // ms Valid = valid (hours) x 60 min/hour x 60 sec/min x 1000 ms/sec
+            final long msValid = validForHours * 60L * 60L * 1000L;
+            final long lastChecked = Long.parseLong(getProperties().getProperty(DatabaseProperties.LAST_CHECKED, "0"));
+            final long now = System.currentTimeMillis();
+            proceed = (now - lastChecked) > msValid;
+            if (proceed) {
+                getProperties().save(DatabaseProperties.LAST_CHECKED, Long.toString(now));
+            } else {
+                LOGGER.info("Skipping NVD check since last check was within {} hours.", validForHours);
+                LOGGER.debug("Last NVD was at {}, and now {} is within {} ms.",
+                    lastChecked, now, msValid);
+            }
+        }
+        return proceed;
+    }
+
     /**
      * Downloads the latest NVD CVE XML file from the web and imports it into the current CVE Database.
      *

dependency-check-core/src/main/java/org/owasp/dependencycheck/utils/DateUtil.java

@@ -36,11 +36,12 @@ public final class DateUtil {
      *
      * @param date the date to be checked.
      * @param compareTo the date to compare to.
-     * @param range the range in days to be considered valid.
+     * @param dayRange the range in days to be considered valid.
      * @return whether or not the date is within the range.
      */
-    public static boolean withinDateRange(long date, long compareTo, int range) {
+    public static boolean withinDateRange(long date, long compareTo, int dayRange) {
-        final double differenceInDays = (compareTo - date) / 1000.0 / 60.0 / 60.0 / 24.0;
+        // ms = dayRange x 24 hours/day x 60 min/hour x 60 sec/min x 1000 ms/sec
-        return differenceInDays < range;
+        final long msRange = dayRange * 24L * 60L * 60L * 1000L;
+        return (compareTo - date) < msRange;
     }
 }

dependency-check-core/src/main/resources/dependencycheck.properties

@@ -41,7 +41,8 @@ data.driver_path=
 # to update the other files if we are within this timespan. Per NIST this file
 # holds 8 days of updates, we are using 7 just to be safe.
 cve.url.modified.validfordays=7
-
+# the number of hours to wait before checking if updates are available from the NVD.
+cve.check.validforhours=4
 # the path to the modified nvd cve xml file.
 cve.url-1.2.modified=https://nvd.nist.gov/download/nvdcve-Modified.xml.gz
 #cve.url-1.2.modified=http://nvd.nist.gov/download/nvdcve-modified.xml

dependency-check-maven/src/main/java/org/owasp/dependencycheck/maven/BaseDependencyCheckMojo.java

@@ -106,16 +106,16 @@ public abstract class BaseDependencyCheckMojo extends AbstractMojo implements Ma
      * is true.
      */
     @SuppressWarnings("CanBeFinal")
-    @Parameter(property = "autoupdate", defaultValue = "true", required = true)
+    @Parameter(property = "autoupdate")
-    private boolean autoUpdate = true;
+    private Boolean autoUpdate;
     /**
      * Generate aggregate reports in multi-module projects.
      *
      * @deprecated use the aggregate goal instead
      */
-    @Parameter(property = "aggregate", defaultValue = "false")
+    @Parameter(property = "aggregate")
     @Deprecated
-    private boolean aggregate;
+    private Boolean aggregate;
     /**
      * The report format to be generated (HTML, XML, VULN, ALL). This configuration option has no affect if using this within the
      * Site plug-in unless the externalReport is set to true. Default is HTML.
@@ -295,6 +295,11 @@ public abstract class BaseDependencyCheckMojo extends AbstractMojo implements Ma
      */
     @Parameter(property = "cveUrl20Base", defaultValue = "", required = false)
     private String cveUrl20Base;
+    /**
+     * Optionally skip excessive CVE update checks for a designated duration in hours.
+     */
+    @Parameter(property = "cveValidForHours", defaultValue = "", required = false)
+    private Integer cveValidForHours;
 
     /**
      * The path to mono for .NET Assembly analysis on non-windows systems.
@@ -593,8 +598,9 @@ public abstract class BaseDependencyCheckMojo extends AbstractMojo implements Ma
                 }
             }
         }
-
+        if (autoUpdate != null) {
-        Settings.setBoolean(Settings.KEYS.AUTO_UPDATE, autoUpdate);
+            Settings.setBoolean(Settings.KEYS.AUTO_UPDATE, autoUpdate);
+        }
         if (externalReport != null) {
             getLog().warn("The 'externalReport' option was set; this configuration option has been removed. "
                     + "Please update the dependency-check-maven plugin's configuration");
@@ -688,6 +694,9 @@ public abstract class BaseDependencyCheckMojo extends AbstractMojo implements Ma
         if (cveUrl20Base != null && !cveUrl20Base.isEmpty()) {
             Settings.setString(Settings.KEYS.CVE_SCHEMA_2_0, cveUrl20Base);
         }
+        if (cveValidForHours != null) {
+            Settings.setInt(Settings.KEYS.CVE_CHECK_VALID_FOR_HOURS, cveValidForHours);
+        }
     }
 
     /**

dependency-check-maven/src/site/markdown/configuration.md

@@ -15,15 +15,16 @@ The following properties can be set on the dependency-check-maven plugin.
 Property             | Description                        | Default Value
 ---------------------|------------------------------------|------------------
 autoUpdate           | Sets whether auto-updating of the NVD CVE/CPE data is enabled. It is not recommended that this be turned to false. | true
-outputDirectory      | The location to write the report(s). Note, this is not used if generating the report as part of a `mvn site` build | 'target'
+cveValidForHours     | Sets the number of hours to wait before checking for new updates from the NVD.                                     | 4
 failBuildOnCVSS      | Specifies if the build should be failed if a CVSS score above a specified level is identified. The default is 11 which means since the CVSS scores are 0-10, by default the build will never fail.         | 11
 format               | The report format to be generated (HTML, XML, VULN, ALL). This configuration option has no affect if using this within the Site plugin unless the externalReport is set to true. | HTML
 name                 | The name of the report in the site | dependency-check or dependency-check:aggregate
-suppressionFile      | The file path to the XML suppression file \- used to suppress [false positives](../general/suppression.html) |  
+outputDirectory      | The location to write the report(s). Note, this is not used if generating the report as part of a `mvn site` build | 'target'
 skip                 | Skips the dependency-check analysis                       | false
 skipTestScope        | Should be skip analysis for artifacts with Test Scope     | true
 skipProvidedScope    | Should be skip analysis for artifacts with Provided Scope | false
 skipRuntimeScope     | Should be skip analysis for artifacts with Runtime Scope  | false
+suppressionFile      | The file path to the XML suppression file \- used to suppress [false positives](../general/suppression.html) |  
 
 Analyzer Configuration
 ====================

dependency-check-utils/src/main/java/org/owasp/dependencycheck/utils/Settings.java

@@ -117,6 +117,10 @@ public final class Settings {
          * The properties key for the URL to retrieve the recently modified and added CVE entries (last 8 days).
          */
         public static final String CVE_MODIFIED_VALID_FOR_DAYS = "cve.url.modified.validfordays";
+        /**
+         * The properties key to control the skipping of the check for CVE updates.
+         */
+        public static final String CVE_CHECK_VALID_FOR_HOURS = "cve.check.validforhours";
         /**
          * The properties key for the telling us how many cve.url.* URLs exists. This is used in combination with CVE_BASE_URL to
          * be able to retrieve the URLs for all of the files that make up the NVD CVE listing.
@@ -463,6 +467,17 @@ public final class Settings {
         setString(key, Boolean.toString(value));
     }
 
+    /**
+     * Sets a property value.
+     *
+     * @param key the key for the property
+     * @param value the value for the property
+     */
+    public static void setInt(String key, int value) {
+        localSettings.get().props.setProperty(key, String.valueOf(value));
+        LOGGER.debug("Setting: {}='{}'", key, value);
+    }
+
     /**
      * Merges a new properties file into the current properties. This method allows for the loading of a user provided properties
      * file.<br/><br/>