From 3a70e25983632306ff192c7cf7b6bbc49a53a7b5 Mon Sep 17 00:00:00 2001
From: Stefan Neuhaus <dr.stefan.neuhaus@gmail.com>
Date: Thu, 16 Feb 2017 07:48:06 +0100
Subject: [PATCH] Refactoring: Move retrieval of last modified timestamps from
 UpdateableNvdCve to NvdCveUpdater - UpdateableNvdCve is from its nature more
 like a simple value object - Facilitates performance optimization for
 retrieval of last modification timestamps

---
 .../data/update/NvdCveUpdater.java            | 55 ++++++++++++++++---
 .../data/update/nvd/UpdateableNvdCve.java     | 12 +---
 .../data/update/nvd/UpdateableNvdCveTest.java | 36 +++++-------
 3 files changed, 66 insertions(+), 37 deletions(-)

diff --git a/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/NvdCveUpdater.java b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/NvdCveUpdater.java
index 93cdbcf1e..c3263bc1e 100644
--- a/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/NvdCveUpdater.java
+++ b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/NvdCveUpdater.java
@@ -19,8 +19,11 @@ package org.owasp.dependencycheck.data.update;
 
 import java.net.MalformedURLException;
 import java.util.Calendar;
+import java.util.HashMap;
 import java.util.HashSet;
+import java.util.Map;
 import java.util.Set;
+import java.net.URL;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
@@ -36,6 +39,7 @@ import org.owasp.dependencycheck.data.update.nvd.NvdCveInfo;
 import org.owasp.dependencycheck.data.update.nvd.ProcessTask;
 import org.owasp.dependencycheck.data.update.nvd.UpdateableNvdCve;
 import org.owasp.dependencycheck.utils.DateUtil;
+import org.owasp.dependencycheck.utils.Downloader;
 import org.owasp.dependencycheck.utils.DownloadFailedException;
 import org.owasp.dependencycheck.utils.InvalidSettingException;
 import org.owasp.dependencycheck.utils.Settings;
@@ -357,20 +361,57 @@ public class NvdCveUpdater extends BaseUpdater implements CachedWebDataSource {
     private UpdateableNvdCve retrieveCurrentTimestampsFromWeb()
             throws MalformedURLException, DownloadFailedException, InvalidDataException, InvalidSettingException {
 
-        final UpdateableNvdCve updates = new UpdateableNvdCve();
-        updates.add(MODIFIED, Settings.getString(Settings.KEYS.CVE_MODIFIED_20_URL),
-                Settings.getString(Settings.KEYS.CVE_MODIFIED_12_URL),
-                false);
 
         final int start = Settings.getInt(Settings.KEYS.CVE_START_YEAR);
         final int end = Calendar.getInstance().get(Calendar.YEAR);
+
+        final Map<String, Long> lastModifiedDates = retrieveLastModifiedDates(start, end);
+
+        final UpdateableNvdCve updates = new UpdateableNvdCve();
+
         final String baseUrl20 = Settings.getString(Settings.KEYS.CVE_SCHEMA_2_0);
         final String baseUrl12 = Settings.getString(Settings.KEYS.CVE_SCHEMA_1_2);
         for (int i = start; i <= end; i++) {
-            updates.add(Integer.toString(i), String.format(baseUrl20, i),
-                    String.format(baseUrl12, i),
-                    true);
+            final String url = String.format(baseUrl20, i);
+            updates.add(Integer.toString(i), url, String.format(baseUrl12, i),
+                    lastModifiedDates.get(url), true);
         }
+
+        final String url = Settings.getString(Settings.KEYS.CVE_MODIFIED_20_URL);
+        updates.add(MODIFIED, url, Settings.getString(Settings.KEYS.CVE_MODIFIED_12_URL),
+                lastModifiedDates.get(url), false);
+
         return updates;
     }
+
+    /**
+     * Retrieves the timestamps from the NVD CVE meta data file.
+     *
+     * @param startYear the first year whose item to check for the timestamp
+     * @param endYear the last year whose item to check for the timestamp
+     * @return the timestamps from the currently published nvdcve downloads page
+     * @throws MalformedURLException thrown if the URL for the NVD CCE Meta data
+     * is incorrect.
+     * @throws DownloadFailedException thrown if there is an error downloading
+     * the nvd cve meta data file
+     */
+    private Map<String, Long> retrieveLastModifiedDates(int startYear, int endYear)
+            throws MalformedURLException, DownloadFailedException {
+
+        final Set<String> urls = new HashSet<String>();
+        final String baseUrl20 = Settings.getString(Settings.KEYS.CVE_SCHEMA_2_0);
+        for (int i = startYear; i <= endYear; i++) {
+            final String url = String.format(baseUrl20, i);
+            urls.add(url);
+        }
+        urls.add(Settings.getString(Settings.KEYS.CVE_MODIFIED_20_URL));
+
+        final Map<String, Long> lastModifiedDates = new HashMap<String, Long>();
+        for(String url: urls) {
+            LOGGER.debug("Checking for updates from: {}", url);
+            lastModifiedDates.put(url, Downloader.getLastModified(new URL(url)));
+        }
+
+        return lastModifiedDates;
+    }
 }
diff --git a/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/nvd/UpdateableNvdCve.java b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/nvd/UpdateableNvdCve.java
index 08327eeac..4287bba4d 100644
--- a/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/nvd/UpdateableNvdCve.java
+++ b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/nvd/UpdateableNvdCve.java
@@ -17,14 +17,10 @@
  */
 package org.owasp.dependencycheck.data.update.nvd;
 
-import java.net.MalformedURLException;
-import java.net.URL;
 import java.util.Iterator;
 import java.util.Map;
 import java.util.Map.Entry;
 import java.util.TreeMap;
-import org.owasp.dependencycheck.utils.DownloadFailedException;
-import org.owasp.dependencycheck.utils.Downloader;
 
 /**
  * Contains a collection of updateable NvdCveInfo objects. This is used to determine which files need to be downloaded and
@@ -68,18 +64,16 @@ public class UpdateableNvdCve implements Iterable<NvdCveInfo>, Iterator<NvdCveIn
      * @param id the key for the item to be added
      * @param url the URL to download the item
      * @param oldUrl the URL for the old version of the item (the NVD CVE old schema still contains useful data we need).
+     * @param timestamp the last modified date of the downloaded item
      * @param needsUpdate whether or not the data needs to be updated
-     * @throws MalformedURLException thrown if the URL provided is invalid
-     * @throws DownloadFailedException thrown if the download fails.
      */
-    public void add(String id, String url, String oldUrl, boolean needsUpdate) throws MalformedURLException, DownloadFailedException {
+    public void add(String id, String url, String oldUrl, long timestamp, boolean needsUpdate) {
         final NvdCveInfo item = new NvdCveInfo();
         item.setNeedsUpdate(needsUpdate); //the others default to true, to make life easier later this should default to false.
         item.setId(id);
         item.setUrl(url);
         item.setOldSchemaVersionUrl(oldUrl);
-        LOGGER.debug("Checking for updates from: {}", url);
-        item.setTimestamp(Downloader.getLastModified(new URL(url)));
+        item.setTimestamp(timestamp);
         collection.put(id, item);
     }
 
diff --git a/dependency-check-core/src/test/java/org/owasp/dependencycheck/data/update/nvd/UpdateableNvdCveTest.java b/dependency-check-core/src/test/java/org/owasp/dependencycheck/data/update/nvd/UpdateableNvdCveTest.java
index f734108b1..fdb8bd989 100644
--- a/dependency-check-core/src/test/java/org/owasp/dependencycheck/data/update/nvd/UpdateableNvdCveTest.java
+++ b/dependency-check-core/src/test/java/org/owasp/dependencycheck/data/update/nvd/UpdateableNvdCveTest.java
@@ -17,17 +17,12 @@
  */
 package org.owasp.dependencycheck.data.update.nvd;
 
-import org.owasp.dependencycheck.data.update.nvd.UpdateableNvdCve;
-import org.owasp.dependencycheck.data.update.nvd.NvdCveInfo;
 import java.io.File;
-import java.io.IOException;
-import java.net.MalformedURLException;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
 import org.junit.Test;
 import org.owasp.dependencycheck.BaseTest;
-import org.owasp.dependencycheck.utils.DownloadFailedException;
 
 /**
  *
@@ -39,18 +34,18 @@ public class UpdateableNvdCveTest extends BaseTest {
      * Test of isUpdateNeeded method, of class UpdateableNvdCve.
      */
     @Test
-    public void testIsUpdateNeeded() throws MalformedURLException, DownloadFailedException, IOException {
+    public void testIsUpdateNeeded() {
         String id = "key";
-        //use a local file as this test will load the result and check the timestamp
         String url = new File("target/test-classes/nvdcve-2.0-2012.xml").toURI().toString();
+        long timestamp = 42;
         UpdateableNvdCve instance = new UpdateableNvdCve();
-        instance.add(id, url, url, false);
+        instance.add(id, url, url, timestamp, false);
 
         boolean expResult = false;
         boolean result = instance.isUpdateNeeded();
         assertEquals(expResult, result);
 
-        instance.add("nextId", url, url, true);
+        instance.add("nextId", url, url, 23, true);
 
         expResult = true;
         result = instance.isUpdateNeeded();
@@ -63,34 +58,34 @@ public class UpdateableNvdCveTest extends BaseTest {
     @Test
     public void testAdd() throws Exception {
         String id = "key";
-        //use a local file as this test will load the result and check the timestamp
         String url = new File("target/test-classes/nvdcve-2.0-2012.xml").toURI().toString();
+        long timestamp = 42;
         UpdateableNvdCve instance = new UpdateableNvdCve();
-        instance.add(id, url, url, false);
+        instance.add(id, url, url, timestamp, false);
 
         boolean expResult = false;
         boolean result = instance.isUpdateNeeded();
         assertEquals(expResult, result);
 
-        instance.add("nextId", url, url, false);
+        instance.add("nextId", url, url, 23, false);
         NvdCveInfo results = instance.get(id);
 
         assertEquals(id, results.getId());
         assertEquals(url, results.getUrl());
         assertEquals(url, results.getOldSchemaVersionUrl());
-
+        assertEquals(timestamp, results.getTimestamp());
     }
 
     /**
      * Test of clear method, of class UpdateableNvdCve.
      */
     @Test
-    public void testClear() throws MalformedURLException, DownloadFailedException, IOException {
+    public void testClear() {
         String id = "key";
-        //use a local file as this test will load the result and check the timestamp
         String url = new File("target/test-classes/nvdcve-2.0-2012.xml").toURI().toString();
+        long timestamp = 42;
         UpdateableNvdCve instance = new UpdateableNvdCve();
-        instance.add(id, url, url, false);
+        instance.add(id, url, url, timestamp, false);
         assertFalse(instance.getCollection().isEmpty());
         instance.clear();
         assertTrue(instance.getCollection().isEmpty());
@@ -100,13 +95,12 @@ public class UpdateableNvdCveTest extends BaseTest {
      * Test of iterator method, of class UpdatableNvdCve.
      */
     @Test
-    public void testIterator() throws IOException {
-        //use a local file as this test will load the result and check the timestamp
+    public void testIterator() {
         String url = new File("target/test-classes/nvdcve-2.0-2012.xml").toURI().toString();
         UpdateableNvdCve instance = new UpdateableNvdCve();
-        instance.add("one", url, url, false);
-        instance.add("two", url, url, false);
-        instance.add("three", url, url, false);
+        instance.add("one", url, url, 42, false);
+        instance.add("two", url, url, 23, false);
+        instance.add("three", url, url, 17, false);
         int itemsProcessed = 0;
         for (NvdCveInfo item : instance) {
             if ("one".equals(item.getId())) {