updated batch update functionality

Former-commit-id: f62347bd25b61f048f18fb8cb23b8de7c053659e

dependency-check-core/src/main/java/org/owasp/dependencycheck/data/nvdcve/CveDB.java

@@ -176,7 +176,7 @@ public class CveDB {
         final File f = new File(fileName, "cve." + DB_SCHEMA_VERSION);
         final File check = new File(f.getAbsolutePath() + ".h2.db");
         final boolean createTables = !check.exists();
-        final String connStr = "jdbc:h2:file:" + f.getAbsolutePath();
+        final String connStr = String.format("jdbc:h2:file:%s;AUTO_SERVER=TRUE", f.getAbsolutePath());
         Class.forName("org.h2.Driver");
         conn = DriverManager.getConnection(connStr, "sa", "");
         if (createTables) {

dependency-check-core/src/main/java/org/owasp/dependencycheck/data/nvdcve/xml/DatabaseUpdater.java

@@ -29,6 +29,7 @@ import java.io.OutputStreamWriter;
 import javax.xml.parsers.ParserConfigurationException;
 import org.owasp.dependencycheck.data.CachedWebDataSource;
 import java.net.MalformedURLException;
+import java.net.URISyntaxException;
 import java.net.URL;
 import java.sql.SQLException;
 import java.util.Calendar;
@@ -114,13 +115,14 @@ public class DatabaseUpdater implements CachedWebDataSource {
                 Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.INFO,
                         "NVD CVE requires several updates; this could take a couple of minutes.");
             }
-            if (maxUpdates > 0) {
+            if (maxUpdates > 0 && !isDoBatchUpdate()) {
                 openDataStores();
             }
 
             if (isBatchUpdateMode() && isDoBatchUpdate()) {
                 try {
                     performBatchUpdate();
+                    openDataStores();
                 } catch (IOException ex) {
                     throw new UpdateException("Unable to perform batch update", ex);
                 }
@@ -498,7 +500,12 @@ public class DatabaseUpdater implements CachedWebDataSource {
                         }
                     }
                 }
+            } else {
+                //properties file does not exist - check about batch update
+                setDoBatchUpdate(isBatchUpdateMode());
             }
+        } else { //this condition will likely never exist - but just in case we need to handle batch updates
+            setDoBatchUpdate(isBatchUpdateMode());
         }
         return currentlyPublished;
     }
@@ -627,7 +634,7 @@ public class DatabaseUpdater implements CachedWebDataSource {
      *
      * @throws IOException thrown if the directory cannot be deleted
      */
-    private void deleteExistingData() throws IOException {
+    protected void deleteExistingData() throws IOException {
         Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.INFO, "The database version is old. Rebuilding the database.");
 
         final File cveDir = CveDB.getDataDirectory();
@@ -637,15 +644,37 @@ public class DatabaseUpdater implements CachedWebDataSource {
         FileUtils.delete(cpeDir);
     }
 
-    private void performBatchUpdate() throws IOException {
+    private void performBatchUpdate() throws UpdateException {
         if (batchUpdateMode && doBatchUpdate) {
-            deleteExistingData();
+            final String batchSrc = Settings.getString(Settings.KEYS.BATCH_UPDATE_URL);
-            String batchSrc = Settings.getString(Settings.KEYS.BATCH_UPDATE_URL);
+            File tmp = null;
-            File dataDirectory = CveDB.getDataDirectory().getParentFile();
+            try {
-            URL batchUrl = new URL(batchSrc);
+                deleteExistingData();
-            File tmp = File.createTempFile("batch_", ".zip");
+                final File dataDirectory = CveDB.getDataDirectory().getParentFile();
-            Downloader.fetchFile(batchUrl, tmp);
+                final URL batchUrl = new URL(batchSrc);
-            FileUtils.extractFiles(tmp, dataDirectory);
+                if ("file".equals(batchUrl.getProtocol())) {
+                    try {
+                        tmp = new File(batchUrl.toURI());
+                    } catch (URISyntaxException ex) {
+                        final String msg = String.format("Invalid batch update URI: %s", batchSrc);
+                        throw new UpdateException(msg, ex);
+                    }
+                } else if ("http".equals(batchUrl.getProtocol())
+                        || "https".equals(batchUrl.getProtocol())) {
+                    tmp = File.createTempFile("batch_", ".zip");
+                    Downloader.fetchFile(batchUrl, tmp);
+                }
+                //TODO add FTP?
+                FileUtils.extractFiles(tmp, dataDirectory);
+
+            } catch (IOException ex) {
+                final String msg = String.format("IO Exception Occured performing batch update using: %s", batchSrc);
+                throw new UpdateException(msg, ex);
+            } finally {
+                if (tmp != null && !tmp.delete()) {
+                    tmp.deleteOnExit();
+                }
+            }
         }
     }
 

dependency-check-core/src/main/java/org/owasp/dependencycheck/utils/Downloader.java

@@ -131,23 +131,23 @@ public final class Downloader {
         long timestamp = 0;
         //TODO add the FPR protocol?
         if ("file".equalsIgnoreCase(url.getProtocol())) {
-            File f;
+            File lastModifiedFile;
             try {
-                if (System.getProperty("os.name").toLowerCase().startsWith("windows")) {
+//                if (System.getProperty("os.name").toLowerCase().startsWith("windows")) {
-                    String filePath = url.toString();
+//                    String filePath = url.toString();
-                    if (filePath.matches("file://[a-zA-Z]:.*")) {
+//                    if (filePath.matches("file://[a-zA-Z]:.*")) {
-                        f = new File(filePath.substring(7));
+//                        f = new File(filePath.substring(7));
-                    } else {
+//                    } else {
-                        f = new File(url.toURI());
+//                        f = new File(url.toURI());
-                    }
+//                    }
-                } else {
+//                } else {
-                    f = new File(url.toURI());
+                lastModifiedFile = new File(url.toURI());
-                }
+//                }
             } catch (URISyntaxException ex) {
                 final String msg = String.format("Unable to locate '%s'; is the cve.url-2.0.modified property set correctly?", url.toString());
                 throw new DownloadFailedException(msg);
             }
-            timestamp = f.lastModified();
+            timestamp = lastModifiedFile.lastModified();
         } else {
             HttpURLConnection conn = null;
             try {
@@ -192,11 +192,8 @@ public final class Downloader {
             } else {
                 conn = (HttpURLConnection) url.openConnection();
             }
-            //added a default timeout of 20000
-            //if (Settings.getString(Settings.KEYS.CONNECTION_TIMEOUT) != null) {
             final int timeout = Settings.getInt(Settings.KEYS.CONNECTION_TIMEOUT, 60000);
             conn.setConnectTimeout(timeout);
-            //}
         } catch (IOException ex) {
             if (conn != null) {
                 try {

dependency-check-core/src/main/java/org/owasp/dependencycheck/utils/FileUtils.java

@@ -200,7 +200,7 @@ public final class FileUtils {
             while ((entry = zis.getNextEntry()) != null) {
                 if (entry.isDirectory()) {
                     final File d = new File(extractTo, entry.getName());
-                    if (!d.mkdirs()) {
+                    if (!d.exists() && !d.mkdirs()) {
                         final String msg = String.format("Unable to create '%s'.", d.getAbsolutePath());
                         throw new ExtractionException(msg);
                     }

dependency-check-core/src/main/resources/dependencycheck.properties

@@ -24,7 +24,7 @@ cve.url.modified.validfordays=7
 # into the specified "data" directory. Additionally, after pulling the data the
 # system will attempt to update the modified. Thus, if one were maintaining an
 # internal copy of the data one would not need to update it nightly.
-batch.update.url=file://C:/Users/jeremy/Desktop/demo/cli/data/data.zip
+#batch.update.url=file:///C:/path/to/data.zip
 
 # the path to the modified nvd cve xml file.
 cve.url-1.2.modified=http://nvd.nist.gov/download/nvdcve-modified.xml

dependency-check-core/src/test/java/org/owasp/dependencycheck/data/nvdcve/xml/DatabaseUpdaterIntegrationTest.java

@@ -18,12 +18,15 @@
  */
 package org.owasp.dependencycheck.data.nvdcve.xml;
 
+import java.io.File;
+import java.net.URL;
 import org.owasp.dependencycheck.data.nvdcve.xml.DatabaseUpdater;
 import org.junit.After;
 import org.junit.AfterClass;
 import org.junit.Before;
 import org.junit.BeforeClass;
 import org.junit.Test;
+import org.owasp.dependencycheck.utils.Settings;
 
 /**
  *
@@ -60,4 +63,18 @@ public class DatabaseUpdaterIntegrationTest {
         DatabaseUpdater instance = new DatabaseUpdater();
         instance.update();
     }
+
+    /**
+     * Test of update method (when in batch mode), of class DatabaseUpdater.
+     *
+     * @throws Exception
+     */
+    @Test
+    public void testBatchUpdate() throws Exception {
+        File file = new File("target/test-classes/nvdcve-2.0-2012.xml");
+        String path = "file:///" + file.getCanonicalPath();
+        Settings.setString(Settings.KEYS.BATCH_UPDATE_URL, path);
+        DatabaseUpdater instance = new DatabaseUpdater();
+        instance.update();
+    }
 }

dependency-check-core/src/test/java/org/owasp/dependencycheck/data/nvdcve/xml/DatabaseUpdaterTest.java

@@ -0,0 +1,85 @@
+/*
+ * This file is part of dependency-check-core.
+ *
+ * Dependency-check-core is free software: you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation, either version 3 of the License, or (at your option) any
+ * later version.
+ *
+ * Dependency-check-core is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+ * details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * dependency-check-core. If not, see http://www.gnu.org/licenses/.
+ *
+ * Copyright (c) 2012 Jeremy Long. All Rights Reserved.
+ */
+package org.owasp.dependencycheck.data.nvdcve.xml;
+
+import org.owasp.dependencycheck.data.nvdcve.xml.DatabaseUpdater;
+import java.io.File;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.owasp.dependencycheck.utils.Settings;
+
+/**
+ *
+ * @author Jeremy Long (jeremy.long@owasp.org)
+ */
+public class DatabaseUpdaterTest {
+
+    public DatabaseUpdaterTest() {
+    }
+
+    @BeforeClass
+    public static void setUpClass() throws Exception {
+    }
+
+    @AfterClass
+    public static void tearDownClass() throws Exception {
+    }
+    private String old12;
+    private String old20;
+
+    @Before
+    public void setUp() throws Exception {
+        old12 = Settings.getString(Settings.KEYS.CVE_MODIFIED_12_URL);
+        old20 = Settings.getString(Settings.KEYS.CVE_MODIFIED_20_URL);
+
+        File file = new File("target/test-classes/nvdcve-2012.xml");
+        String path = "file:///" + file.getCanonicalPath();
+        Settings.setString(Settings.KEYS.CVE_MODIFIED_12_URL, path);
+
+        file = new File("target/test-classes/nvdcve-2.0-2012.xml");
+        path = "file:///" + file.getCanonicalPath();
+        Settings.setString(Settings.KEYS.CVE_MODIFIED_20_URL, path);
+
+        file = new File("target/test-classes/data.zip");
+        path = "file:///" + file.getCanonicalPath();
+        Settings.setString(Settings.KEYS.BATCH_UPDATE_URL, path);
+    }
+
+    @After
+    public void tearDown() {
+        Settings.setString(Settings.KEYS.CVE_MODIFIED_12_URL, old12);
+        Settings.setString(Settings.KEYS.CVE_MODIFIED_20_URL, old20);
+        Settings.setString(Settings.KEYS.BATCH_UPDATE_URL, "");
+    }
+
+    /**
+     * Test of update method (when in batch mode), of class DatabaseUpdater.
+     *
+     * @throws Exception
+     */
+    @Test
+    public void testBatchUpdate() throws Exception {
+        DatabaseUpdater instance = new DatabaseUpdater();
+        instance.deleteExistingData();
+        instance.update();
+    }
+}

dependency-check-core/src/test/java/org/owasp/dependencycheck/utils/DownloaderIntegrationTest.java

@@ -82,12 +82,4 @@ public class DownloaderIntegrationTest {
         long timestamp = Downloader.getLastModified(url);
         assertTrue("timestamp equal to zero?", timestamp > 0);
     }
-
-    @Test
-    public void testGetLastModified_file() throws Exception {
-        File f = new File("target/test-classes/nvdcve-2.0-2012.xml");
-        URL url = new URL("file://" + f.getCanonicalPath());
-        long timestamp = Downloader.getLastModified(url);
-        assertTrue("timestamp equal to zero?", timestamp > 0);
-    }
 }