Merge pull request #839 from jeremylong/h2upgrade

H2 Upgrade

dependency-check-cli/src/main/java/org/owasp/dependencycheck/App.java

@@ -112,7 +112,7 @@ public class App {
                 }
                 File db;
                 try {
-                    db = new File(Settings.getDataDirectory(), "dc.h2.db");
+                    db = new File(Settings.getDataDirectory(), Settings.getString(Settings.KEYS.DB_FILE_NAME, "dc.h2.db"));
                     if (db.exists()) {
                         if (db.delete()) {
                             LOGGER.info("Database file purged; local copy of the NVD has been removed");

dependency-check-core/src/main/java/org/owasp/dependencycheck/data/nvdcve/ConnectionFactory.java

@@ -82,6 +82,11 @@ public final class ConnectionFactory {
      * The password for the database.
      */
     private static String password = null;
+    /**
+     * Counter to ensure that calls to ensureSchemaVersion does not end up in an
+     * endless loop.
+     */
+    private static int callDepth = 0;
 
     /**
      * Private constructor for this factory class; no instance is ever needed.
@@ -369,12 +374,6 @@ public final class ConnectionFactory {
         }
     }
 
-    /**
-     * Counter to ensure that calls to ensureSchemaVersion does not end up in an
-     * endless loop.
-     */
-    private static int callDepth = 0;
-
     /**
      * Uses the provided connection to check the specified schema version within
      * the database.

dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/NvdCveUpdater.java

@@ -47,9 +47,11 @@ import org.owasp.dependencycheck.data.update.nvd.DownloadTask;
 import org.owasp.dependencycheck.data.update.nvd.NvdCveInfo;
 import org.owasp.dependencycheck.data.update.nvd.ProcessTask;
 import org.owasp.dependencycheck.data.update.nvd.UpdateableNvdCve;
+import org.owasp.dependencycheck.exception.H2DBLockException;
 import org.owasp.dependencycheck.utils.DateUtil;
 import org.owasp.dependencycheck.utils.Downloader;
 import org.owasp.dependencycheck.utils.DownloadFailedException;
+import org.owasp.dependencycheck.utils.H2DBLock;
 import org.owasp.dependencycheck.utils.InvalidSettingException;
 import org.owasp.dependencycheck.utils.Settings;
 import org.slf4j.Logger;
@@ -107,46 +109,9 @@ public class NvdCveUpdater implements CachedWebDataSource {
         if (isUpdateConfiguredFalse()) {
             return;
         }
-        FileLock lock = null;
+        H2DBLock dbupdate = new H2DBLock();
-        RandomAccessFile ulFile = null;
-        File lockFile = null;
         try {
-            if (ConnectionFactory.isH2Connection()) {
+            dbupdate.lock();
-                final File dir = Settings.getDataDirectory();
-                lockFile = new File(dir, "odc.update.lock");
-                if (lockFile.isFile() && getFileAge(lockFile) > 5 && !lockFile.delete()) {
-                    LOGGER.warn("An old db update lock file was found but the system was unable to delete "
-                            + "the file. Consider manually deleting {}", lockFile.getAbsolutePath());
-                }
-                int ctr = 0;
-                do {
-                    try {
-                        if (!lockFile.exists() && lockFile.createNewFile()) {
-                            ulFile = new RandomAccessFile(lockFile, "rw");
-                            lock = ulFile.getChannel().lock();
-                        }
-                    } catch (IOException ex) {
-                        LOGGER.trace("Expected error as another thread has likely locked the file", ex);
-                    } finally {
-                        if (lock == null && ulFile != null) {
-                            ulFile.close();
-                        }
-                    }
-                    if (lock == null || !lock.isValid()) {
-                        try {
-                            LOGGER.debug("Sleeping thread {} for 5 seconds because we could not obtain the update lock.",
-                                    Thread.currentThread().getName());
-                            Thread.sleep(5000);
-                        } catch (InterruptedException ex) {
-                            LOGGER.trace("ignorable error, sleep was interrupted.", ex);
-                            Thread.currentThread().interrupt();
-                        }
-                    }
-                } while (++ctr < 60 && (lock == null || !lock.isValid()));
-                if (lock == null || !lock.isValid()) {
-                    throw new UpdateException("Unable to obtain the update lock, skipping the database update. Skippinig the database update.");
-                }
-            }
             initializeExecutorServices();
             cveDb = CveDB.getInstance();
             dbProperties = cveDb.getDatabaseProperties();
@@ -168,30 +133,14 @@ public class NvdCveUpdater implements CachedWebDataSource {
             throw new UpdateException("Unable to download the NVD CVE data.", ex);
         } catch (DatabaseException ex) {
             throw new UpdateException("Database Exception, unable to update the data to use the most current data.", ex);
-        } catch (IOException ex) {
+        } catch (H2DBLockException ex) {
-            throw new UpdateException("Database Exception", ex);
+            throw new UpdateException("Unable to obtain an exclusive lock on the H2 database to perform updates", ex);
         } finally {
-            shutdownExecutorServices();
             if (cveDb != null) {
                 cveDb.close();
             }
-            if (lock != null) {
+            dbupdate.release();
-                try {
+            shutdownExecutorServices();
-                    lock.release();
-                } catch (IOException ex) {
-                    LOGGER.trace("Ignorable exception", ex);
-                }
-            }
-            if (ulFile != null) {
-                try {
-                    ulFile.close();
-                } catch (IOException ex) {
-                    LOGGER.trace("Ignorable exception", ex);
-                }
-            }
-            if (lockFile != null && lockFile.isFile() && !lockFile.delete()) {
-                LOGGER.error("Lock file '{}' was unable to be deleted. Please manually delete this file.", lockFile.toString());
-            }
         }
     }
 
@@ -218,18 +167,6 @@ public class NvdCveUpdater implements CachedWebDataSource {
         return !autoUpdate;
     }
 
-    /**
-     * Returns the age of the file in minutes.
-     *
-     * @param file the file to calculate the age
-     * @return the age of the file
-     */
-    private long getFileAge(File file) {
-        final Date d = new Date();
-        final long modified = file.lastModified();
-        return (d.getTime() - modified) / 1000 / 60;
-    }
-
     /**
      * Initialize the executor services for download and processing of the NVD
      * CVE XML data.

dependency-check-core/src/main/java/org/owasp/dependencycheck/exception/H2DBLockException.java

@@ -0,0 +1,66 @@
+/*
+ * This file is part of dependency-check-core.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * Copyright (c) 2017 Jeremy Long. All Rights Reserved.
+ */
+package org.owasp.dependencycheck.exception;
+
+/**
+ * An exception used when trying to obtain a lock on the H2 database.
+ *
+ * @author Jeremy Long
+ */
+public class H2DBLockException extends Exception {
+
+    /**
+     * The serial version uid.
+     */
+    private static final long serialVersionUID = 1L;
+
+    /**
+     * Creates a new H2DBLockException.
+     */
+    public H2DBLockException() {
+        super();
+    }
+
+    /**
+     * Creates a new H2DBLockException.
+     *
+     * @param msg a message for the exception.
+     */
+    public H2DBLockException(String msg) {
+        super(msg);
+    }
+
+    /**
+     * Creates a new H2DBLockException.
+     *
+     * @param ex the cause of the exception.
+     */
+    public H2DBLockException(Throwable ex) {
+        super(ex);
+    }
+
+    /**
+     * Creates a new H2DBLockException.
+     *
+     * @param msg a message for the exception.
+     * @param ex the cause of the exception.
+     */
+    public H2DBLockException(String msg, Throwable ex) {
+        super(msg, ex);
+    }
+}

dependency-check-core/src/main/java/org/owasp/dependencycheck/utils/H2DBLock.java

@@ -0,0 +1,152 @@
+/*
+ * This file is part of dependency-check-core.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * Copyright (c) 2017 Jeremy Long. All Rights Reserved.
+ */
+package org.owasp.dependencycheck.utils;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.RandomAccessFile;
+import java.nio.channels.FileLock;
+import java.util.Date;
+import org.owasp.dependencycheck.data.nvdcve.ConnectionFactory;
+import org.owasp.dependencycheck.exception.H2DBLockException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ *
+ * @author Jeremy Long
+ */
+public class H2DBLock {
+
+    /**
+     * The logger.
+     */
+    private static final Logger LOGGER = LoggerFactory.getLogger(H2DBLock.class);
+    /**
+     * The file lock.
+     */
+    private FileLock lock = null;
+    /**
+     * Reference to the file that we are locking.
+     */
+    private RandomAccessFile file = null;
+    /**
+     * The lock file.
+     */
+    private File lockFile = null;
+
+    /**
+     * Determine if the lock is currently held.
+     *
+     * @return true if the lock is currently held
+     */
+    public boolean isLocked() {
+        return lock != null && lock.isValid();
+    }
+
+    /**
+     * Obtains a lock on the H2 database.
+     *
+     * @throws H2DBLockException thrown if a lock could not be obtained
+     */
+    public void lock() throws H2DBLockException {
+        if (ConnectionFactory.isH2Connection()) {
+            try {
+                final File dir = Settings.getDataDirectory();
+                lockFile = new File(dir, "dc.update.lock");
+                if (lockFile.isFile() && getFileAge(lockFile) > 5 && !lockFile.delete()) {
+                    LOGGER.warn("An old db update lock file was found but the system was unable to delete "
+                            + "the file. Consider manually deleting {}", lockFile.getAbsolutePath());
+                }
+                int ctr = 0;
+                do {
+                    try {
+                        if (!lockFile.exists() && lockFile.createNewFile()) {
+                            file = new RandomAccessFile(lockFile, "rw");
+                            lock = file.getChannel().lock();
+                        }
+                    } catch (IOException ex) {
+                        LOGGER.trace("Expected error as another thread has likely locked the file", ex);
+                    } finally {
+                        if (lock == null && file != null) {
+                            try {
+                                file.close();
+                            } catch (IOException ex) {
+                                LOGGER.trace("Unable to close the ulFile", ex);
+                            }
+                        }
+                    }
+                    if (lock == null || !lock.isValid()) {
+                        try {
+                            LOGGER.debug("Sleeping thread {} for 5 seconds because we could not obtain the update lock.",
+                                    Thread.currentThread().getName());
+                            Thread.sleep(5000);
+                        } catch (InterruptedException ex) {
+                            LOGGER.trace("ignorable error, sleep was interrupted.", ex);
+                            Thread.currentThread().interrupt();
+                        }
+                    }
+                } while (++ctr < 60 && (lock == null || !lock.isValid()));
+                if (lock == null || !lock.isValid()) {
+                    throw new H2DBLockException("Unable to obtain the update lock, skipping the database update. Skippinig the database update.");
+                }
+            } catch (IOException ex) {
+                throw new H2DBLockException(ex.getMessage(), ex);
+            }
+        }
+    }
+
+    /**
+     * Releases the lock on the H2 database.
+     */
+    public void release() {
+        if (lock != null) {
+            try {
+                lock.release();
+                lock = null;
+            } catch (IOException ex) {
+                LOGGER.trace("Ignorable exception", ex);
+            }
+        }
+        if (file != null) {
+            try {
+                file.close();
+                file = null;
+            } catch (IOException ex) {
+                LOGGER.trace("Ignorable exception", ex);
+            }
+        }
+        if (lockFile != null && lockFile.isFile() && !lockFile.delete()) {
+            LOGGER.error("Lock file '{}' was unable to be deleted. Please manually delete this file.", lockFile.toString());
+            lockFile.deleteOnExit();
+        }
+        lockFile = null;
+    }
+
+    /**
+     * Returns the age of the file in minutes.
+     *
+     * @param file the file to calculate the age
+     * @return the age of the file
+     */
+    private long getFileAge(File file) {
+        final Date d = new Date();
+        final long modified = file.lastModified();
+        return (d.getTime() - modified) / 1000 / 60;
+    }
+}

dependency-check-core/src/main/resources/dependencycheck.properties

@@ -23,7 +23,7 @@ data.file_name=dc.h2.db
 ### the gradle PurgeDataExtension.
 data.version=3.0
 
-data.connection_string=jdbc:h2:file:%s;FILE_LOCK=SERIALIZED;AUTOCOMMIT=ON;
+data.connection_string=jdbc:h2:file:%s;MV_STORE=FALSE;AUTOCOMMIT=ON;LOCK_MODE=0;FILE_LOCK=NO
 #data.connection_string=jdbc:mysql://localhost:3306/dependencycheck
 
 # user name and password for the database connection. The inherent case is to use H2.

dependency-check-core/src/test/java/org/owasp/dependencycheck/BaseDBTestCase.java

@@ -70,9 +70,10 @@ public abstract class BaseDBTestCase extends BaseTest {
                         d.mkdir();
                         continue;
                     }
-                    File o = new File(dataPath, entry.getName());
+                    //File o = new File(dataPath, entry.getName());
-                    o.createNewFile();
+                    //o.createNewFile();
-                    try (FileOutputStream fos = new FileOutputStream(o, false);
+                    dataFile.createNewFile();
+                    try (FileOutputStream fos = new FileOutputStream(dataFile, false);
                             BufferedOutputStream dest = new BufferedOutputStream(fos, BUFFER_SIZE)) {
                         IOUtils.copy(zin, dest);
                     } catch (Throwable ex) {

dependency-check-core/src/test/java/org/owasp/dependencycheck/EngineModeIT.java

@@ -116,6 +116,11 @@ public class EngineModeIT extends BaseTest {
         assertThat(Files.exists(directory), is(true));
         assertThat(Files.isDirectory(directory), is(true));
         Path database = directory.resolve(Settings.getString(Settings.KEYS.DB_FILE_NAME));
+        System.err.println(database.toString());
+        for (String f : directory.toFile().list()) {
+            System.err.println(f);
+        }
+
         assertThat(Files.exists(database), is(exists));
     }
 }

dependency-check-core/src/test/resources/dependencycheck.properties

@@ -18,7 +18,7 @@ data.directory=[JAR]/data
 #if the filename has a %s it will be replaced with the current expected version
 data.file_name=dc.h2.db
 data.version=3.0
-data.connection_string=jdbc:h2:file:%s;FILE_LOCK=SERIALIZED;AUTOCOMMIT=ON;
+data.connection_string=jdbc:h2:file:%s;MV_STORE=FALSE;AUTOCOMMIT=ON;LOCK_MODE=0;FILE_LOCK=NO
 #data.connection_string=jdbc:mysql://localhost:3306/dependencycheck
 
 # user name and password for the database connection. The inherent case is to use H2.

dependency-check-maven/src/main/java/org/owasp/dependencycheck/maven/PurgeMojo.java

@@ -73,7 +73,7 @@ public class PurgeMojo extends BaseDependencyCheckMojo {
             populateSettings();
             File db;
             try {
-                db = new File(Settings.getDataDirectory(), "dc.h2.db");
+                db = new File(Settings.getDataDirectory(), Settings.getString(Settings.KEYS.DB_FILE_NAME, "dc.h2.db"));
                 if (db.exists()) {
                     if (db.delete()) {
                         getLog().info("Database file purged; local copy of the NVD has been removed");

dependency-check-utils/src/main/java/org/owasp/dependencycheck/utils/Settings.java

@@ -1010,7 +1010,7 @@ public final class Settings {
             // yes, for H2 this path won't actually exists - but this is sufficient to get the value needed
             final File dbFile = new File(directory, fileName);
             final String cString = String.format(connStr, dbFile.getCanonicalPath());
-            LOGGER.debug("Connection String: '{}'", cString);
+            LOGGER.error("Connection String: '{}'", cString);
             return cString;
         }
         return connStr;

dependency-check-utils/src/test/resources/dependencycheck.properties

@@ -17,7 +17,7 @@ engine.version.url=http://jeremylong.github.io/DependencyCheck/current.txt
 data.directory=[JAR]/data
 data.file_name=dc.h2.db
 data.version=3.0
-data.connection_string=jdbc:h2:file:%s;FILE_LOCK=SERIALIZED;AUTOCOMMIT=ON;
+data.connection_string=jdbc:h2:file:%s;MV_STORE=FALSE;AUTOCOMMIT=ON;LOCK_MODE=0;FILE_LOCK=NO
 #data.connection_string=jdbc:h2:file:%s;AUTO_SERVER=TRUE;AUTOCOMMIT=ON;
 #data.connection_string=jdbc:mysql://localhost:3306/dependencycheck
 

pom.xml

@@ -624,7 +624,7 @@ Copyright (c) 2012 - Jeremy Long
             <dependency>
                 <groupId>com.h2database</groupId>
                 <artifactId>h2</artifactId>
-                <version>1.3.176</version>
+                <version>1.4.196</version>
             </dependency>
             <dependency>
                 <groupId>commons-cli</groupId>