Merge branch 'awhitford-DbMerge'

dependency-check-core/pom.xml

@@ -468,7 +468,6 @@ Copyright (c) 2012 Jeremy Long. All Rights Reserved.
                     <plugin>
                         <groupId>org.apache.maven.plugins</groupId>
                         <artifactId>maven-surefire-plugin</artifactId>
-                        <version>2.18.1</version>
                         <configuration>
                             <skip>true</skip>
                         </configuration>
@@ -476,12 +475,11 @@ Copyright (c) 2012 Jeremy Long. All Rights Reserved.
                     <plugin>
                         <groupId>org.apache.maven.plugins</groupId>
                         <artifactId>maven-failsafe-plugin</artifactId>
-                        <version>2.18.1</version>
                         <configuration>
                             <systemProperties>
                                 <property>
                                     <name>data.driver_path</name>
-                                    <value>${basedir}/${driver_path}</value>
+                                    <value>${driver_path}</value>
                                 </property>
                                 <property>
                                     <name>data.driver_name</name>

dependency-check-core/src/main/java/org/owasp/dependencycheck/data/nvdcve/ConnectionFactory.java

@@ -29,7 +29,10 @@ import java.sql.DriverManager;
 import java.sql.ResultSet;
 import java.sql.SQLException;
 import java.sql.Statement;
+import java.util.logging.Level;
 import org.owasp.dependencycheck.utils.DBUtils;
+import org.owasp.dependencycheck.utils.DependencyVersion;
+import org.owasp.dependencycheck.utils.DependencyVersionUtil;
 import org.owasp.dependencycheck.utils.Settings;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -58,6 +61,10 @@ public final class ConnectionFactory {
      * Resource location for SQL file used to create the database schema.
      */
     public static final String DB_STRUCTURE_UPDATE_RESOURCE = "data/upgrade_%s.sql";
+    /**
+     * The URL that discusses upgrading non-H2 databases.
+     */
+    public static final String UPGRADE_HELP_URL = "http://jeremylong.github.io/DependencyCheck/data/upgrade.html";
     /**
      * The database driver used to connect to the database.
      */
@@ -288,6 +295,13 @@ public final class ConnectionFactory {
      * @throws DatabaseException thrown if there is an exception upgrading the database schema
      */
     private static void updateSchema(Connection conn, String schema) throws DatabaseException {
+        final String databaseProductName;
+        try {
+            databaseProductName = conn.getMetaData().getDatabaseProductName();
+        } catch (SQLException ex) {
+            throw new DatabaseException("Unable to get the database product name");
+        }
+        if ("h2".equalsIgnoreCase(databaseProductName)) {
             LOGGER.debug("Updating database structure");
             InputStream is;
             InputStreamReader reader;
@@ -328,7 +342,16 @@ public final class ConnectionFactory {
                     }
                 }
             }
+        } else {
+            LOGGER.error("The database schema must be upgraded to use this version of dependency-check. Please see {} for more information.", UPGRADE_HELP_URL);
+            throw new DatabaseException("Database schema is out of date");
         }
+    }
+
+    /**
+     * Counter to ensure that calls to ensureSchemaVersion does not end up in an endless loop.
+     */
+    private static int callDepth = 0;
 
     /**
      * Uses the provided connection to check the specified schema version within the database.
@@ -344,10 +367,15 @@ public final class ConnectionFactory {
             cs = conn.prepareCall("SELECT value FROM properties WHERE id = 'version'");
             rs = cs.executeQuery();
             if (rs.next()) {
-                if (!DB_SCHEMA_VERSION.equals(rs.getString(1))) {
+                final DependencyVersion current = DependencyVersionUtil.parseVersion(DB_SCHEMA_VERSION);
+                final DependencyVersion db = DependencyVersionUtil.parseVersion(rs.getString(1));
+                if (current.compareTo(db) > 0) {
                     LOGGER.debug("Current Schema: " + DB_SCHEMA_VERSION);
                     LOGGER.debug("DB Schema: " + rs.getString(1));
                     updateSchema(conn, rs.getString(1));
+                    if (++callDepth < 10) {
+                        ensureSchemaVersion(conn);
+                    }
                 }
             } else {
                 throw new DatabaseException("Database schema is missing");

dependency-check-core/src/main/java/org/owasp/dependencycheck/data/nvdcve/CveDB.java

@@ -29,8 +29,10 @@ import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
+import java.util.Locale;
 import java.util.Map;
 import java.util.Map.Entry;
+import java.util.MissingResourceException;
 import java.util.Properties;
 import java.util.ResourceBundle;
 import java.util.Set;
@@ -74,9 +76,17 @@ public class CveDB {
      */
     public CveDB() throws DatabaseException {
         super();
-        statementBundle = ResourceBundle.getBundle("data/dbStatements");
         try {
             open();
+            try {
+                final String databaseProductName = conn.getMetaData().getDatabaseProductName();
+                LOGGER.debug("Database dialect: {}", databaseProductName);
+                final Locale dbDialect = new Locale(databaseProductName);
+                statementBundle = ResourceBundle.getBundle("data/dbStatements", dbDialect);
+            } catch (SQLException se) {
+                LOGGER.warn("Problem loading database specific dialect!", se);
+                statementBundle = ResourceBundle.getBundle("data/dbStatements");
+            }
             databaseProperties = new DatabaseProperties(this);
         } catch (DatabaseException ex) {
             throw ex;
@@ -252,44 +262,6 @@ public class CveDB {
         return prop;
     }
 
-    /**
-     * Saves a set of properties to the database.
-     *
-     * @param props a collection of properties
-     */
-    void saveProperties(Properties props) {
-        PreparedStatement updateProperty = null;
-        PreparedStatement insertProperty = null;
-        try {
-            try {
-                updateProperty = getConnection().prepareStatement(statementBundle.getString("UPDATE_PROPERTY"));
-                insertProperty = getConnection().prepareStatement(statementBundle.getString("INSERT_PROPERTY"));
-            } catch (SQLException ex) {
-                LOGGER.warn("Unable to save properties to the database");
-                LOGGER.debug("Unable to save properties to the database", ex);
-                return;
-            }
-            for (Entry<Object, Object> entry : props.entrySet()) {
-                final String key = entry.getKey().toString();
-                final String value = entry.getValue().toString();
-                try {
-                    updateProperty.setString(1, value);
-                    updateProperty.setString(2, key);
-                    if (updateProperty.executeUpdate() == 0) {
-                        insertProperty.setString(1, key);
-                        insertProperty.setString(2, value);
-                    }
-                } catch (SQLException ex) {
-                    LOGGER.warn("Unable to save property '{}' with a value of '{}' to the database", key, value);
-                    LOGGER.debug("", ex);
-                }
-            }
-        } finally {
-            DBUtils.closeStatement(updateProperty);
-            DBUtils.closeStatement(insertProperty);
-        }
-    }
-
     /**
      * Saves a property to the database.
      *
@@ -297,40 +269,40 @@ public class CveDB {
      * @param value the property value
      */
     void saveProperty(String key, String value) {
+        try {
+            try {
+                final PreparedStatement mergeProperty = getConnection().prepareStatement(statementBundle.getString("MERGE_PROPERTY"));
+                try {
+                    mergeProperty.setString(1, key);
+                    mergeProperty.setString(2, value);
+                    mergeProperty.executeUpdate();
+                } finally {
+                    DBUtils.closeStatement(mergeProperty);
+                }
+            } catch (MissingResourceException mre) {
+                // No Merge statement, so doing an Update/Insert...
                 PreparedStatement updateProperty = null;
                 PreparedStatement insertProperty = null;
-        try {
                 try {
                     updateProperty = getConnection().prepareStatement(statementBundle.getString("UPDATE_PROPERTY"));
-            } catch (SQLException ex) {
-                LOGGER.warn("Unable to save properties to the database");
-                LOGGER.debug("Unable to save properties to the database", ex);
-                return;
-            }
-            try {
                     updateProperty.setString(1, value);
                     updateProperty.setString(2, key);
                     if (updateProperty.executeUpdate() == 0) {
-                    try {
                         insertProperty = getConnection().prepareStatement(statementBundle.getString("INSERT_PROPERTY"));
-                    } catch (SQLException ex) {
-                        LOGGER.warn("Unable to save properties to the database");
-                        LOGGER.debug("Unable to save properties to the database", ex);
-                        return;
-                    }
                         insertProperty.setString(1, key);
                         insertProperty.setString(2, value);
-                    insertProperty.execute();
+                        insertProperty.executeUpdate();
-                }
-            } catch (SQLException ex) {
-                LOGGER.warn("Unable to save property '{}' with a value of '{}' to the database", key, value);
-                LOGGER.debug("", ex);
                     }
                 } finally {
                     DBUtils.closeStatement(updateProperty);
                     DBUtils.closeStatement(insertProperty);
                 }
             }
+        } catch (SQLException ex) {
+            LOGGER.warn("Unable to save property '{}' with a value of '{}' to the database", key, value);
+            LOGGER.debug("", ex);
+        }
+    }
 
     /**
      * Retrieves the vulnerabilities associated with the specified CPE.

dependency-check-core/src/main/resources/data/dbStatements_h2.properties

@@ -0,0 +1,15 @@
+# Copyright 2015 OWASP.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+MERGE_PROPERTY=MERGE INTO properties (id, value) KEY(id) VALUES(?, ?)

dependency-check-core/src/main/resources/data/upgrade_2.9.sql

@@ -4,4 +4,4 @@
 --ALTER TABLE cpeEntry ALTER COLUMN dictionaryEntry  SET DEFAULT FALSE;
 --UPDATE cpeEntry SET dictionaryEntry=false;
 
---UPDATE Properties SET value='3.0' WHERE ID='version';
+UPDATE Properties SET value='3.0' WHERE ID='version';

dependency-check-core/src/main/resources/data/upgrade_mysql_2.9.sql

@@ -0,0 +1,15 @@
+
+DROP PROCEDURE IF EXISTS save_property;
+
+DELIMITER //
+CREATE PROCEDURE save_property
+(IN prop varchar(50), IN val varchar(500))
+BEGIN
+INSERT INTO properties (`id`, `value`) VALUES (prop, val)
+	ON DUPLICATE KEY UPDATE `value`=val;
+END //
+DELIMITER ;
+
+GRANT EXECUTE ON PROCEDURE dependencycheck.save_property TO 'dcuser';
+
+UPDATE Properties SET value='3.0' WHERE ID='version';

dependency-check-core/src/main/resources/dependencycheck.properties

@@ -18,7 +18,7 @@ engine.version.url=http://jeremylong.github.io/DependencyCheck/current.txt
 data.directory=[JAR]/data
 #if the filename has a %s it will be replaced with the current expected version
 data.file_name=dc.h2.db
-data.version=2.9
+data.version=3.0
 data.connection_string=jdbc:h2:file:%s;FILE_LOCK=SERIALIZED;AUTOCOMMIT=ON;
 #data.connection_string=jdbc:mysql://localhost:3306/dependencycheck
 

dependency-check-core/src/test/java/org/owasp/dependencycheck/data/nvdcve/CveDBMySQLTest.java

@@ -26,6 +26,7 @@ import org.junit.Before;
 import org.junit.BeforeClass;
 import org.junit.Test;
 import org.owasp.dependencycheck.dependency.VulnerableSoftware;
+import org.owasp.dependencycheck.utils.Settings;
 
 /**
  *
@@ -35,10 +36,12 @@ public class CveDBMySQLTest {
 
     @BeforeClass
     public static void setUpClass() {
+        Settings.initialize();
     }
 
     @AfterClass
     public static void tearDownClass() {
+        Settings.cleanup();
     }
 
     @Before

dependency-check-core/src/test/resources/dependencycheck.properties

@@ -20,7 +20,7 @@ data.directory=[JAR]/data
 # based databases the below filename will be added to the data directory above and then
 # if the connection string has a %s it will be replaced by the directory/filename path.
 data.file_name=dc.h2.db
-data.version=2.9
+data.version=3.0
 data.connection_string=jdbc:h2:file:%s;FILE_LOCK=SERIALIZED;AUTOCOMMIT=ON;
 #data.connection_string=jdbc:mysql://localhost:3306/dependencycheck
 

dependency-check-utils/src/test/resources/dependencycheck.properties

@@ -17,7 +17,7 @@ engine.version.url=http://jeremylong.github.io/DependencyCheck/current.txt
 # below contains a %s then the data.directory will replace the %s.
 data.directory=[JAR]/data
 data.file_name=dc.h2.db
-data.version=2.9
+data.version=3.0
 data.connection_string=jdbc:h2:file:%s;FILE_LOCK=SERIALIZED;AUTOCOMMIT=ON;
 #data.connection_string=jdbc:h2:file:%s;AUTO_SERVER=TRUE;AUTOCOMMIT=ON;
 #data.connection_string=jdbc:mysql://localhost:3306/dependencycheck

src/site/markdown/data/database.md

@@ -32,6 +32,11 @@ To setup a centralized database the following generalized steps can be used:
    </ul>
 </li></ol>
 Depending on the database being used, you may need to customize the [dbStatements.properties](https://github.com/jeremylong/DependencyCheck/blob/master/dependency-check-core/src/main/resources/data/dbStatements.properties).
+Alternatively to modifying the dbStatements.properties it is now possible to use a dialect file to support other databases.
+See [dbStatements_h2.properties](https://github.com/jeremylong/DependencyCheck/blob/master/dependency-check-core/src/main/resources/data/dbStatements_h2.properties)
+as an example.
+
+Also, if using an external database you will need to manually upgrade the schema. See [database upgrades](./upgrade.html) for more information.
 
 As always, feel free to open an [issue](https://github.com/jeremylong/DependencyCheck/issues)
 or post a question to the [dependency-check google group](https://groups.google.com/forum/#!forum/dependency-check).

src/site/markdown/data/upgrade.md

@@ -0,0 +1,8 @@
+Database Upgrades
+=================
+If using an external database server, such as MySQL, a DBA must manually perform
+the database upgrades. Currently, a copy of the initialization and upgrade scripts
+for MySQL can be found in the [github repository](https://github.com/jeremylong/DependencyCheck/tree/master/dependency-check-core/src/main/resources/data).
+
+If you want to use an external database other then MySQL please open an issue in our [issue tracker](https://github.com/jeremylong/DependencyCheck/issues)
+as a dialect properties file will need to be created.