diff --git a/dependency-check-core/pom.xml b/dependency-check-core/pom.xml index 8b5fc91d9..f64477a74 100644 --- a/dependency-check-core/pom.xml +++ b/dependency-check-core/pom.xml @@ -468,7 +468,6 @@ Copyright (c) 2012 Jeremy Long. All Rights Reserved. org.apache.maven.plugins maven-surefire-plugin - 2.18.1 true @@ -476,12 +475,11 @@ Copyright (c) 2012 Jeremy Long. All Rights Reserved. org.apache.maven.plugins maven-failsafe-plugin - 2.18.1 data.driver_path - ${basedir}/${driver_path} + ${driver_path} data.driver_name diff --git a/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/nvdcve/ConnectionFactory.java b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/nvdcve/ConnectionFactory.java index 4378e363f..798deb750 100644 --- a/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/nvdcve/ConnectionFactory.java +++ b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/nvdcve/ConnectionFactory.java @@ -29,7 +29,10 @@ import java.sql.DriverManager; import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Statement; +import java.util.logging.Level; import org.owasp.dependencycheck.utils.DBUtils; +import org.owasp.dependencycheck.utils.DependencyVersion; +import org.owasp.dependencycheck.utils.DependencyVersionUtil; import org.owasp.dependencycheck.utils.Settings; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -58,6 +61,10 @@ public final class ConnectionFactory { * Resource location for SQL file used to create the database schema. */ public static final String DB_STRUCTURE_UPDATE_RESOURCE = "data/upgrade_%s.sql"; + /** + * The URL that discusses upgrading non-H2 databases. + */ + public static final String UPGRADE_HELP_URL = "http://jeremylong.github.io/DependencyCheck/data/upgrade.html"; /** * The database driver used to connect to the database. */ @@ -288,48 +295,64 @@ public final class ConnectionFactory { * @throws DatabaseException thrown if there is an exception upgrading the database schema */ private static void updateSchema(Connection conn, String schema) throws DatabaseException { - LOGGER.debug("Updating database structure"); - InputStream is; - InputStreamReader reader; - BufferedReader in = null; - String updateFile = null; + final String databaseProductName; try { - updateFile = String.format(DB_STRUCTURE_UPDATE_RESOURCE, schema); - is = ConnectionFactory.class.getClassLoader().getResourceAsStream(updateFile); - if (is == null) { - throw new DatabaseException(String.format("Unable to load update file '%s'", updateFile)); - } - reader = new InputStreamReader(is, "UTF-8"); - in = new BufferedReader(reader); - final StringBuilder sb = new StringBuilder(2110); - String tmp; - while ((tmp = in.readLine()) != null) { - sb.append(tmp); - } - Statement statement = null; + databaseProductName = conn.getMetaData().getDatabaseProductName(); + } catch (SQLException ex) { + throw new DatabaseException("Unable to get the database product name"); + } + if ("h2".equalsIgnoreCase(databaseProductName)) { + LOGGER.debug("Updating database structure"); + InputStream is; + InputStreamReader reader; + BufferedReader in = null; + String updateFile = null; try { - statement = conn.createStatement(); - statement.execute(sb.toString()); - } catch (SQLException ex) { - LOGGER.debug("", ex); - throw new DatabaseException("Unable to update database schema", ex); - } finally { - DBUtils.closeStatement(statement); - } - } catch (IOException ex) { - final String msg = String.format("Upgrade SQL file does not exist: %s", updateFile); - throw new DatabaseException(msg, ex); - } finally { - if (in != null) { + updateFile = String.format(DB_STRUCTURE_UPDATE_RESOURCE, schema); + is = ConnectionFactory.class.getClassLoader().getResourceAsStream(updateFile); + if (is == null) { + throw new DatabaseException(String.format("Unable to load update file '%s'", updateFile)); + } + reader = new InputStreamReader(is, "UTF-8"); + in = new BufferedReader(reader); + final StringBuilder sb = new StringBuilder(2110); + String tmp; + while ((tmp = in.readLine()) != null) { + sb.append(tmp); + } + Statement statement = null; try { - in.close(); - } catch (IOException ex) { - LOGGER.trace("", ex); + statement = conn.createStatement(); + statement.execute(sb.toString()); + } catch (SQLException ex) { + LOGGER.debug("", ex); + throw new DatabaseException("Unable to update database schema", ex); + } finally { + DBUtils.closeStatement(statement); + } + } catch (IOException ex) { + final String msg = String.format("Upgrade SQL file does not exist: %s", updateFile); + throw new DatabaseException(msg, ex); + } finally { + if (in != null) { + try { + in.close(); + } catch (IOException ex) { + LOGGER.trace("", ex); + } } } + } else { + LOGGER.error("The database schema must be upgraded to use this version of dependency-check. Please see {} for more information.", UPGRADE_HELP_URL); + throw new DatabaseException("Database schema is out of date"); } } + /** + * Counter to ensure that calls to ensureSchemaVersion does not end up in an endless loop. + */ + private static int callDepth = 0; + /** * Uses the provided connection to check the specified schema version within the database. * @@ -344,10 +367,15 @@ public final class ConnectionFactory { cs = conn.prepareCall("SELECT value FROM properties WHERE id = 'version'"); rs = cs.executeQuery(); if (rs.next()) { - if (!DB_SCHEMA_VERSION.equals(rs.getString(1))) { + final DependencyVersion current = DependencyVersionUtil.parseVersion(DB_SCHEMA_VERSION); + final DependencyVersion db = DependencyVersionUtil.parseVersion(rs.getString(1)); + if (current.compareTo(db) > 0) { LOGGER.debug("Current Schema: " + DB_SCHEMA_VERSION); LOGGER.debug("DB Schema: " + rs.getString(1)); updateSchema(conn, rs.getString(1)); + if (++callDepth < 10) { + ensureSchemaVersion(conn); + } } } else { throw new DatabaseException("Database schema is missing"); diff --git a/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/nvdcve/CveDB.java b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/nvdcve/CveDB.java index 9df790976..28e835072 100644 --- a/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/nvdcve/CveDB.java +++ b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/nvdcve/CveDB.java @@ -29,8 +29,10 @@ import java.util.ArrayList; import java.util.HashMap; import java.util.HashSet; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Map.Entry; +import java.util.MissingResourceException; import java.util.Properties; import java.util.ResourceBundle; import java.util.Set; @@ -74,9 +76,17 @@ public class CveDB { */ public CveDB() throws DatabaseException { super(); - statementBundle = ResourceBundle.getBundle("data/dbStatements"); try { open(); + try { + final String databaseProductName = conn.getMetaData().getDatabaseProductName(); + LOGGER.debug("Database dialect: {}", databaseProductName); + final Locale dbDialect = new Locale(databaseProductName); + statementBundle = ResourceBundle.getBundle("data/dbStatements", dbDialect); + } catch (SQLException se) { + LOGGER.warn("Problem loading database specific dialect!", se); + statementBundle = ResourceBundle.getBundle("data/dbStatements"); + } databaseProperties = new DatabaseProperties(this); } catch (DatabaseException ex) { throw ex; @@ -252,44 +262,6 @@ public class CveDB { return prop; } - /** - * Saves a set of properties to the database. - * - * @param props a collection of properties - */ - void saveProperties(Properties props) { - PreparedStatement updateProperty = null; - PreparedStatement insertProperty = null; - try { - try { - updateProperty = getConnection().prepareStatement(statementBundle.getString("UPDATE_PROPERTY")); - insertProperty = getConnection().prepareStatement(statementBundle.getString("INSERT_PROPERTY")); - } catch (SQLException ex) { - LOGGER.warn("Unable to save properties to the database"); - LOGGER.debug("Unable to save properties to the database", ex); - return; - } - for (Entry entry : props.entrySet()) { - final String key = entry.getKey().toString(); - final String value = entry.getValue().toString(); - try { - updateProperty.setString(1, value); - updateProperty.setString(2, key); - if (updateProperty.executeUpdate() == 0) { - insertProperty.setString(1, key); - insertProperty.setString(2, value); - } - } catch (SQLException ex) { - LOGGER.warn("Unable to save property '{}' with a value of '{}' to the database", key, value); - LOGGER.debug("", ex); - } - } - } finally { - DBUtils.closeStatement(updateProperty); - DBUtils.closeStatement(insertProperty); - } - } - /** * Saves a property to the database. * @@ -297,38 +269,38 @@ public class CveDB { * @param value the property value */ void saveProperty(String key, String value) { - PreparedStatement updateProperty = null; - PreparedStatement insertProperty = null; try { try { - updateProperty = getConnection().prepareStatement(statementBundle.getString("UPDATE_PROPERTY")); - } catch (SQLException ex) { - LOGGER.warn("Unable to save properties to the database"); - LOGGER.debug("Unable to save properties to the database", ex); - return; - } - try { - updateProperty.setString(1, value); - updateProperty.setString(2, key); - if (updateProperty.executeUpdate() == 0) { - try { - insertProperty = getConnection().prepareStatement(statementBundle.getString("INSERT_PROPERTY")); - } catch (SQLException ex) { - LOGGER.warn("Unable to save properties to the database"); - LOGGER.debug("Unable to save properties to the database", ex); - return; - } - insertProperty.setString(1, key); - insertProperty.setString(2, value); - insertProperty.execute(); + final PreparedStatement mergeProperty = getConnection().prepareStatement(statementBundle.getString("MERGE_PROPERTY")); + try { + mergeProperty.setString(1, key); + mergeProperty.setString(2, value); + mergeProperty.executeUpdate(); + } finally { + DBUtils.closeStatement(mergeProperty); + } + } catch (MissingResourceException mre) { + // No Merge statement, so doing an Update/Insert... + PreparedStatement updateProperty = null; + PreparedStatement insertProperty = null; + try { + updateProperty = getConnection().prepareStatement(statementBundle.getString("UPDATE_PROPERTY")); + updateProperty.setString(1, value); + updateProperty.setString(2, key); + if (updateProperty.executeUpdate() == 0) { + insertProperty = getConnection().prepareStatement(statementBundle.getString("INSERT_PROPERTY")); + insertProperty.setString(1, key); + insertProperty.setString(2, value); + insertProperty.executeUpdate(); + } + } finally { + DBUtils.closeStatement(updateProperty); + DBUtils.closeStatement(insertProperty); } - } catch (SQLException ex) { - LOGGER.warn("Unable to save property '{}' with a value of '{}' to the database", key, value); - LOGGER.debug("", ex); } - } finally { - DBUtils.closeStatement(updateProperty); - DBUtils.closeStatement(insertProperty); + } catch (SQLException ex) { + LOGGER.warn("Unable to save property '{}' with a value of '{}' to the database", key, value); + LOGGER.debug("", ex); } } diff --git a/dependency-check-core/src/main/resources/data/dbStatements_h2.properties b/dependency-check-core/src/main/resources/data/dbStatements_h2.properties new file mode 100644 index 000000000..aea9f986a --- /dev/null +++ b/dependency-check-core/src/main/resources/data/dbStatements_h2.properties @@ -0,0 +1,15 @@ +# Copyright 2015 OWASP. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +MERGE_PROPERTY=MERGE INTO properties (id, value) KEY(id) VALUES(?, ?) diff --git a/dependency-check-core/src/main/resources/data/upgrade_2.9.sql b/dependency-check-core/src/main/resources/data/upgrade_2.9.sql index 6c2c3382d..87f5b1a4a 100644 --- a/dependency-check-core/src/main/resources/data/upgrade_2.9.sql +++ b/dependency-check-core/src/main/resources/data/upgrade_2.9.sql @@ -4,4 +4,4 @@ --ALTER TABLE cpeEntry ALTER COLUMN dictionaryEntry SET DEFAULT FALSE; --UPDATE cpeEntry SET dictionaryEntry=false; ---UPDATE Properties SET value='3.0' WHERE ID='version'; \ No newline at end of file +UPDATE Properties SET value='3.0' WHERE ID='version'; \ No newline at end of file diff --git a/dependency-check-core/src/main/resources/data/upgrade_mysql_2.9.sql b/dependency-check-core/src/main/resources/data/upgrade_mysql_2.9.sql new file mode 100644 index 000000000..08b89ceac --- /dev/null +++ b/dependency-check-core/src/main/resources/data/upgrade_mysql_2.9.sql @@ -0,0 +1,15 @@ + +DROP PROCEDURE IF EXISTS save_property; + +DELIMITER // +CREATE PROCEDURE save_property +(IN prop varchar(50), IN val varchar(500)) +BEGIN +INSERT INTO properties (`id`, `value`) VALUES (prop, val) + ON DUPLICATE KEY UPDATE `value`=val; +END // +DELIMITER ; + +GRANT EXECUTE ON PROCEDURE dependencycheck.save_property TO 'dcuser'; + +UPDATE Properties SET value='3.0' WHERE ID='version'; \ No newline at end of file diff --git a/dependency-check-core/src/main/resources/dependencycheck.properties b/dependency-check-core/src/main/resources/dependencycheck.properties index 9a45d8f3f..c6b2b26fa 100644 --- a/dependency-check-core/src/main/resources/dependencycheck.properties +++ b/dependency-check-core/src/main/resources/dependencycheck.properties @@ -18,7 +18,7 @@ engine.version.url=http://jeremylong.github.io/DependencyCheck/current.txt data.directory=[JAR]/data #if the filename has a %s it will be replaced with the current expected version data.file_name=dc.h2.db -data.version=2.9 +data.version=3.0 data.connection_string=jdbc:h2:file:%s;FILE_LOCK=SERIALIZED;AUTOCOMMIT=ON; #data.connection_string=jdbc:mysql://localhost:3306/dependencycheck diff --git a/dependency-check-core/src/test/java/org/owasp/dependencycheck/data/nvdcve/CveDBMySQLTest.java b/dependency-check-core/src/test/java/org/owasp/dependencycheck/data/nvdcve/CveDBMySQLTest.java index bed456159..db0b22f2c 100644 --- a/dependency-check-core/src/test/java/org/owasp/dependencycheck/data/nvdcve/CveDBMySQLTest.java +++ b/dependency-check-core/src/test/java/org/owasp/dependencycheck/data/nvdcve/CveDBMySQLTest.java @@ -26,6 +26,7 @@ import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; import org.owasp.dependencycheck.dependency.VulnerableSoftware; +import org.owasp.dependencycheck.utils.Settings; /** * @@ -35,10 +36,12 @@ public class CveDBMySQLTest { @BeforeClass public static void setUpClass() { + Settings.initialize(); } @AfterClass public static void tearDownClass() { + Settings.cleanup(); } @Before diff --git a/dependency-check-core/src/test/resources/dependencycheck.properties b/dependency-check-core/src/test/resources/dependencycheck.properties index b7cdffb69..fdaed3919 100644 --- a/dependency-check-core/src/test/resources/dependencycheck.properties +++ b/dependency-check-core/src/test/resources/dependencycheck.properties @@ -20,7 +20,7 @@ data.directory=[JAR]/data # based databases the below filename will be added to the data directory above and then # if the connection string has a %s it will be replaced by the directory/filename path. data.file_name=dc.h2.db -data.version=2.9 +data.version=3.0 data.connection_string=jdbc:h2:file:%s;FILE_LOCK=SERIALIZED;AUTOCOMMIT=ON; #data.connection_string=jdbc:mysql://localhost:3306/dependencycheck diff --git a/dependency-check-utils/src/test/resources/dependencycheck.properties b/dependency-check-utils/src/test/resources/dependencycheck.properties index 6c156df32..49f47d480 100644 --- a/dependency-check-utils/src/test/resources/dependencycheck.properties +++ b/dependency-check-utils/src/test/resources/dependencycheck.properties @@ -17,7 +17,7 @@ engine.version.url=http://jeremylong.github.io/DependencyCheck/current.txt # below contains a %s then the data.directory will replace the %s. data.directory=[JAR]/data data.file_name=dc.h2.db -data.version=2.9 +data.version=3.0 data.connection_string=jdbc:h2:file:%s;FILE_LOCK=SERIALIZED;AUTOCOMMIT=ON; #data.connection_string=jdbc:h2:file:%s;AUTO_SERVER=TRUE;AUTOCOMMIT=ON; #data.connection_string=jdbc:mysql://localhost:3306/dependencycheck diff --git a/src/site/markdown/data/database.md b/src/site/markdown/data/database.md index 3992d4973..b41b7fb61 100644 --- a/src/site/markdown/data/database.md +++ b/src/site/markdown/data/database.md @@ -32,6 +32,11 @@ To setup a centralized database the following generalized steps can be used: Depending on the database being used, you may need to customize the [dbStatements.properties](https://github.com/jeremylong/DependencyCheck/blob/master/dependency-check-core/src/main/resources/data/dbStatements.properties). +Alternatively to modifying the dbStatements.properties it is now possible to use a dialect file to support other databases. +See [dbStatements_h2.properties](https://github.com/jeremylong/DependencyCheck/blob/master/dependency-check-core/src/main/resources/data/dbStatements_h2.properties) +as an example. + +Also, if using an external database you will need to manually upgrade the schema. See [database upgrades](./upgrade.html) for more information. As always, feel free to open an [issue](https://github.com/jeremylong/DependencyCheck/issues) or post a question to the [dependency-check google group](https://groups.google.com/forum/#!forum/dependency-check). diff --git a/src/site/markdown/data/upgrade.md b/src/site/markdown/data/upgrade.md new file mode 100644 index 000000000..abf5f1f97 --- /dev/null +++ b/src/site/markdown/data/upgrade.md @@ -0,0 +1,8 @@ +Database Upgrades +================= +If using an external database server, such as MySQL, a DBA must manually perform +the database upgrades. Currently, a copy of the initialization and upgrade scripts +for MySQL can be found in the [github repository](https://github.com/jeremylong/DependencyCheck/tree/master/dependency-check-core/src/main/resources/data). + +If you want to use an external database other then MySQL please open an issue in our [issue tracker](https://github.com/jeremylong/DependencyCheck/issues) +as a dialect properties file will need to be created. \ No newline at end of file