* It should be also called when DB is closed. + *
*/ private synchronized void clearCache() { vulnerabilitiesForCpeCache.clear(); @@ -684,6 +672,7 @@ public final class CveDB implements AutoCloseable { deleteSoftware.setInt(1, vulnerabilityId); deleteSoftware.execute(); } + DBUtils.closeResultSet(rs); if (vulnerabilityId != 0) { @@ -730,36 +719,55 @@ public final class CveDB implements AutoCloseable { } } - final PreparedStatement insertReference = getPreparedStatement(INSERT_REFERENCE); + PreparedStatement insertReference = getPreparedStatement(INSERT_REFERENCE); + int countReferences = 0; for (Reference r : vuln.getReferences()) { insertReference.setInt(1, vulnerabilityId); insertReference.setString(2, r.getName()); insertReference.setString(3, r.getUrl()); insertReference.setString(4, r.getSource()); - insertReference.execute(); + if (isBatchInsertEnabled()) { + insertReference.addBatch(); + countReferences++; + if (countReferences % getBatchSize() == 0) { + insertReference.executeBatch(); + insertReference = getPreparedStatement(INSERT_REFERENCE); + LOGGER.trace(getLogForBatchInserts(countReferences, "Completed %s batch inserts to references table: %s")); + countReferences = 0; + } else if (countReferences == vuln.getReferences().size()) { + if (LOGGER.isTraceEnabled()) { + LOGGER.trace(getLogForBatchInserts(countReferences, "Completed %s batch inserts to reference table: %s")); + } + insertReference.executeBatch(); + countReferences = 0; + } + } else { + insertReference.execute(); + } } - final PreparedStatement insertSoftware = getPreparedStatement(INSERT_SOFTWARE); - for (VulnerableSoftware s : vuln.getVulnerableSoftware()) { + PreparedStatement insertSoftware = getPreparedStatement(INSERT_SOFTWARE); + int countSoftware = 0; + for (VulnerableSoftware vulnerableSoftware : vuln.getVulnerableSoftware()) { int cpeProductId = 0; final PreparedStatement selectCpeId = getPreparedStatement(SELECT_CPE_ID); - selectCpeId.setString(1, s.getName()); + selectCpeId.setString(1, vulnerableSoftware.getName()); try { rs = selectCpeId.executeQuery(); if (rs.next()) { cpeProductId = rs.getInt(1); } } catch (SQLException ex) { - throw new DatabaseException("Unable to get primary key for new cpe: " + s.getName(), ex); + throw new DatabaseException("Unable to get primary key for new cpe: " + vulnerableSoftware.getName(), ex); } finally { DBUtils.closeResultSet(rs); } if (cpeProductId == 0) { final PreparedStatement insertCpe = getPreparedStatement(INSERT_CPE); - insertCpe.setString(1, s.getName()); - insertCpe.setString(2, s.getVendor()); - insertCpe.setString(3, s.getProduct()); + insertCpe.setString(1, vulnerableSoftware.getName()); + insertCpe.setString(2, vulnerableSoftware.getVendor()); + insertCpe.setString(3, vulnerableSoftware.getProduct()); insertCpe.executeUpdate(); cpeProductId = DBUtils.getGeneratedKey(insertCpe); } @@ -770,22 +778,38 @@ public final class CveDB implements AutoCloseable { insertSoftware.setInt(1, vulnerabilityId); insertSoftware.setInt(2, cpeProductId); - if (s.getPreviousVersion() == null) { + if (vulnerableSoftware.getPreviousVersion() == null) { insertSoftware.setNull(3, java.sql.Types.VARCHAR); } else { - insertSoftware.setString(3, s.getPreviousVersion()); + insertSoftware.setString(3, vulnerableSoftware.getPreviousVersion()); } - try { - insertSoftware.execute(); - } catch (SQLException ex) { - if (ex.getMessage().contains("Duplicate entry")) { - final String msg = String.format("Duplicate software key identified in '%s:%s'", vuln.getName(), s.getName()); - LOGGER.info(msg, ex); - } else { - throw ex; + if (isBatchInsertEnabled()) { + insertSoftware.addBatch(); + countSoftware++; + if (countSoftware % getBatchSize() == 0) { + executeBatch(vuln, vulnerableSoftware, insertSoftware); + insertSoftware = getPreparedStatement(INSERT_SOFTWARE); + LOGGER.trace(getLogForBatchInserts(countSoftware, "Completed %s batch inserts software table: %s")); + countSoftware = 0; + } else if (countSoftware == vuln.getVulnerableSoftware().size()) { + if (LOGGER.isTraceEnabled()) { + LOGGER.trace(getLogForBatchInserts(countSoftware, "Completed %s batch inserts software table: %s")); + countReferences = 0; + } + executeBatch(vuln, vulnerableSoftware, insertSoftware); + } + } else { + try { + insertSoftware.execute(); + } catch (SQLException ex) { + if (ex.getMessage().contains("Duplicate entry")) { + final String msg = String.format("Duplicate software key identified in '%s:%s'", vuln.getName(), vuln.getName()); + LOGGER.info(msg, ex); + } else { + throw ex; + } } } - } } catch (SQLException ex) { final String msg = String.format("Error updating '%s'", vuln.getName()); @@ -796,6 +820,53 @@ public final class CveDB implements AutoCloseable { } } + private int getBatchSize() { + int max; + try { + max = settings.getInt(Settings.KEYS.MAX_BATCH_SIZE); + } catch (InvalidSettingException pE) { + max = 1000; + } + return max; + } + + private boolean isBatchInsertEnabled() { + boolean batch = false; + try { + batch = settings.getBoolean(Settings.KEYS.ENABLE_BATCH_UPDATES); + } catch (InvalidSettingException pE) { + //If there's no configuration, default is to not perform batch inserts + batch = false; + } + return batch; + } + + private String getLogForBatchInserts(int pCountReferences, String pFormat) { + return String.format(pFormat, pCountReferences, new Date()); + } + + /** + * Executes batch inserts of vulnerabilities when property + * database.batchinsert.maxsize is reached + * + * @param pVulnerability + * @param pVulnerableSoftware + * @param pInsertSoftware + * @throws SQLException + */ + private void executeBatch(Vulnerability pVulnerability, VulnerableSoftware pVulnerableSoftware, PreparedStatement pInsertSoftware) throws SQLException { + try { + pInsertSoftware.executeBatch(); + } catch (SQLException ex) { + if (ex.getMessage().contains("Duplicate entry")) { + final String msg = String.format("Duplicate software key identified in '%s:%s'", pVulnerability.getName(), pVulnerableSoftware.getName()); + LOGGER.info(msg, ex); + } else { + throw ex; + } + } + } + /** * Checks to see if data exists so that analysis can be performed. * @@ -965,8 +1036,9 @@ public final class CveDB implements AutoCloseable { /** * This method is only referenced in unused code. - * + ** Deletes unused dictionary entries from the database. + *
*/ public synchronized void deleteUnusedCpe() { clearCache(); @@ -984,8 +1056,9 @@ public final class CveDB implements AutoCloseable { /** * This method is only referenced in unused code and will likely break on * MySQL if ever used due to the MERGE statement. - * + ** Merges CPE entries into the database. + *
* * @param cpe the CPE identifier * @param vendor the CPE vendor diff --git a/dependency-check-core/src/main/resources/dependencycheck.properties b/dependency-check-core/src/main/resources/dependencycheck.properties index 259c724d4..761d09984 100644 --- a/dependency-check-core/src/main/resources/dependencycheck.properties +++ b/dependency-check-core/src/main/resources/dependencycheck.properties @@ -127,5 +127,7 @@ analyzer.vulnerabilitysuppression.enabled=true updater.nvdcve.enabled=true updater.versioncheck.enabled=true analyzer.versionfilter.enabled=true +ecosystem.skip.cpeanalyzer=npm +database.batchinsert.enabled=true +database.batchinsert.maxsize=1000 -ecosystem.skip.cpeanalyzer=npm \ No newline at end of file diff --git a/dependency-check-core/src/test/resources/dependencycheck.properties b/dependency-check-core/src/test/resources/dependencycheck.properties index ce2875943..eec27880c 100644 --- a/dependency-check-core/src/test/resources/dependencycheck.properties +++ b/dependency-check-core/src/test/resources/dependencycheck.properties @@ -124,4 +124,6 @@ analyzer.vulnerabilitysuppression.enabled=true updater.nvdcve.enabled=true updater.versioncheck.enabled=true -ecosystem.skip.cpeanalyzer=npm \ No newline at end of file +ecosystem.skip.cpeanalyzer=npm +database.batchinsert.enabled=true +database.batchinsert.maxsize=1000 \ No newline at end of file diff --git a/dependency-check-utils/src/main/java/org/owasp/dependencycheck/utils/Settings.java b/dependency-check-utils/src/main/java/org/owasp/dependencycheck/utils/Settings.java index a33ba86cd..2ba4f1feb 100644 --- a/dependency-check-utils/src/main/java/org/owasp/dependencycheck/utils/Settings.java +++ b/dependency-check-utils/src/main/java/org/owasp/dependencycheck/utils/Settings.java @@ -447,6 +447,16 @@ public final class Settings { */ public static final String ECOSYSTEM_SKIP_CPEANALYZER = "ecosystem.skip.cpeanalyzer"; + /** + * + * Adds capabilities to batch insert. Tested on PostgreSQL and H2. + */ + public static final String ENABLE_BATCH_UPDATES = "database.batchinsert.enabled"; + /** + * Size of database batch inserts + */ + public static final String MAX_BATCH_SIZE = "database.batchinsert.maxsize"; + /** * private constructor because this is a "utility" class containing * constants