From 38499898aa0e4fc4058eed23b1cdd285ab7a094e Mon Sep 17 00:00:00 2001 From: Ale Feltes Date: Fri, 17 Nov 2017 10:36:15 -0300 Subject: [PATCH 1/6] Added Batch insert for References and Vulnerabilities Applies batch inserts for reference and vulnerability tables, solves the slow one-by-one insert process, for Vulnerabilities with several references/vulnerabilities associated. --- .../dependencycheck/data/nvdcve/CveDB.java | 84 ++++++++++++++----- 1 file changed, 65 insertions(+), 19 deletions(-) diff --git a/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/nvdcve/CveDB.java b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/nvdcve/CveDB.java index f67ffb49d..9d024dbff 100644 --- a/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/nvdcve/CveDB.java +++ b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/nvdcve/CveDB.java @@ -25,6 +25,7 @@ import java.sql.ResultSet; import java.sql.SQLException; import java.util.ArrayList; import java.util.Collections; +import java.util.Date; import java.util.EnumMap; import java.util.HashMap; import java.util.HashSet; @@ -70,6 +71,7 @@ public final class CveDB implements AutoCloseable { * The logger. */ private static final Logger LOGGER = LoggerFactory.getLogger(CveDB.class); + public static final int MAX_BATCH_SIZE = 1000; /** * The database connection factory. */ @@ -730,36 +732,51 @@ public final class CveDB implements AutoCloseable { } } - final PreparedStatement insertReference = getPreparedStatement(INSERT_REFERENCE); + PreparedStatement insertReference = getPreparedStatement(INSERT_REFERENCE); + int countReferences = 0; for (Reference r : vuln.getReferences()) { insertReference.setInt(1, vulnerabilityId); insertReference.setString(2, r.getName()); insertReference.setString(3, r.getUrl()); insertReference.setString(4, r.getSource()); - insertReference.execute(); + insertReference.addBatch(); + countReferences++; + if (countReferences % MAX_BATCH_SIZE == 0) { + insertReference.executeBatch(); + insertReference = getPreparedStatement(INSERT_REFERENCE); + LOGGER.info(getLogForBatchInserts(countReferences, "Completed %s batch inserts to references table: %s")); + countReferences = 0; + } else if (countReferences == vuln.getReferences().size()) { + if (LOGGER.isTraceEnabled()) { + LOGGER.trace(getLogForBatchInserts(countReferences, "Completed %s batch inserts to reference table: %s")); + } + insertReference.executeBatch(); + countReferences = 0; + } } - final PreparedStatement insertSoftware = getPreparedStatement(INSERT_SOFTWARE); - for (VulnerableSoftware s : vuln.getVulnerableSoftware()) { + PreparedStatement insertSoftware = getPreparedStatement(INSERT_SOFTWARE); + int countSoftware = 0; + for (VulnerableSoftware vulnerableSoftware : vuln.getVulnerableSoftware()) { int cpeProductId = 0; final PreparedStatement selectCpeId = getPreparedStatement(SELECT_CPE_ID); - selectCpeId.setString(1, s.getName()); + selectCpeId.setString(1, vulnerableSoftware.getName()); try { rs = selectCpeId.executeQuery(); if (rs.next()) { cpeProductId = rs.getInt(1); } } catch (SQLException ex) { - throw new DatabaseException("Unable to get primary key for new cpe: " + s.getName(), ex); + throw new DatabaseException("Unable to get primary key for new cpe: " + vulnerableSoftware.getName(), ex); } finally { DBUtils.closeResultSet(rs); } if (cpeProductId == 0) { final PreparedStatement insertCpe = getPreparedStatement(INSERT_CPE); - insertCpe.setString(1, s.getName()); - insertCpe.setString(2, s.getVendor()); - insertCpe.setString(3, s.getProduct()); + insertCpe.setString(1, vulnerableSoftware.getName()); + insertCpe.setString(2, vulnerableSoftware.getVendor()); + insertCpe.setString(3, vulnerableSoftware.getProduct()); insertCpe.executeUpdate(); cpeProductId = DBUtils.getGeneratedKey(insertCpe); } @@ -770,20 +787,24 @@ public final class CveDB implements AutoCloseable { insertSoftware.setInt(1, vulnerabilityId); insertSoftware.setInt(2, cpeProductId); - if (s.getPreviousVersion() == null) { + if (vulnerableSoftware.getPreviousVersion() == null) { insertSoftware.setNull(3, java.sql.Types.VARCHAR); } else { - insertSoftware.setString(3, s.getPreviousVersion()); + insertSoftware.setString(3, vulnerableSoftware.getPreviousVersion()); } - try { - insertSoftware.execute(); - } catch (SQLException ex) { - if (ex.getMessage().contains("Duplicate entry")) { - final String msg = String.format("Duplicate software key identified in '%s:%s'", vuln.getName(), s.getName()); - LOGGER.info(msg, ex); - } else { - throw ex; + insertSoftware.addBatch(); + countSoftware++; + if (countSoftware % MAX_BATCH_SIZE == 0) { + executeBatch(vuln, vulnerableSoftware, insertSoftware); + insertSoftware = getPreparedStatement(INSERT_SOFTWARE); + LOGGER.info(getLogForBatchInserts(countSoftware, "Completed %s batch inserts software table: %s")); + countSoftware = 0; + } else if (countSoftware == vuln.getVulnerableSoftware().size()) { + if (LOGGER.isTraceEnabled()) { + LOGGER.trace(getLogForBatchInserts(countSoftware, "Completed %s batch inserts software table: %s")); + countReferences = 0; } + executeBatch(vuln, vulnerableSoftware, insertSoftware); } } @@ -796,6 +817,31 @@ public final class CveDB implements AutoCloseable { } } + private String getLogForBatchInserts(int pCountReferences, String pFormat) { + return String.format(pFormat, pCountReferences, new Date()); + } + + /** + * Executes batch inserts of vulnerabilities when MAX_BATCH_SIZE is reached + * + * @param pVulnerability + * @param pVulnerableSoftware + * @param pInsertSoftware + * @throws SQLException + */ + private void executeBatch(Vulnerability pVulnerability, VulnerableSoftware pVulnerableSoftware, PreparedStatement pInsertSoftware) throws SQLException { + try { + pInsertSoftware.executeBatch(); + } catch (SQLException ex) { + if (ex.getMessage().contains("Duplicate entry")) { + final String msg = String.format("Duplicate software key identified in '%s:%s'", pVulnerability.getName(), pVulnerableSoftware.getName()); + LOGGER.info(msg, ex); + } else { + throw ex; + } + } + } + /** * Checks to see if data exists so that analysis can be performed. * From 334829604fcb757e529b279d37e0f0e0429c85ad Mon Sep 17 00:00:00 2001 From: Ale Feltes Date: Thu, 30 Nov 2017 14:08:45 -0300 Subject: [PATCH 2/6] Adds configurable batch insert for References and Vulnerabilities Applies batch inserts for reference and vulnerability tables, solves slow one-by-one insert process, for Vulnerabilities with several references/vulnerabilities associated. Feature is configurable through properties: database.batchinsert.enabled and database.batchinsert.maxsize. --- .../dependencycheck/data/nvdcve/CveDB.java | 181 ++++++++++-------- .../main/resources/dependencycheck.properties | 6 +- .../owasp/dependencycheck/utils/Settings.java | 10 + 3 files changed, 117 insertions(+), 80 deletions(-) diff --git a/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/nvdcve/CveDB.java b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/nvdcve/CveDB.java index 9d024dbff..aa31b1b6c 100644 --- a/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/nvdcve/CveDB.java +++ b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/nvdcve/CveDB.java @@ -17,45 +17,31 @@ */ package org.owasp.dependencycheck.data.nvdcve; +import org.apache.commons.collections.map.ReferenceMap; +import org.owasp.dependencycheck.data.cwe.CweDB; +import org.owasp.dependencycheck.dependency.Reference; +import org.owasp.dependencycheck.dependency.Vulnerability; +import org.owasp.dependencycheck.dependency.VulnerableSoftware; +import org.owasp.dependencycheck.utils.*; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.annotation.concurrent.ThreadSafe; import java.io.IOException; import java.io.UnsupportedEncodingException; import java.sql.Connection; import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.SQLException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.Date; -import java.util.EnumMap; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Locale; -import java.util.Map; +import java.util.*; import java.util.Map.Entry; -import java.util.MissingResourceException; -import java.util.Properties; -import java.util.ResourceBundle; -import java.util.Set; -import javax.annotation.concurrent.ThreadSafe; -import org.apache.commons.collections.map.ReferenceMap; -import org.owasp.dependencycheck.data.cwe.CweDB; -import org.owasp.dependencycheck.dependency.Reference; -import org.owasp.dependencycheck.dependency.Vulnerability; -import org.owasp.dependencycheck.dependency.VulnerableSoftware; -import org.owasp.dependencycheck.utils.DBUtils; -import org.owasp.dependencycheck.utils.DependencyVersion; -import org.owasp.dependencycheck.utils.DependencyVersionUtil; -import org.owasp.dependencycheck.utils.Pair; -import org.owasp.dependencycheck.utils.Settings; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -//CSOFF: AvoidStarImport -import static org.owasp.dependencycheck.data.nvdcve.CveDB.PreparedStatementCveDb.*; -//CSON: AvoidStarImport import static org.apache.commons.collections.map.AbstractReferenceMap.HARD; import static org.apache.commons.collections.map.AbstractReferenceMap.SOFT; +import static org.owasp.dependencycheck.data.nvdcve.CveDB.PreparedStatementCveDb.*; + +//CSOFF: AvoidStarImport +//CSON: AvoidStarImport /** * The database holding information about the NVD CVE data. This class is safe @@ -71,7 +57,7 @@ public final class CveDB implements AutoCloseable { * The logger. */ private static final Logger LOGGER = LoggerFactory.getLogger(CveDB.class); - public static final int MAX_BATCH_SIZE = 1000; + /** * The database connection factory. */ @@ -205,7 +191,7 @@ public final class CveDB implements AutoCloseable { * * @param settings the configured settings * @throws DatabaseException thrown if there is an exception opening the - * database. + * database. */ public CveDB(Settings settings) throws DatabaseException { this.settings = settings; @@ -235,7 +221,7 @@ public final class CveDB implements AutoCloseable { * create a new one. * * @throws DatabaseException thrown if there is an error opening the - * database connection + * database connection */ private synchronized void open() throws DatabaseException { try { @@ -300,7 +286,7 @@ public final class CveDB implements AutoCloseable { * Prepares all statements to be used. * * @throws DatabaseException thrown if there is an error preparing the - * statements + * statements */ private void prepareStatements() throws DatabaseException { for (PreparedStatementCveDb key : values()) { @@ -338,7 +324,7 @@ public final class CveDB implements AutoCloseable { * Returns the specified prepared statement. * * @param key the prepared statement from {@link PreparedStatementCveDb} to - * return + * return * @return the prepared statement * @throws SQLException thrown if a SQL Exception occurs */ @@ -401,9 +387,9 @@ public final class CveDB implements AutoCloseable { * given vendor and product combination. The returned list will include all * versions of the product that are registered in the NVD CVE data. * - * @param vendor the identified vendor name of the dependency being analyzed + * @param vendor the identified vendor name of the dependency being analyzed * @param product the identified name of the product of the dependency being - * analyzed + * analyzed * @return a set of vulnerable software */ public synchronized Set getCPEs(String vendor, String product) { @@ -434,7 +420,7 @@ public final class CveDB implements AutoCloseable { * * @return the entire list of vendor/product combinations * @throws DatabaseException thrown when there is an error retrieving the - * data from the DB + * data from the DB */ public synchronized Set> getVendorProductList() throws DatabaseException { final Set> data = new HashSet<>(); @@ -480,7 +466,7 @@ public final class CveDB implements AutoCloseable { /** * Saves a property to the database. * - * @param key the property key + * @param key the property key * @param value the property value */ public synchronized void saveProperty(String key, String value) { @@ -514,7 +500,7 @@ public final class CveDB implements AutoCloseable { * is not the optimal cache eviction strategy, this is good enough for * typical usage (update DB and then only read) and it is easier to maintain * the code. - * + *

* It should be also called when DB is closed. */ private synchronized void clearCache() { @@ -672,6 +658,7 @@ public final class CveDB implements AutoCloseable { ResultSet rs = null; try { int vulnerabilityId = 0; + long countVulnerabilities = 0; final PreparedStatement selectVulnerabilityId = getPreparedStatement(SELECT_VULNERABILITY_ID); selectVulnerabilityId.setString(1, vuln.getName()); rs = selectVulnerabilityId.executeQuery(); @@ -686,6 +673,7 @@ public final class CveDB implements AutoCloseable { deleteSoftware.setInt(1, vulnerabilityId); deleteSoftware.execute(); } + DBUtils.closeResultSet(rs); if (vulnerabilityId != 0) { @@ -739,19 +727,23 @@ public final class CveDB implements AutoCloseable { insertReference.setString(2, r.getName()); insertReference.setString(3, r.getUrl()); insertReference.setString(4, r.getSource()); - insertReference.addBatch(); - countReferences++; - if (countReferences % MAX_BATCH_SIZE == 0) { - insertReference.executeBatch(); - insertReference = getPreparedStatement(INSERT_REFERENCE); - LOGGER.info(getLogForBatchInserts(countReferences, "Completed %s batch inserts to references table: %s")); - countReferences = 0; - } else if (countReferences == vuln.getReferences().size()) { - if (LOGGER.isTraceEnabled()) { - LOGGER.trace(getLogForBatchInserts(countReferences, "Completed %s batch inserts to reference table: %s")); + if(isBatchInsertEnabled()) { + insertReference.addBatch(); + countReferences++; + if (countReferences % getBatchSize() == 0) { + insertReference.executeBatch(); + insertReference = getPreparedStatement(INSERT_REFERENCE); + LOGGER.info(getLogForBatchInserts(countReferences, "Completed %s batch inserts to references table: %s")); + countReferences = 0; + } else if (countReferences == vuln.getReferences().size()) { + if (LOGGER.isTraceEnabled()) { + LOGGER.trace(getLogForBatchInserts(countReferences, "Completed %s batch inserts to reference table: %s")); + } + insertReference.executeBatch(); + countReferences = 0; } - insertReference.executeBatch(); - countReferences = 0; + } else { + insertReference.execute(); } } @@ -792,21 +784,33 @@ public final class CveDB implements AutoCloseable { } else { insertSoftware.setString(3, vulnerableSoftware.getPreviousVersion()); } - insertSoftware.addBatch(); - countSoftware++; - if (countSoftware % MAX_BATCH_SIZE == 0) { - executeBatch(vuln, vulnerableSoftware, insertSoftware); - insertSoftware = getPreparedStatement(INSERT_SOFTWARE); - LOGGER.info(getLogForBatchInserts(countSoftware, "Completed %s batch inserts software table: %s")); - countSoftware = 0; - } else if (countSoftware == vuln.getVulnerableSoftware().size()) { - if (LOGGER.isTraceEnabled()) { - LOGGER.trace(getLogForBatchInserts(countSoftware, "Completed %s batch inserts software table: %s")); - countReferences = 0; + if(isBatchInsertEnabled()) { + insertSoftware.addBatch(); + countSoftware++; + if (countSoftware % getBatchSize() == 0) { + executeBatch(vuln, vulnerableSoftware, insertSoftware); + insertSoftware = getPreparedStatement(INSERT_SOFTWARE); + LOGGER.info(getLogForBatchInserts(countSoftware, "Completed %s batch inserts software table: %s")); + countSoftware = 0; + } else if (countSoftware == vuln.getVulnerableSoftware().size()) { + if (LOGGER.isTraceEnabled()) { + LOGGER.trace(getLogForBatchInserts(countSoftware, "Completed %s batch inserts software table: %s")); + countReferences = 0; + } + executeBatch(vuln, vulnerableSoftware, insertSoftware); + } + } else { + try { + insertSoftware.execute(); + } catch (SQLException ex) { + if (ex.getMessage().contains("Duplicate entry")) { + final String msg = String.format("Duplicate software key identified in '%s:%s'", vuln.getName(), vuln.getName()); + LOGGER.info(msg, ex); + } else { + throw ex; + } } - executeBatch(vuln, vulnerableSoftware, insertSoftware); } - } } catch (SQLException ex) { final String msg = String.format("Error updating '%s'", vuln.getName()); @@ -817,12 +821,33 @@ public final class CveDB implements AutoCloseable { } } + private int getBatchSize() { + int max; + try { + max = settings.getInt(Settings.KEYS.MAX_BATCH_SIZE); + } catch (InvalidSettingException pE) { + max = 1000; + } + return max; + } + + private boolean isBatchInsertEnabled() { + boolean batch = false; + try { + batch = settings.getBoolean(Settings.KEYS.ENABLE_BATCH_UPDATES); + } catch (InvalidSettingException pE) { + //If there's no configuration, default is to not perform batch inserts + batch = false; + } + return batch; + } + private String getLogForBatchInserts(int pCountReferences, String pFormat) { return String.format(pFormat, pCountReferences, new Date()); } /** - * Executes batch inserts of vulnerabilities when MAX_BATCH_SIZE is reached + * Executes batch inserts of vulnerabilities when property database.batchinsert.maxsize is reached * * @param pVulnerability * @param pVulnerableSoftware @@ -863,9 +888,9 @@ public final class CveDB implements AutoCloseable { dd = settings.getString(Settings.KEYS.DATA_DIRECTORY); } LOGGER.error("Unable to access the local database.\n\nEnsure that '{}' is a writable directory. " - + "If the problem persist try deleting the files in '{}' and running {} again. If the problem continues, please " - + "create a log file (see documentation at http://jeremylong.github.io/DependencyCheck/) and open a ticket at " - + "https://github.com/jeremylong/DependencyCheck/issues and include the log file.\n\n", + + "If the problem persist try deleting the files in '{}' and running {} again. If the problem continues, please " + + "create a log file (see documentation at http://jeremylong.github.io/DependencyCheck/) and open a ticket at " + + "https://github.com/jeremylong/DependencyCheck/issues and include the log file.\n\n", dd, dd, settings.getString(Settings.KEYS.APPLICATION_NAME)); LOGGER.debug("", ex); } finally { @@ -898,16 +923,16 @@ public final class CveDB implements AutoCloseable { * previous version argument indicates that all previous versions are * affected. * - * @param vendor the vendor of the dependency being analyzed - * @param product the product name of the dependency being analyzed + * @param vendor the vendor of the dependency being analyzed + * @param product the product name of the dependency being analyzed * @param vulnerableSoftware a map of the vulnerable software with a boolean - * indicating if all previous versions are affected - * @param identifiedVersion the identified version of the dependency being - * analyzed + * indicating if all previous versions are affected + * @param identifiedVersion the identified version of the dependency being + * analyzed * @return true if the identified version is affected, otherwise false */ protected Entry getMatchingSoftware(Map vulnerableSoftware, String vendor, String product, - DependencyVersion identifiedVersion) { + DependencyVersion identifiedVersion) { final boolean isVersionTwoADifferentProduct = "apache".equals(vendor) && "struts".equals(product); @@ -1011,7 +1036,7 @@ public final class CveDB implements AutoCloseable { /** * This method is only referenced in unused code. - * + *

* Deletes unused dictionary entries from the database. */ public synchronized void deleteUnusedCpe() { @@ -1030,11 +1055,11 @@ public final class CveDB implements AutoCloseable { /** * This method is only referenced in unused code and will likely break on * MySQL if ever used due to the MERGE statement. - * + *

* Merges CPE entries into the database. * - * @param cpe the CPE identifier - * @param vendor the CPE vendor + * @param cpe the CPE identifier + * @param vendor the CPE vendor * @param product the CPE product */ public synchronized void addCpe(String cpe, String vendor, String product) { diff --git a/dependency-check-core/src/main/resources/dependencycheck.properties b/dependency-check-core/src/main/resources/dependencycheck.properties index 0a6053d36..33df1bfd5 100644 --- a/dependency-check-core/src/main/resources/dependencycheck.properties +++ b/dependency-check-core/src/main/resources/dependencycheck.properties @@ -50,7 +50,7 @@ cve.url.modified.validfordays=7 # the number of hours to wait before checking if updates are available from the NVD. cve.check.validforhours=4 #first year to pull data from the URLs below -cve.startyear=2002 +cve.startyear=2017 # the path to the modified nvd cve xml file. cve.url-1.2.modified=https://nvd.nist.gov/download/nvdcve-Modified.xml.gz #cve.url-1.2.modified=http://nvd.nist.gov/download/nvdcve-modified.xml @@ -126,4 +126,6 @@ analyzer.nvdcve.enabled=true analyzer.vulnerabilitysuppression.enabled=true updater.nvdcve.enabled=true updater.versioncheck.enabled=true -analyzer.versionfilter.enabled=true \ No newline at end of file +analyzer.versionfilter.enabled=true +database.batchinsert.enabled=true +database.batchinsert.maxsize=1000 diff --git a/dependency-check-utils/src/main/java/org/owasp/dependencycheck/utils/Settings.java b/dependency-check-utils/src/main/java/org/owasp/dependencycheck/utils/Settings.java index dff450027..819680a3a 100644 --- a/dependency-check-utils/src/main/java/org/owasp/dependencycheck/utils/Settings.java +++ b/dependency-check-utils/src/main/java/org/owasp/dependencycheck/utils/Settings.java @@ -443,6 +443,16 @@ public final class Settings { */ public static final String UPDATE_VERSION_CHECK_ENABLED = "updater.versioncheck.enabled"; + /** + * + * Adds capabilities to batch insert. Tested on PostgreSQL and H2. + */ + public static final String ENABLE_BATCH_UPDATES = "database.batchinsert.enabled"; + /** + * Size of database batch inserts + */ + public static final String MAX_BATCH_SIZE = "database.batchinsert.maxsize"; + /** * private constructor because this is a "utility" class containing * constants From 482f8daaf3503a30ba9d158c03709a5b4885b973 Mon Sep 17 00:00:00 2001 From: Ale Feltes Date: Thu, 30 Nov 2017 15:18:32 -0300 Subject: [PATCH 3/6] Restore CVE start year cve.startyear value is 2002. --- .../src/main/resources/dependencycheck.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dependency-check-core/src/main/resources/dependencycheck.properties b/dependency-check-core/src/main/resources/dependencycheck.properties index 33df1bfd5..40e548e76 100644 --- a/dependency-check-core/src/main/resources/dependencycheck.properties +++ b/dependency-check-core/src/main/resources/dependencycheck.properties @@ -50,7 +50,7 @@ cve.url.modified.validfordays=7 # the number of hours to wait before checking if updates are available from the NVD. cve.check.validforhours=4 #first year to pull data from the URLs below -cve.startyear=2017 +cve.startyear=2002 # the path to the modified nvd cve xml file. cve.url-1.2.modified=https://nvd.nist.gov/download/nvdcve-Modified.xml.gz #cve.url-1.2.modified=http://nvd.nist.gov/download/nvdcve-modified.xml From b46d13fc186c8e22947c06e4ecb7393c8bf41263 Mon Sep 17 00:00:00 2001 From: Ale Feltes Quenhan Date: Thu, 30 Nov 2017 16:21:39 -0300 Subject: [PATCH 4/6] Update CveDB.java Remove unused variable. --- .../main/java/org/owasp/dependencycheck/data/nvdcve/CveDB.java | 1 - 1 file changed, 1 deletion(-) diff --git a/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/nvdcve/CveDB.java b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/nvdcve/CveDB.java index aa31b1b6c..eb69ddd1b 100644 --- a/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/nvdcve/CveDB.java +++ b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/nvdcve/CveDB.java @@ -658,7 +658,6 @@ public final class CveDB implements AutoCloseable { ResultSet rs = null; try { int vulnerabilityId = 0; - long countVulnerabilities = 0; final PreparedStatement selectVulnerabilityId = getPreparedStatement(SELECT_VULNERABILITY_ID); selectVulnerabilityId.setString(1, vuln.getName()); rs = selectVulnerabilityId.executeQuery(); From 63a2874cecb4ff714256b4c412fb0125bfaad5a9 Mon Sep 17 00:00:00 2001 From: Jeremy Long Date: Sat, 9 Dec 2017 07:51:06 -0500 Subject: [PATCH 5/6] updated logging levels to reduce spamming the console --- .../java/org/owasp/dependencycheck/data/nvdcve/CveDB.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/nvdcve/CveDB.java b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/nvdcve/CveDB.java index eb69ddd1b..f7b9b9e49 100644 --- a/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/nvdcve/CveDB.java +++ b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/nvdcve/CveDB.java @@ -732,7 +732,7 @@ public final class CveDB implements AutoCloseable { if (countReferences % getBatchSize() == 0) { insertReference.executeBatch(); insertReference = getPreparedStatement(INSERT_REFERENCE); - LOGGER.info(getLogForBatchInserts(countReferences, "Completed %s batch inserts to references table: %s")); + LOGGER.trace(getLogForBatchInserts(countReferences, "Completed %s batch inserts to references table: %s")); countReferences = 0; } else if (countReferences == vuln.getReferences().size()) { if (LOGGER.isTraceEnabled()) { @@ -789,7 +789,7 @@ public final class CveDB implements AutoCloseable { if (countSoftware % getBatchSize() == 0) { executeBatch(vuln, vulnerableSoftware, insertSoftware); insertSoftware = getPreparedStatement(INSERT_SOFTWARE); - LOGGER.info(getLogForBatchInserts(countSoftware, "Completed %s batch inserts software table: %s")); + LOGGER.trace(getLogForBatchInserts(countSoftware, "Completed %s batch inserts software table: %s")); countSoftware = 0; } else if (countSoftware == vuln.getVulnerableSoftware().size()) { if (LOGGER.isTraceEnabled()) { From 1e72c49eb29050936dec4207f9ce7b74687fc4dd Mon Sep 17 00:00:00 2001 From: Jeremy Long Date: Sat, 9 Dec 2017 08:14:21 -0500 Subject: [PATCH 6/6] javadoc update --- .../dependencycheck/data/nvdcve/CveDB.java | 49 ++++++++++--------- 1 file changed, 26 insertions(+), 23 deletions(-) diff --git a/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/nvdcve/CveDB.java b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/nvdcve/CveDB.java index f7b9b9e49..2793db045 100644 --- a/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/nvdcve/CveDB.java +++ b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/nvdcve/CveDB.java @@ -42,7 +42,6 @@ import static org.owasp.dependencycheck.data.nvdcve.CveDB.PreparedStatementCveDb //CSOFF: AvoidStarImport //CSON: AvoidStarImport - /** * The database holding information about the NVD CVE data. This class is safe * to be accessed from multiple threads in parallel, however internally only one @@ -191,7 +190,7 @@ public final class CveDB implements AutoCloseable { * * @param settings the configured settings * @throws DatabaseException thrown if there is an exception opening the - * database. + * database. */ public CveDB(Settings settings) throws DatabaseException { this.settings = settings; @@ -221,7 +220,7 @@ public final class CveDB implements AutoCloseable { * create a new one. * * @throws DatabaseException thrown if there is an error opening the - * database connection + * database connection */ private synchronized void open() throws DatabaseException { try { @@ -286,7 +285,7 @@ public final class CveDB implements AutoCloseable { * Prepares all statements to be used. * * @throws DatabaseException thrown if there is an error preparing the - * statements + * statements */ private void prepareStatements() throws DatabaseException { for (PreparedStatementCveDb key : values()) { @@ -324,7 +323,7 @@ public final class CveDB implements AutoCloseable { * Returns the specified prepared statement. * * @param key the prepared statement from {@link PreparedStatementCveDb} to - * return + * return * @return the prepared statement * @throws SQLException thrown if a SQL Exception occurs */ @@ -387,9 +386,9 @@ public final class CveDB implements AutoCloseable { * given vendor and product combination. The returned list will include all * versions of the product that are registered in the NVD CVE data. * - * @param vendor the identified vendor name of the dependency being analyzed + * @param vendor the identified vendor name of the dependency being analyzed * @param product the identified name of the product of the dependency being - * analyzed + * analyzed * @return a set of vulnerable software */ public synchronized Set getCPEs(String vendor, String product) { @@ -420,7 +419,7 @@ public final class CveDB implements AutoCloseable { * * @return the entire list of vendor/product combinations * @throws DatabaseException thrown when there is an error retrieving the - * data from the DB + * data from the DB */ public synchronized Set> getVendorProductList() throws DatabaseException { final Set> data = new HashSet<>(); @@ -466,7 +465,7 @@ public final class CveDB implements AutoCloseable { /** * Saves a property to the database. * - * @param key the property key + * @param key the property key * @param value the property value */ public synchronized void saveProperty(String key, String value) { @@ -502,6 +501,7 @@ public final class CveDB implements AutoCloseable { * the code. *

* It should be also called when DB is closed. + *

*/ private synchronized void clearCache() { vulnerabilitiesForCpeCache.clear(); @@ -726,7 +726,7 @@ public final class CveDB implements AutoCloseable { insertReference.setString(2, r.getName()); insertReference.setString(3, r.getUrl()); insertReference.setString(4, r.getSource()); - if(isBatchInsertEnabled()) { + if (isBatchInsertEnabled()) { insertReference.addBatch(); countReferences++; if (countReferences % getBatchSize() == 0) { @@ -783,7 +783,7 @@ public final class CveDB implements AutoCloseable { } else { insertSoftware.setString(3, vulnerableSoftware.getPreviousVersion()); } - if(isBatchInsertEnabled()) { + if (isBatchInsertEnabled()) { insertSoftware.addBatch(); countSoftware++; if (countSoftware % getBatchSize() == 0) { @@ -846,7 +846,8 @@ public final class CveDB implements AutoCloseable { } /** - * Executes batch inserts of vulnerabilities when property database.batchinsert.maxsize is reached + * Executes batch inserts of vulnerabilities when property + * database.batchinsert.maxsize is reached * * @param pVulnerability * @param pVulnerableSoftware @@ -887,9 +888,9 @@ public final class CveDB implements AutoCloseable { dd = settings.getString(Settings.KEYS.DATA_DIRECTORY); } LOGGER.error("Unable to access the local database.\n\nEnsure that '{}' is a writable directory. " - + "If the problem persist try deleting the files in '{}' and running {} again. If the problem continues, please " - + "create a log file (see documentation at http://jeremylong.github.io/DependencyCheck/) and open a ticket at " - + "https://github.com/jeremylong/DependencyCheck/issues and include the log file.\n\n", + + "If the problem persist try deleting the files in '{}' and running {} again. If the problem continues, please " + + "create a log file (see documentation at http://jeremylong.github.io/DependencyCheck/) and open a ticket at " + + "https://github.com/jeremylong/DependencyCheck/issues and include the log file.\n\n", dd, dd, settings.getString(Settings.KEYS.APPLICATION_NAME)); LOGGER.debug("", ex); } finally { @@ -922,16 +923,16 @@ public final class CveDB implements AutoCloseable { * previous version argument indicates that all previous versions are * affected. * - * @param vendor the vendor of the dependency being analyzed - * @param product the product name of the dependency being analyzed + * @param vendor the vendor of the dependency being analyzed + * @param product the product name of the dependency being analyzed * @param vulnerableSoftware a map of the vulnerable software with a boolean - * indicating if all previous versions are affected - * @param identifiedVersion the identified version of the dependency being - * analyzed + * indicating if all previous versions are affected + * @param identifiedVersion the identified version of the dependency being + * analyzed * @return true if the identified version is affected, otherwise false */ protected Entry getMatchingSoftware(Map vulnerableSoftware, String vendor, String product, - DependencyVersion identifiedVersion) { + DependencyVersion identifiedVersion) { final boolean isVersionTwoADifferentProduct = "apache".equals(vendor) && "struts".equals(product); @@ -1037,6 +1038,7 @@ public final class CveDB implements AutoCloseable { * This method is only referenced in unused code. *

* Deletes unused dictionary entries from the database. + *

*/ public synchronized void deleteUnusedCpe() { clearCache(); @@ -1056,9 +1058,10 @@ public final class CveDB implements AutoCloseable { * MySQL if ever used due to the MERGE statement. *

* Merges CPE entries into the database. + *

* - * @param cpe the CPE identifier - * @param vendor the CPE vendor + * @param cpe the CPE identifier + * @param vendor the CPE vendor * @param product the CPE product */ public synchronized void addCpe(String cpe, String vendor, String product) {