Adds configurable batch insert for References and Vulnerabilities

Applies batch inserts for reference and vulnerability tables, solves slow one-by-one insert process, for Vulnerabilities with several references/vulnerabilities associated. Feature is configurable through properties: database.batchinsert.enabled and database.batchinsert.maxsize.
2026-01-14 15:53:36 +01:00 · 2017-11-30 14:08:45 -03:00
parent ebff547b6f
commit 334829604f
3 changed files with 117 additions and 80 deletions
--- a/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/nvdcve/CveDB.java
+++ b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/nvdcve/CveDB.java
@@ -17,45 +17,31 @@
 */
 package org.owasp.dependencycheck.data.nvdcve;

+import org.apache.commons.collections.map.ReferenceMap;
+import org.owasp.dependencycheck.data.cwe.CweDB;
+import org.owasp.dependencycheck.dependency.Reference;
+import org.owasp.dependencycheck.dependency.Vulnerability;
+import org.owasp.dependencycheck.dependency.VulnerableSoftware;
+import org.owasp.dependencycheck.utils.*;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.annotation.concurrent.ThreadSafe;
 import java.io.IOException;
 import java.io.UnsupportedEncodingException;
 import java.sql.Connection;
 import java.sql.PreparedStatement;
 import java.sql.ResultSet;
 import java.sql.SQLException;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.Date;
-import java.util.EnumMap;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Locale;
-import java.util.Map;
+import java.util.*;
 import java.util.Map.Entry;
-import java.util.MissingResourceException;
-import java.util.Properties;
-import java.util.ResourceBundle;
-import java.util.Set;
-import javax.annotation.concurrent.ThreadSafe;
-import org.apache.commons.collections.map.ReferenceMap;
-import org.owasp.dependencycheck.data.cwe.CweDB;
-import org.owasp.dependencycheck.dependency.Reference;
-import org.owasp.dependencycheck.dependency.Vulnerability;
-import org.owasp.dependencycheck.dependency.VulnerableSoftware;
-import org.owasp.dependencycheck.utils.DBUtils;
-import org.owasp.dependencycheck.utils.DependencyVersion;
-import org.owasp.dependencycheck.utils.DependencyVersionUtil;
-import org.owasp.dependencycheck.utils.Pair;
-import org.owasp.dependencycheck.utils.Settings;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;

-//CSOFF: AvoidStarImport
-import static org.owasp.dependencycheck.data.nvdcve.CveDB.PreparedStatementCveDb.*;
-//CSON: AvoidStarImport
 import static org.apache.commons.collections.map.AbstractReferenceMap.HARD;
 import static org.apache.commons.collections.map.AbstractReferenceMap.SOFT;
+import static org.owasp.dependencycheck.data.nvdcve.CveDB.PreparedStatementCveDb.*;
+
+//CSOFF: AvoidStarImport
+//CSON: AvoidStarImport

 /**
 * The database holding information about the NVD CVE data. This class is safe
@@ -71,7 +57,7 @@ public final class CveDB implements AutoCloseable {
     * The logger.
     */
    private static final Logger LOGGER = LoggerFactory.getLogger(CveDB.class);
-    public static final int MAX_BATCH_SIZE = 1000;
+
    /**
     * The database connection factory.
     */
@@ -205,7 +191,7 @@ public final class CveDB implements AutoCloseable {
     *
     * @param settings the configured settings
     * @throws DatabaseException thrown if there is an exception opening the
-     * database.
+     *                           database.
     */
    public CveDB(Settings settings) throws DatabaseException {
        this.settings = settings;
@@ -235,7 +221,7 @@ public final class CveDB implements AutoCloseable {
     * create a new one.
     *
     * @throws DatabaseException thrown if there is an error opening the
-     * database connection
+     *                           database connection
     */
    private synchronized void open() throws DatabaseException {
        try {
@@ -300,7 +286,7 @@ public final class CveDB implements AutoCloseable {
     * Prepares all statements to be used.
     *
     * @throws DatabaseException thrown if there is an error preparing the
-     * statements
+     *                           statements
     */
    private void prepareStatements() throws DatabaseException {
        for (PreparedStatementCveDb key : values()) {
@@ -338,7 +324,7 @@ public final class CveDB implements AutoCloseable {
     * Returns the specified prepared statement.
     *
     * @param key the prepared statement from {@link PreparedStatementCveDb} to
-     * return
+     *            return
     * @return the prepared statement
     * @throws SQLException thrown if a SQL Exception occurs
     */
@@ -401,9 +387,9 @@ public final class CveDB implements AutoCloseable {
     * given vendor and product combination. The returned list will include all
     * versions of the product that are registered in the NVD CVE data.
     *
-     * @param vendor the identified vendor name of the dependency being analyzed
+     * @param vendor  the identified vendor name of the dependency being analyzed
     * @param product the identified name of the product of the dependency being
-     * analyzed
+     *                analyzed
     * @return a set of vulnerable software
     */
    public synchronized Set<VulnerableSoftware> getCPEs(String vendor, String product) {
@@ -434,7 +420,7 @@ public final class CveDB implements AutoCloseable {
     *
     * @return the entire list of vendor/product combinations
     * @throws DatabaseException thrown when there is an error retrieving the
-     * data from the DB
+     *                           data from the DB
     */
    public synchronized Set<Pair<String, String>> getVendorProductList() throws DatabaseException {
        final Set<Pair<String, String>> data = new HashSet<>();
@@ -480,7 +466,7 @@ public final class CveDB implements AutoCloseable {
    /**
     * Saves a property to the database.
     *
-     * @param key the property key
+     * @param key   the property key
     * @param value the property value
     */
    public synchronized void saveProperty(String key, String value) {
@@ -514,7 +500,7 @@ public final class CveDB implements AutoCloseable {
     * is not the optimal cache eviction strategy, this is good enough for
     * typical usage (update DB and then only read) and it is easier to maintain
     * the code.
-     *
+     * <p>
     * It should be also called when DB is closed.
     */
    private synchronized void clearCache() {
@@ -672,6 +658,7 @@ public final class CveDB implements AutoCloseable {
        ResultSet rs = null;
        try {
            int vulnerabilityId = 0;
+            long countVulnerabilities = 0;
            final PreparedStatement selectVulnerabilityId = getPreparedStatement(SELECT_VULNERABILITY_ID);
            selectVulnerabilityId.setString(1, vuln.getName());
            rs = selectVulnerabilityId.executeQuery();
@@ -686,6 +673,7 @@ public final class CveDB implements AutoCloseable {
                deleteSoftware.setInt(1, vulnerabilityId);
                deleteSoftware.execute();
            }
+
            DBUtils.closeResultSet(rs);

            if (vulnerabilityId != 0) {
@@ -739,19 +727,23 @@ public final class CveDB implements AutoCloseable {
                insertReference.setString(2, r.getName());
                insertReference.setString(3, r.getUrl());
                insertReference.setString(4, r.getSource());
-                insertReference.addBatch();
-                countReferences++;
-                if (countReferences % MAX_BATCH_SIZE == 0) {
-                    insertReference.executeBatch();
-                    insertReference = getPreparedStatement(INSERT_REFERENCE);
-                    LOGGER.info(getLogForBatchInserts(countReferences, "Completed %s batch inserts to references table: %s"));
-                    countReferences = 0;
-                } else if (countReferences == vuln.getReferences().size()) {
-                    if (LOGGER.isTraceEnabled()) {
-                        LOGGER.trace(getLogForBatchInserts(countReferences, "Completed %s batch inserts to reference table: %s"));
+                if(isBatchInsertEnabled()) {
+                    insertReference.addBatch();
+                    countReferences++;
+                    if (countReferences % getBatchSize() == 0) {
+                        insertReference.executeBatch();
+                        insertReference = getPreparedStatement(INSERT_REFERENCE);
+                        LOGGER.info(getLogForBatchInserts(countReferences, "Completed %s batch inserts to references table: %s"));
+                        countReferences = 0;
+                    } else if (countReferences == vuln.getReferences().size()) {
+                        if (LOGGER.isTraceEnabled()) {
+                            LOGGER.trace(getLogForBatchInserts(countReferences, "Completed %s batch inserts to reference table: %s"));
+                        }
+                        insertReference.executeBatch();
+                        countReferences = 0;
                    }
-                    insertReference.executeBatch();
-                    countReferences = 0;
+                } else {
+                    insertReference.execute();
                }
            }

@@ -792,21 +784,33 @@ public final class CveDB implements AutoCloseable {
                } else {
                    insertSoftware.setString(3, vulnerableSoftware.getPreviousVersion());
                }
-                insertSoftware.addBatch();
-                countSoftware++;
-                if (countSoftware % MAX_BATCH_SIZE == 0) {
-                    executeBatch(vuln, vulnerableSoftware, insertSoftware);
-                    insertSoftware = getPreparedStatement(INSERT_SOFTWARE);
-                    LOGGER.info(getLogForBatchInserts(countSoftware, "Completed %s batch inserts software table: %s"));
-                    countSoftware = 0;
-                } else if (countSoftware == vuln.getVulnerableSoftware().size()) {
-                    if (LOGGER.isTraceEnabled()) {
-                        LOGGER.trace(getLogForBatchInserts(countSoftware, "Completed %s batch inserts software table: %s"));
-                        countReferences = 0;
+                if(isBatchInsertEnabled()) {
+                    insertSoftware.addBatch();
+                    countSoftware++;
+                    if (countSoftware % getBatchSize() == 0) {
+                        executeBatch(vuln, vulnerableSoftware, insertSoftware);
+                        insertSoftware = getPreparedStatement(INSERT_SOFTWARE);
+                        LOGGER.info(getLogForBatchInserts(countSoftware, "Completed %s batch inserts software table: %s"));
+                        countSoftware = 0;
+                    } else if (countSoftware == vuln.getVulnerableSoftware().size()) {
+                        if (LOGGER.isTraceEnabled()) {
+                            LOGGER.trace(getLogForBatchInserts(countSoftware, "Completed %s batch inserts software table: %s"));
+                            countReferences = 0;
+                        }
+                        executeBatch(vuln, vulnerableSoftware, insertSoftware);
+                    }
+                } else {
+                    try {
+                        insertSoftware.execute();
+                    } catch (SQLException ex) {
+                        if (ex.getMessage().contains("Duplicate entry")) {
+                            final String msg = String.format("Duplicate software key identified in '%s:%s'", vuln.getName(), vuln.getName());
+                            LOGGER.info(msg, ex);
+                        } else {
+                            throw ex;
+                        }
                    }
-                    executeBatch(vuln, vulnerableSoftware, insertSoftware);
                }
-
            }
        } catch (SQLException ex) {
            final String msg = String.format("Error updating '%s'", vuln.getName());
@@ -817,12 +821,33 @@ public final class CveDB implements AutoCloseable {
        }
    }

+    private int getBatchSize() {
+        int max;
+        try {
+            max = settings.getInt(Settings.KEYS.MAX_BATCH_SIZE);
+        } catch (InvalidSettingException pE) {
+            max = 1000;
+        }
+        return max;
+    }
+
+    private boolean isBatchInsertEnabled() {
+        boolean batch = false;
+        try {
+            batch = settings.getBoolean(Settings.KEYS.ENABLE_BATCH_UPDATES);
+        } catch (InvalidSettingException pE) {
+            //If there's no configuration, default is to not perform batch inserts
+            batch = false;
+        }
+        return batch;
+    }
+
    private String getLogForBatchInserts(int pCountReferences, String pFormat) {
        return String.format(pFormat, pCountReferences, new Date());
    }

    /**
-     * Executes batch inserts of vulnerabilities when MAX_BATCH_SIZE is reached
+     * Executes batch inserts of vulnerabilities when property database.batchinsert.maxsize is reached
     *
     * @param pVulnerability
     * @param pVulnerableSoftware
@@ -863,9 +888,9 @@ public final class CveDB implements AutoCloseable {
                dd = settings.getString(Settings.KEYS.DATA_DIRECTORY);
            }
            LOGGER.error("Unable to access the local database.\n\nEnsure that '{}' is a writable directory. "
-                    + "If the problem persist try deleting the files in '{}' and running {} again. If the problem continues, please "
-                    + "create a log file (see documentation at http://jeremylong.github.io/DependencyCheck/) and open a ticket at "
-                    + "https://github.com/jeremylong/DependencyCheck/issues and include the log file.\n\n",
+                            + "If the problem persist try deleting the files in '{}' and running {} again. If the problem continues, please "
+                            + "create a log file (see documentation at http://jeremylong.github.io/DependencyCheck/) and open a ticket at "
+                            + "https://github.com/jeremylong/DependencyCheck/issues and include the log file.\n\n",
                    dd, dd, settings.getString(Settings.KEYS.APPLICATION_NAME));
            LOGGER.debug("", ex);
        } finally {
@@ -898,16 +923,16 @@ public final class CveDB implements AutoCloseable {
     * previous version argument indicates that all previous versions are
     * affected.
     *
-     * @param vendor the vendor of the dependency being analyzed
-     * @param product the product name of the dependency being analyzed
+     * @param vendor             the vendor of the dependency being analyzed
+     * @param product            the product name of the dependency being analyzed
     * @param vulnerableSoftware a map of the vulnerable software with a boolean
-     * indicating if all previous versions are affected
-     * @param identifiedVersion the identified version of the dependency being
-     * analyzed
+     *                           indicating if all previous versions are affected
+     * @param identifiedVersion  the identified version of the dependency being
+     *                           analyzed
     * @return true if the identified version is affected, otherwise false
     */
    protected Entry<String, Boolean> getMatchingSoftware(Map<String, Boolean> vulnerableSoftware, String vendor, String product,
-            DependencyVersion identifiedVersion) {
+                                                         DependencyVersion identifiedVersion) {

        final boolean isVersionTwoADifferentProduct = "apache".equals(vendor) && "struts".equals(product);

@@ -1011,7 +1036,7 @@ public final class CveDB implements AutoCloseable {

    /**
     * This method is only referenced in unused code.
-     *
+     * <p>
     * Deletes unused dictionary entries from the database.
     */
    public synchronized void deleteUnusedCpe() {
@@ -1030,11 +1055,11 @@ public final class CveDB implements AutoCloseable {
    /**
     * This method is only referenced in unused code and will likely break on
     * MySQL if ever used due to the MERGE statement.
-     *
+     * <p>
     * Merges CPE entries into the database.
     *
-     * @param cpe the CPE identifier
-     * @param vendor the CPE vendor
+     * @param cpe     the CPE identifier
+     * @param vendor  the CPE vendor
     * @param product the CPE product
     */
    public synchronized void addCpe(String cpe, String vendor, String product) {
--- a/dependency-check-core/src/main/resources/dependencycheck.properties
+++ b/dependency-check-core/src/main/resources/dependencycheck.properties
@@ -50,7 +50,7 @@ cve.url.modified.validfordays=7
 # the number of hours to wait before checking if updates are available from the NVD.
 cve.check.validforhours=4
 #first year to pull data from the URLs below
-cve.startyear=2002
+cve.startyear=2017
 # the path to the modified nvd cve xml file.
 cve.url-1.2.modified=https://nvd.nist.gov/download/nvdcve-Modified.xml.gz
 #cve.url-1.2.modified=http://nvd.nist.gov/download/nvdcve-modified.xml
@@ -126,4 +126,6 @@ analyzer.nvdcve.enabled=true
 analyzer.vulnerabilitysuppression.enabled=true
 updater.nvdcve.enabled=true
 updater.versioncheck.enabled=true
-analyzer.versionfilter.enabled=true
+analyzer.versionfilter.enabled=true
+database.batchinsert.enabled=true
+database.batchinsert.maxsize=1000
--- a/dependency-check-utils/src/main/java/org/owasp/dependencycheck/utils/Settings.java
+++ b/dependency-check-utils/src/main/java/org/owasp/dependencycheck/utils/Settings.java
@@ -443,6 +443,16 @@ public final class Settings {
         */
        public static final String UPDATE_VERSION_CHECK_ENABLED = "updater.versioncheck.enabled";

+        /**
+         *
+         * Adds capabilities to batch insert. Tested on PostgreSQL and H2.
+         */
+        public static final String ENABLE_BATCH_UPDATES = "database.batchinsert.enabled";
+        /**
+         * Size of database batch inserts
+         */
+        public static final String MAX_BATCH_SIZE = "database.batchinsert.maxsize";
+
        /**
         * private constructor because this is a "utility" class containing
         * constants