github-mirrors/DependencyCheck
1
0
Fork 0
mirror of https://github.com/ysoftdevs/DependencyCheck.git synced 2026-03-28 11:52:15 +01:00
Code Issues Packages Projects Releases Wiki Activity

resolve merge conflicts

Browse Source
This commit is contained in:
Jeremy Long
2017-12-09 07:03:33 -05:00
parent c0a0636fe9 b46d13fc18
commit fd47ede9d6
4 changed files with 156 additions and 72 deletions
Download Patch File Download Diff File Expand all files Collapse all files

210
dependency-check-core/src/main/java/org/owasp/dependencycheck/data/nvdcve/CveDB.java
View File

@@ -17,44 +17,31 @@
*/ */
package org.owasp.dependencycheck.data.nvdcve; package org.owasp.dependencycheck.data.nvdcve;
import org.apache.commons.collections.map.ReferenceMap;
import org.owasp.dependencycheck.data.cwe.CweDB;
import org.owasp.dependencycheck.dependency.Reference;
import org.owasp.dependencycheck.dependency.Vulnerability;
import org.owasp.dependencycheck.dependency.VulnerableSoftware;
import org.owasp.dependencycheck.utils.*;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.annotation.concurrent.ThreadSafe;
import java.io.IOException; import java.io.IOException;
import java.io.UnsupportedEncodingException; import java.io.UnsupportedEncodingException;
import java.sql.Connection; import java.sql.Connection;
import java.sql.PreparedStatement; import java.sql.PreparedStatement;
import java.sql.ResultSet; import java.sql.ResultSet;
import java.sql.SQLException; import java.sql.SQLException;
import java.util.ArrayList; import java.util.*;
import java.util.Collections;
import java.util.EnumMap;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Map.Entry; import java.util.Map.Entry;
import java.util.MissingResourceException;
import java.util.Properties;
import java.util.ResourceBundle;
import java.util.Set;
import javax.annotation.concurrent.ThreadSafe;
import org.apache.commons.collections.map.ReferenceMap;
import org.owasp.dependencycheck.data.cwe.CweDB;
import org.owasp.dependencycheck.dependency.Reference;
import org.owasp.dependencycheck.dependency.Vulnerability;
import org.owasp.dependencycheck.dependency.VulnerableSoftware;
import org.owasp.dependencycheck.utils.DBUtils;
import org.owasp.dependencycheck.utils.DependencyVersion;
import org.owasp.dependencycheck.utils.DependencyVersionUtil;
import org.owasp.dependencycheck.utils.Pair;
import org.owasp.dependencycheck.utils.Settings;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
//CSOFF: AvoidStarImport
import static org.owasp.dependencycheck.data.nvdcve.CveDB.PreparedStatementCveDb.*;
//CSON: AvoidStarImport
import static org.apache.commons.collections.map.AbstractReferenceMap.HARD; import static org.apache.commons.collections.map.AbstractReferenceMap.HARD;
import static org.apache.commons.collections.map.AbstractReferenceMap.SOFT; import static org.apache.commons.collections.map.AbstractReferenceMap.SOFT;
import static org.owasp.dependencycheck.data.nvdcve.CveDB.PreparedStatementCveDb.*;
//CSOFF: AvoidStarImport
//CSON: AvoidStarImport
/** /**
* The database holding information about the NVD CVE data. This class is safe * The database holding information about the NVD CVE data. This class is safe
@@ -70,6 +57,7 @@ public final class CveDB implements AutoCloseable {
* The logger. * The logger.
*/ */
private static final Logger LOGGER = LoggerFactory.getLogger(CveDB.class); private static final Logger LOGGER = LoggerFactory.getLogger(CveDB.class);
/** /**
* The database connection factory. * The database connection factory.
*/ */
@@ -203,7 +191,7 @@ public final class CveDB implements AutoCloseable {
* *
* @param settings the configured settings * @param settings the configured settings
* @throws DatabaseException thrown if there is an exception opening the * @throws DatabaseException thrown if there is an exception opening the
* database. * database.
*/ */
public CveDB(Settings settings) throws DatabaseException { public CveDB(Settings settings) throws DatabaseException {
this.settings = settings; this.settings = settings;
@@ -233,7 +221,7 @@ public final class CveDB implements AutoCloseable {
* create a new one. * create a new one.
* *
* @throws DatabaseException thrown if there is an error opening the * @throws DatabaseException thrown if there is an error opening the
* database connection * database connection
*/ */
private synchronized void open() throws DatabaseException { private synchronized void open() throws DatabaseException {
try { try {
@@ -298,7 +286,7 @@ public final class CveDB implements AutoCloseable {
* Prepares all statements to be used. * Prepares all statements to be used.
* *
* @throws DatabaseException thrown if there is an error preparing the * @throws DatabaseException thrown if there is an error preparing the
* statements * statements
*/ */
private void prepareStatements() throws DatabaseException { private void prepareStatements() throws DatabaseException {
for (PreparedStatementCveDb key : values()) { for (PreparedStatementCveDb key : values()) {
@@ -336,7 +324,7 @@ public final class CveDB implements AutoCloseable {
* Returns the specified prepared statement. * Returns the specified prepared statement.
* *
* @param key the prepared statement from {@link PreparedStatementCveDb} to * @param key the prepared statement from {@link PreparedStatementCveDb} to
* return * return
* @return the prepared statement * @return the prepared statement
* @throws SQLException thrown if a SQL Exception occurs * @throws SQLException thrown if a SQL Exception occurs
*/ */
@@ -399,9 +387,9 @@ public final class CveDB implements AutoCloseable {
* given vendor and product combination. The returned list will include all * given vendor and product combination. The returned list will include all
* versions of the product that are registered in the NVD CVE data. * versions of the product that are registered in the NVD CVE data.
* *
* @param vendor the identified vendor name of the dependency being analyzed * @param vendor the identified vendor name of the dependency being analyzed
* @param product the identified name of the product of the dependency being * @param product the identified name of the product of the dependency being
* analyzed * analyzed
* @return a set of vulnerable software * @return a set of vulnerable software
*/ */
public synchronized Set<VulnerableSoftware> getCPEs(String vendor, String product) { public synchronized Set<VulnerableSoftware> getCPEs(String vendor, String product) {
@@ -432,7 +420,7 @@ public final class CveDB implements AutoCloseable {
* *
* @return the entire list of vendor/product combinations * @return the entire list of vendor/product combinations
* @throws DatabaseException thrown when there is an error retrieving the * @throws DatabaseException thrown when there is an error retrieving the
* data from the DB * data from the DB
*/ */
public synchronized Set<Pair<String, String>> getVendorProductList() throws DatabaseException { public synchronized Set<Pair<String, String>> getVendorProductList() throws DatabaseException {
final Set<Pair<String, String>> data = new HashSet<>(); final Set<Pair<String, String>> data = new HashSet<>();
@@ -478,7 +466,7 @@ public final class CveDB implements AutoCloseable {
/** /**
* Saves a property to the database. * Saves a property to the database.
* *
* @param key the property key * @param key the property key
* @param value the property value * @param value the property value
*/ */
public synchronized void saveProperty(String key, String value) { public synchronized void saveProperty(String key, String value) {
@@ -512,7 +500,7 @@ public final class CveDB implements AutoCloseable {
* is not the optimal cache eviction strategy, this is good enough for * is not the optimal cache eviction strategy, this is good enough for
* typical usage (update DB and then only read) and it is easier to maintain * typical usage (update DB and then only read) and it is easier to maintain
* the code. * the code.
* * <p>
* It should be also called when DB is closed. * It should be also called when DB is closed.
*/ */
private synchronized void clearCache() { private synchronized void clearCache() {
@@ -684,6 +672,7 @@ public final class CveDB implements AutoCloseable {
deleteSoftware.setInt(1, vulnerabilityId); deleteSoftware.setInt(1, vulnerabilityId);
deleteSoftware.execute(); deleteSoftware.execute();
} }
DBUtils.closeResultSet(rs); DBUtils.closeResultSet(rs);
if (vulnerabilityId != 0) { if (vulnerabilityId != 0) {
@@ -730,36 +719,55 @@ public final class CveDB implements AutoCloseable {
} }
} }
final PreparedStatement insertReference = getPreparedStatement(INSERT_REFERENCE); PreparedStatement insertReference = getPreparedStatement(INSERT_REFERENCE);
int countReferences = 0;
for (Reference r : vuln.getReferences()) { for (Reference r : vuln.getReferences()) {
insertReference.setInt(1, vulnerabilityId); insertReference.setInt(1, vulnerabilityId);
insertReference.setString(2, r.getName()); insertReference.setString(2, r.getName());
insertReference.setString(3, r.getUrl()); insertReference.setString(3, r.getUrl());
insertReference.setString(4, r.getSource()); insertReference.setString(4, r.getSource());
insertReference.execute(); if(isBatchInsertEnabled()) {
insertReference.addBatch();
countReferences++;
if (countReferences % getBatchSize() == 0) {
insertReference.executeBatch();
insertReference = getPreparedStatement(INSERT_REFERENCE);
LOGGER.info(getLogForBatchInserts(countReferences, "Completed %s batch inserts to references table: %s"));
countReferences = 0;
} else if (countReferences == vuln.getReferences().size()) {
if (LOGGER.isTraceEnabled()) {
LOGGER.trace(getLogForBatchInserts(countReferences, "Completed %s batch inserts to reference table: %s"));
}
insertReference.executeBatch();
countReferences = 0;
}
} else {
insertReference.execute();
}
} }
final PreparedStatement insertSoftware = getPreparedStatement(INSERT_SOFTWARE); PreparedStatement insertSoftware = getPreparedStatement(INSERT_SOFTWARE);
for (VulnerableSoftware s : vuln.getVulnerableSoftware()) { int countSoftware = 0;
for (VulnerableSoftware vulnerableSoftware : vuln.getVulnerableSoftware()) {
int cpeProductId = 0; int cpeProductId = 0;
final PreparedStatement selectCpeId = getPreparedStatement(SELECT_CPE_ID); final PreparedStatement selectCpeId = getPreparedStatement(SELECT_CPE_ID);
selectCpeId.setString(1, s.getName()); selectCpeId.setString(1, vulnerableSoftware.getName());
try { try {
rs = selectCpeId.executeQuery(); rs = selectCpeId.executeQuery();
if (rs.next()) { if (rs.next()) {
cpeProductId = rs.getInt(1); cpeProductId = rs.getInt(1);
} }
} catch (SQLException ex) { } catch (SQLException ex) {
throw new DatabaseException("Unable to get primary key for new cpe: " + s.getName(), ex); throw new DatabaseException("Unable to get primary key for new cpe: " + vulnerableSoftware.getName(), ex);
} finally { } finally {
DBUtils.closeResultSet(rs); DBUtils.closeResultSet(rs);
} }
if (cpeProductId == 0) { if (cpeProductId == 0) {
final PreparedStatement insertCpe = getPreparedStatement(INSERT_CPE); final PreparedStatement insertCpe = getPreparedStatement(INSERT_CPE);
insertCpe.setString(1, s.getName()); insertCpe.setString(1, vulnerableSoftware.getName());
insertCpe.setString(2, s.getVendor()); insertCpe.setString(2, vulnerableSoftware.getVendor());
insertCpe.setString(3, s.getProduct()); insertCpe.setString(3, vulnerableSoftware.getProduct());
insertCpe.executeUpdate(); insertCpe.executeUpdate();
cpeProductId = DBUtils.getGeneratedKey(insertCpe); cpeProductId = DBUtils.getGeneratedKey(insertCpe);
} }
@@ -770,22 +778,38 @@ public final class CveDB implements AutoCloseable {
insertSoftware.setInt(1, vulnerabilityId); insertSoftware.setInt(1, vulnerabilityId);
insertSoftware.setInt(2, cpeProductId); insertSoftware.setInt(2, cpeProductId);
if (s.getPreviousVersion() == null) { if (vulnerableSoftware.getPreviousVersion() == null) {
insertSoftware.setNull(3, java.sql.Types.VARCHAR); insertSoftware.setNull(3, java.sql.Types.VARCHAR);
} else { } else {
insertSoftware.setString(3, s.getPreviousVersion()); insertSoftware.setString(3, vulnerableSoftware.getPreviousVersion());
} }
try { if(isBatchInsertEnabled()) {
insertSoftware.execute(); insertSoftware.addBatch();
} catch (SQLException ex) { countSoftware++;
if (ex.getMessage().contains("Duplicate entry")) { if (countSoftware % getBatchSize() == 0) {
final String msg = String.format("Duplicate software key identified in '%s:%s'", vuln.getName(), s.getName()); executeBatch(vuln, vulnerableSoftware, insertSoftware);
LOGGER.info(msg, ex); insertSoftware = getPreparedStatement(INSERT_SOFTWARE);
} else { LOGGER.info(getLogForBatchInserts(countSoftware, "Completed %s batch inserts software table: %s"));
throw ex; countSoftware = 0;
} else if (countSoftware == vuln.getVulnerableSoftware().size()) {
if (LOGGER.isTraceEnabled()) {
LOGGER.trace(getLogForBatchInserts(countSoftware, "Completed %s batch inserts software table: %s"));
countReferences = 0;
}
executeBatch(vuln, vulnerableSoftware, insertSoftware);
}
} else {
try {
insertSoftware.execute();
} catch (SQLException ex) {
if (ex.getMessage().contains("Duplicate entry")) {
final String msg = String.format("Duplicate software key identified in '%s:%s'", vuln.getName(), vuln.getName());
LOGGER.info(msg, ex);
} else {
throw ex;
}
} }
} }
} }
} catch (SQLException ex) { } catch (SQLException ex) {
final String msg = String.format("Error updating '%s'", vuln.getName()); final String msg = String.format("Error updating '%s'", vuln.getName());
@@ -796,6 +820,52 @@ public final class CveDB implements AutoCloseable {
} }
} }
private int getBatchSize() {
int max;
try {
max = settings.getInt(Settings.KEYS.MAX_BATCH_SIZE);
} catch (InvalidSettingException pE) {
max = 1000;
}
return max;
}
private boolean isBatchInsertEnabled() {
boolean batch = false;
try {
batch = settings.getBoolean(Settings.KEYS.ENABLE_BATCH_UPDATES);
} catch (InvalidSettingException pE) {
//If there's no configuration, default is to not perform batch inserts
batch = false;
}
return batch;
}
private String getLogForBatchInserts(int pCountReferences, String pFormat) {
return String.format(pFormat, pCountReferences, new Date());
}
/**
* Executes batch inserts of vulnerabilities when property database.batchinsert.maxsize is reached
*
* @param pVulnerability
* @param pVulnerableSoftware
* @param pInsertSoftware
* @throws SQLException
*/
private void executeBatch(Vulnerability pVulnerability, VulnerableSoftware pVulnerableSoftware, PreparedStatement pInsertSoftware) throws SQLException {
try {
pInsertSoftware.executeBatch();
} catch (SQLException ex) {
if (ex.getMessage().contains("Duplicate entry")) {
final String msg = String.format("Duplicate software key identified in '%s:%s'", pVulnerability.getName(), pVulnerableSoftware.getName());
LOGGER.info(msg, ex);
} else {
throw ex;
}
}
}
/** /**
* Checks to see if data exists so that analysis can be performed. * Checks to see if data exists so that analysis can be performed.
* *
@@ -817,9 +887,9 @@ public final class CveDB implements AutoCloseable {
dd = settings.getString(Settings.KEYS.DATA_DIRECTORY); dd = settings.getString(Settings.KEYS.DATA_DIRECTORY);
} }
LOGGER.error("Unable to access the local database.\n\nEnsure that '{}' is a writable directory. " LOGGER.error("Unable to access the local database.\n\nEnsure that '{}' is a writable directory. "
+ "If the problem persist try deleting the files in '{}' and running {} again. If the problem continues, please " + "If the problem persist try deleting the files in '{}' and running {} again. If the problem continues, please "
+ "create a log file (see documentation at http://jeremylong.github.io/DependencyCheck/) and open a ticket at " + "create a log file (see documentation at http://jeremylong.github.io/DependencyCheck/) and open a ticket at "
+ "https://github.com/jeremylong/DependencyCheck/issues and include the log file.\n\n", + "https://github.com/jeremylong/DependencyCheck/issues and include the log file.\n\n",
dd, dd, settings.getString(Settings.KEYS.APPLICATION_NAME)); dd, dd, settings.getString(Settings.KEYS.APPLICATION_NAME));
LOGGER.debug("", ex); LOGGER.debug("", ex);
} finally { } finally {
@@ -852,16 +922,16 @@ public final class CveDB implements AutoCloseable {
* previous version argument indicates that all previous versions are * previous version argument indicates that all previous versions are
* affected. * affected.
* *
* @param vendor the vendor of the dependency being analyzed * @param vendor the vendor of the dependency being analyzed
* @param product the product name of the dependency being analyzed * @param product the product name of the dependency being analyzed
* @param vulnerableSoftware a map of the vulnerable software with a boolean * @param vulnerableSoftware a map of the vulnerable software with a boolean
* indicating if all previous versions are affected * indicating if all previous versions are affected
* @param identifiedVersion the identified version of the dependency being * @param identifiedVersion the identified version of the dependency being
* analyzed * analyzed
* @return true if the identified version is affected, otherwise false * @return true if the identified version is affected, otherwise false
*/ */
protected Entry<String, Boolean> getMatchingSoftware(Map<String, Boolean> vulnerableSoftware, String vendor, String product, protected Entry<String, Boolean> getMatchingSoftware(Map<String, Boolean> vulnerableSoftware, String vendor, String product,
DependencyVersion identifiedVersion) { DependencyVersion identifiedVersion) {
final boolean isVersionTwoADifferentProduct = "apache".equals(vendor) && "struts".equals(product); final boolean isVersionTwoADifferentProduct = "apache".equals(vendor) && "struts".equals(product);
@@ -965,7 +1035,7 @@ public final class CveDB implements AutoCloseable {
/** /**
* This method is only referenced in unused code. * This method is only referenced in unused code.
* * <p>
* Deletes unused dictionary entries from the database. * Deletes unused dictionary entries from the database.
*/ */
public synchronized void deleteUnusedCpe() { public synchronized void deleteUnusedCpe() {
@@ -984,11 +1054,11 @@ public final class CveDB implements AutoCloseable {
/** /**
* This method is only referenced in unused code and will likely break on * This method is only referenced in unused code and will likely break on
* MySQL if ever used due to the MERGE statement. * MySQL if ever used due to the MERGE statement.
* * <p>
* Merges CPE entries into the database. * Merges CPE entries into the database.
* *
* @param cpe the CPE identifier * @param cpe the CPE identifier
* @param vendor the CPE vendor * @param vendor the CPE vendor
* @param product the CPE product * @param product the CPE product
*/ */
public synchronized void addCpe(String cpe, String vendor, String product) { public synchronized void addCpe(String cpe, String vendor, String product) {

4
dependency-check-core/src/main/resources/dependencycheck.properties
View File

@@ -127,5 +127,7 @@ analyzer.vulnerabilitysuppression.enabled=true
updater.nvdcve.enabled=true updater.nvdcve.enabled=true
updater.versioncheck.enabled=true updater.versioncheck.enabled=true
analyzer.versionfilter.enabled=true analyzer.versionfilter.enabled=true
ecosystem.skip.cpeanalyzer=npm ecosystem.skip.cpeanalyzer=npm
database.batchinsert.enabled=true
database.batchinsert.maxsize=1000

2
dependency-check-core/src/test/resources/dependencycheck.properties
View File

@@ -125,3 +125,5 @@ updater.nvdcve.enabled=true
updater.versioncheck.enabled=true updater.versioncheck.enabled=true
ecosystem.skip.cpeanalyzer=npm ecosystem.skip.cpeanalyzer=npm
database.batchinsert.enabled=true
database.batchinsert.maxsize=1000

10
dependency-check-utils/src/main/java/org/owasp/dependencycheck/utils/Settings.java
View File

@@ -447,6 +447,16 @@ public final class Settings {
*/ */
public static final String ECOSYSTEM_SKIP_CPEANALYZER = "ecosystem.skip.cpeanalyzer"; public static final String ECOSYSTEM_SKIP_CPEANALYZER = "ecosystem.skip.cpeanalyzer";
/**
*
* Adds capabilities to batch insert. Tested on PostgreSQL and H2.
*/
public static final String ENABLE_BATCH_UPDATES = "database.batchinsert.enabled";
/**
* Size of database batch inserts
*/
public static final String MAX_BATCH_SIZE = "database.batchinsert.maxsize";
/** /**
* private constructor because this is a "utility" class containing * private constructor because this is a "utility" class containing
* constants * constants