From e86225c1e105b7374450ff971213da9a8ee7ba55 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0est=C3=A1k=20V=C3=ADt?= Date: Wed, 24 May 2017 09:24:40 +0200 Subject: [PATCH 1/2] Added some caching for CveDB in order to speedup some scans --- .../dependencycheck/data/nvdcve/CveDB.java | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/nvdcve/CveDB.java b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/nvdcve/CveDB.java index 7b42d82c3..a1430d28d 100644 --- a/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/nvdcve/CveDB.java +++ b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/nvdcve/CveDB.java @@ -36,6 +36,7 @@ import java.util.Properties; import java.util.ResourceBundle; import java.util.Set; import javax.annotation.concurrent.ThreadSafe; +import org.apache.commons.collections.map.ReferenceMap; import org.owasp.dependencycheck.data.cwe.CweDB; import org.owasp.dependencycheck.dependency.Reference; import org.owasp.dependencycheck.dependency.Vulnerability; @@ -48,6 +49,8 @@ import org.owasp.dependencycheck.utils.Settings; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import static org.apache.commons.collections.map.AbstractReferenceMap.HARD; +import static org.apache.commons.collections.map.AbstractReferenceMap.SOFT; import static org.owasp.dependencycheck.data.nvdcve.CveDB.PreparedStatementCveDb.*; /** @@ -91,6 +94,9 @@ public final class CveDB implements AutoCloseable { */ private final EnumMap preparedStatements = new EnumMap<>(PreparedStatementCveDb.class); + @SuppressWarnings("unchecked") + private final Map> vulnerabilitiesForCpeCache = new ReferenceMap(HARD, SOFT); + /** * The enum value names must match the keys of the statements in the * statement bundles "dbStatements*.properties". @@ -265,6 +271,7 @@ public final class CveDB implements AutoCloseable { */ @Override public synchronized void close() { + clearCache(); if (instance != null) { instance.usageCount -= 1; if (instance.usageCount <= 0 && instance.isOpen()) { @@ -474,6 +481,7 @@ public final class CveDB implements AutoCloseable { * @param value the property value */ public synchronized void saveProperty(String key, String value) { + clearCache(); try { try { final PreparedStatement mergeProperty = getPreparedStatement(MERGE_PROPERTY); @@ -498,6 +506,17 @@ public final class CveDB implements AutoCloseable { } } + /** + * Clears cache. Should be called whenever something is modified. While this is not the optimal cache eviction + * strategy, this is good enough for typical usage (update DB and then only read) and it is easier to maintain + * the code. + * + * It should be also called when DB is closed. + */ + private void clearCache() { + vulnerabilitiesForCpeCache.clear(); + } + /** * Retrieves the vulnerabilities associated with the specified CPE. * @@ -506,6 +525,13 @@ public final class CveDB implements AutoCloseable { * @throws DatabaseException thrown if there is an exception retrieving data */ public synchronized List getVulnerabilities(String cpeStr) throws DatabaseException { + final List cachedVulnerabilities = vulnerabilitiesForCpeCache.get(cpeStr); + if (cachedVulnerabilities != null) { + LOGGER.debug("Cache hit for {}", cpeStr); + return cachedVulnerabilities; + } else { + LOGGER.debug("Cache miss for {}", cpeStr); + } final VulnerableSoftware cpe = new VulnerableSoftware(); try { cpe.parseName(cpeStr); @@ -554,6 +580,7 @@ public final class CveDB implements AutoCloseable { } finally { DBUtils.closeResultSet(rs); } + vulnerabilitiesForCpeCache.put(cpeStr, vulnerabilities); return vulnerabilities; } @@ -633,6 +660,7 @@ public final class CveDB implements AutoCloseable { * @throws DatabaseException is thrown if the database */ public synchronized void updateVulnerability(Vulnerability vuln) throws DatabaseException { + clearCache(); try { int vulnerabilityId = 0; final PreparedStatement selectVulnerabilityId = getPreparedStatement(SELECT_VULNERABILITY_ID); @@ -799,6 +827,7 @@ public final class CveDB implements AutoCloseable { * ensure orphan entries are removed. */ public synchronized void cleanupDatabase() { + clearCache(); try { final PreparedStatement ps = getPreparedStatement(CLEANUP_ORPHANS); if (ps != null) { @@ -934,6 +963,7 @@ public final class CveDB implements AutoCloseable { * Deletes unused dictionary entries from the database. */ public synchronized void deleteUnusedCpe() { + clearCache(); PreparedStatement ps = null; try { ps = connection.prepareStatement(statementBundle.getString("DELETE_UNUSED_DICT_CPE")); @@ -956,6 +986,7 @@ public final class CveDB implements AutoCloseable { * @param product the CPE product */ public synchronized void addCpe(String cpe, String vendor, String product) { + clearCache(); PreparedStatement ps = null; try { ps = connection.prepareStatement(statementBundle.getString("ADD_DICT_CPE")); From 3bdc6988b3a36e1bbace057f8dc1d8b726d39a8b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0est=C3=A1k=20V=C3=ADt?= Date: Wed, 24 May 2017 16:00:37 +0200 Subject: [PATCH 2/2] Minor adjustments of caching for CveDB, based on code review --- .../java/org/owasp/dependencycheck/data/nvdcve/CveDB.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/nvdcve/CveDB.java b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/nvdcve/CveDB.java index a1430d28d..4fe5ad951 100644 --- a/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/nvdcve/CveDB.java +++ b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/nvdcve/CveDB.java @@ -24,6 +24,7 @@ import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.SQLException; import java.util.ArrayList; +import java.util.Collections; import java.util.EnumMap; import java.util.HashMap; import java.util.HashSet; @@ -95,7 +96,7 @@ public final class CveDB implements AutoCloseable { private final EnumMap preparedStatements = new EnumMap<>(PreparedStatementCveDb.class); @SuppressWarnings("unchecked") - private final Map> vulnerabilitiesForCpeCache = new ReferenceMap(HARD, SOFT); + private final Map> vulnerabilitiesForCpeCache = Collections.synchronizedMap(new ReferenceMap(HARD, SOFT)); /** * The enum value names must match the keys of the statements in the @@ -271,11 +272,11 @@ public final class CveDB implements AutoCloseable { */ @Override public synchronized void close() { - clearCache(); if (instance != null) { instance.usageCount -= 1; if (instance.usageCount <= 0 && instance.isOpen()) { instance.usageCount = 0; + clearCache(); instance.closeStatements(); try { instance.connection.close();