From 79c31b5f545b63d67d90f393583241431dbfce8d Mon Sep 17 00:00:00 2001 From: Jeremy Long Date: Tue, 8 Oct 2013 10:58:29 -0400 Subject: [PATCH] Improved update process, including initial lock support Former-commit-id: d6fc00406765f4680d900eb0474fed9ade727a0a --- .../data/update/AbstractUpdate.java | 10 +- .../data/update/BatchUpdate.java | 5 - .../data/update/StandardUpdate.java | 29 ++-- .../data/update/Updateable.java | 5 + .../data/update/AbstractUpdateTest.java | 134 +++++++++++++++ ...dater_1_Test.java => BatchUpdateTest.java} | 42 +++-- .../data/update/DataStoreMetaInfoTest.java | 118 +++++++++++++ .../DatabaseUpdaterIntegrationTest.java | 13 +- .../data/update/DatabaseUpdater_2_Test.java | 87 ---------- .../data/update/NvdCveInfoTest.java | 113 +++++++++++++ .../update/StandardUpdateIntegrationTest.java | 72 ++++++++ .../data/update/UpdateableTest.java | 160 ++++++++++++++++++ 12 files changed, 657 insertions(+), 131 deletions(-) create mode 100644 dependency-check-core/src/test/java/org/owasp/dependencycheck/data/update/AbstractUpdateTest.java rename dependency-check-core/src/test/java/org/owasp/dependencycheck/data/update/{DatabaseUpdater_1_Test.java => BatchUpdateTest.java} (70%) create mode 100644 dependency-check-core/src/test/java/org/owasp/dependencycheck/data/update/DataStoreMetaInfoTest.java delete mode 100644 dependency-check-core/src/test/java/org/owasp/dependencycheck/data/update/DatabaseUpdater_2_Test.java create mode 100644 dependency-check-core/src/test/java/org/owasp/dependencycheck/data/update/NvdCveInfoTest.java create mode 100644 dependency-check-core/src/test/java/org/owasp/dependencycheck/data/update/StandardUpdateIntegrationTest.java create mode 100644 dependency-check-core/src/test/java/org/owasp/dependencycheck/data/update/UpdateableTest.java diff --git a/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/AbstractUpdate.java b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/AbstractUpdate.java index e7211e512..86a6040e3 100644 --- a/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/AbstractUpdate.java +++ b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/AbstractUpdate.java @@ -77,6 +77,15 @@ public abstract class AbstractUpdate { return updateable.isUpdateNeeded(); } + /** + * Gets the updateable NVD CVE Entries. + * + * @return an Updateable object containing the NVD CVE entries + */ + public Updateable getUpdateable() { + return updateable; + } + /** * Determines if the index needs to be updated. * @@ -102,7 +111,6 @@ public abstract class AbstractUpdate { * deleted. */ private boolean deleteAndRecreate = false; - protected Updateable updatesNeeded = null; /** * Get the value of deleteAndRecreate diff --git a/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/BatchUpdate.java b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/BatchUpdate.java index 4095e072c..fb19afee1 100644 --- a/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/BatchUpdate.java +++ b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/BatchUpdate.java @@ -20,14 +20,10 @@ package org.owasp.dependencycheck.data.update; import org.owasp.dependencycheck.data.nvdcve.InvalidDataException; import java.io.File; -import java.io.FileNotFoundException; import java.io.IOException; -import javax.xml.parsers.ParserConfigurationException; -import org.xml.sax.SAXException; import java.net.MalformedURLException; import java.net.URISyntaxException; import java.net.URL; -import java.sql.SQLException; import java.util.Calendar; import java.util.Date; import java.util.logging.Level; @@ -38,7 +34,6 @@ import org.owasp.dependencycheck.utils.DownloadFailedException; import org.owasp.dependencycheck.utils.Downloader; import org.owasp.dependencycheck.utils.FileUtils; import org.owasp.dependencycheck.utils.Settings; -import org.owasp.dependencycheck.data.nvdcve.DatabaseException; import org.owasp.dependencycheck.utils.InvalidSettingException; import static org.owasp.dependencycheck.data.update.DataStoreMetaInfo.BATCH; import static org.owasp.dependencycheck.data.update.DataStoreMetaInfo.MODIFIED; diff --git a/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/StandardUpdate.java b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/StandardUpdate.java index 6cefcaa0f..cfb5f26af 100644 --- a/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/StandardUpdate.java +++ b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/StandardUpdate.java @@ -55,17 +55,14 @@ public class StandardUpdate extends AbstractUpdate { *

Downloads the latest NVD CVE XML file from the web and imports it into * the current CVE Database.

* - * @param updatesNeeded a collection of NvdCveInfo containing information - * about needed updates. * @throws UpdateException is thrown if there is an error updating the * database */ @Override public void update() throws UpdateException { try { - properties = new DataStoreMetaInfo(); int maxUpdates = 0; - for (NvdCveInfo cve : updatesNeeded) { + for (NvdCveInfo cve : getUpdateable()) { if (cve.getNeedsUpdate()) { maxUpdates += 1; } @@ -79,7 +76,7 @@ public class StandardUpdate extends AbstractUpdate { } int count = 0; - for (NvdCveInfo cve : updatesNeeded) { + for (NvdCveInfo cve : getUpdateable()) { if (cve.getNeedsUpdate()) { count += 1; Logger.getLogger(StandardUpdate.class.getName()).log(Level.INFO, @@ -148,7 +145,7 @@ public class StandardUpdate extends AbstractUpdate { } } if (maxUpdates >= 1) { //ensure the modified file date gets written - properties.save(updatesNeeded.get(MODIFIED)); + properties.save(getUpdateable().get(MODIFIED)); cveDB.cleanupDatabase(); } } catch (MalformedURLException ex) { @@ -275,18 +272,16 @@ public class StandardUpdate extends AbstractUpdate { Settings.getString(Settings.KEYS.CVE_MODIFIED_12_URL), false); - //only add these urls if we are not in batch mode - if (!properties.isBatchUpdateMode()) { - final int start = Settings.getInt(Settings.KEYS.CVE_START_YEAR); - final int end = Calendar.getInstance().get(Calendar.YEAR); - final String baseUrl20 = Settings.getString(Settings.KEYS.CVE_SCHEMA_2_0); - final String baseUrl12 = Settings.getString(Settings.KEYS.CVE_SCHEMA_1_2); - for (int i = start; i <= end; i++) { - updates.add(Integer.toString(i), String.format(baseUrl20, i), - String.format(baseUrl12, i), - true); - } + final int start = Settings.getInt(Settings.KEYS.CVE_START_YEAR); + final int end = Calendar.getInstance().get(Calendar.YEAR); + final String baseUrl20 = Settings.getString(Settings.KEYS.CVE_SCHEMA_2_0); + final String baseUrl12 = Settings.getString(Settings.KEYS.CVE_SCHEMA_1_2); + for (int i = start; i <= end; i++) { + updates.add(Integer.toString(i), String.format(baseUrl20, i), + String.format(baseUrl12, i), + true); } + return updates; } } diff --git a/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/Updateable.java b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/Updateable.java index 02bb14c74..516094b69 100644 --- a/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/Updateable.java +++ b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/Updateable.java @@ -149,4 +149,9 @@ public class Updateable implements java.lang.Iterable, Iterator