From 672e59e6579cdb07ee874a3cdac87f8fa252eebe Mon Sep 17 00:00:00 2001 From: Jeremy Long Date: Sun, 4 Aug 2013 14:47:54 -0400 Subject: [PATCH 01/50] commiting initial (and likely broken) version including batch updates - commiting as I need tests done with more OSes Former-commit-id: 38f07a64633e0762623cdb5a00478a5bafd159b3 --- .../data/nvdcve/xml/DatabaseUpdater.java | 171 ++++++++++++++---- 1 file changed, 135 insertions(+), 36 deletions(-) diff --git a/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/nvdcve/xml/DatabaseUpdater.java b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/nvdcve/xml/DatabaseUpdater.java index a1d9a9823..38256e4bb 100644 --- a/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/nvdcve/xml/DatabaseUpdater.java +++ b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/nvdcve/xml/DatabaseUpdater.java @@ -88,6 +88,11 @@ public class DatabaseUpdater implements CachedWebDataSource { */ private Index cpeIndex = null; + public DatabaseUpdater() { + batchUpdateMode = !Settings.getString(Settings.KEYS.BATCH_UPDATE_URL, "").isEmpty(); + doBatchUpdate = false; + } + /** *

Downloads the latest NVD CVE XML file from the web and imports it into * the current CVE Database.

@@ -95,6 +100,7 @@ public class DatabaseUpdater implements CachedWebDataSource { * @throws UpdateException is thrown if there is an error updating the * database */ + @Override public void update() throws UpdateException { try { final Map update = updateNeeded(); @@ -111,6 +117,15 @@ public class DatabaseUpdater implements CachedWebDataSource { if (maxUpdates > 0) { openDataStores(); } + + if (isBatchUpdateMode() && isDoBatchUpdate()) { + try { + performBatchUpdate(); + } catch (IOException ex) { + throw new UpdateException("Unable to perform batch update", ex); + } + } + int count = 0; for (NvdCveUrl cve : update.values()) { @@ -126,11 +141,11 @@ public class DatabaseUpdater implements CachedWebDataSource { "Downloading {0}", cve.getUrl()); outputPath = File.createTempFile("cve" + cve.getId() + "_", ".xml"); - Downloader.fetchFile(url, outputPath, false); + Downloader.fetchFile(url, outputPath); url = new URL(cve.getOldSchemaVersionUrl()); outputPath12 = File.createTempFile("cve_1_2_" + cve.getId() + "_", ".xml"); - Downloader.fetchFile(url, outputPath12, false); + Downloader.fetchFile(url, outputPath12); Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.INFO, "Processing {0}", cve.getUrl()); @@ -203,8 +218,8 @@ public class DatabaseUpdater implements CachedWebDataSource { * @throws ParserConfigurationException is thrown if there is a parser * configuration exception * @throws SAXException is thrown if there is a SAXException - * @throws IOException is thrown if there is a ioexception - * @throws SQLException is thrown if there is a sql exception + * @throws IOException is thrown if there is a IO Exception + * @throws SQLException is thrown if there is a SQL exception * @throws DatabaseException is thrown if there is a database exception * @throws ClassNotFoundException thrown if the h2 database driver cannot be * loaded @@ -381,17 +396,25 @@ public class DatabaseUpdater implements CachedWebDataSource { if (currentlyPublished == null) { throw new DownloadFailedException("Unable to retrieve valid timestamp from nvd cve downloads page"); } - String dir; - try { - dir = CveDB.getDataDirectory().getCanonicalPath(); - } catch (IOException ex) { - Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.FINE, "CveDB data directory doesn't exist?", ex); - throw new UpdateException("Unable to locate last updated properties file.", ex); - } - final File f = new File(dir); - if (f.exists()) { - final File cveProp = new File(dir, UPDATE_PROPERTIES_FILE); + final File cpeDataDirectory; + try { + cpeDataDirectory = CveDB.getDataDirectory(); + } catch (IOException ex) { + String msg; + try { + msg = String.format("Unable to create the CVE Data Directory '%s'", + Settings.getFile(Settings.KEYS.CVE_DATA_DIRECTORY).getCanonicalPath()); + } catch (IOException ex1) { + msg = String.format("Unable to create the CVE Data Directory, this is likely a configuration issue: '%s%s%s'", + Settings.getString(Settings.KEYS.DATA_DIRECTORY, ""), + File.separator, + Settings.getString(Settings.KEYS.CVE_DATA_DIRECTORY, "")); + } + throw new UpdateException(msg, ex); + } + if (cpeDataDirectory.exists()) { + final File cveProp = new File(cpeDataDirectory, UPDATE_PROPERTIES_FILE); if (cveProp.exists()) { final Properties prop = new Properties(); InputStream is = null; @@ -416,15 +439,10 @@ public class DatabaseUpdater implements CachedWebDataSource { } } if (deleteAndRecreate) { - Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.INFO, "The database version is old. Rebuilding the database."); is.close(); - //this is an old version of the lucene index - just delete it - FileUtils.delete(f); - - //this importer also updates the CPE index and it is also using an old version - final Index cpeId = new Index(); - final File cpeDir = cpeId.getDataDirectory(); - FileUtils.delete(cpeDir); + is = null; + deleteExistingData(); + setDoBatchUpdate(isBatchUpdateMode()); return currentlyPublished; } @@ -435,11 +453,19 @@ public class DatabaseUpdater implements CachedWebDataSource { final int end = Calendar.getInstance().get(Calendar.YEAR); if (lastUpdated == currentlyPublished.get(MODIFIED).timestamp) { currentlyPublished.clear(); //we don't need to update anything. + setDoBatchUpdate(batchUpdateMode); } else if (withinRange(lastUpdated, now.getTime(), days)) { currentlyPublished.get(MODIFIED).setNeedsUpdate(true); - for (int i = start; i <= end; i++) { - currentlyPublished.get(String.valueOf(i)).setNeedsUpdate(false); + if (isBatchUpdateMode()) { + setDoBatchUpdate(false); + } else { + for (int i = start; i <= end; i++) { + currentlyPublished.get(String.valueOf(i)).setNeedsUpdate(false); + } } + } else if (isBatchUpdateMode()) { + currentlyPublished.get(MODIFIED).setNeedsUpdate(true); + setDoBatchUpdate(true); } else { //we figure out which of the several XML files need to be downloaded. currentlyPublished.get(MODIFIED).setNeedsUpdate(false); for (int i = start; i <= end; i++) { @@ -492,6 +518,49 @@ public class DatabaseUpdater implements CachedWebDataSource { final double differenceInDays = (compareTo - date) / 1000.0 / 60.0 / 60.0 / 24.0; return differenceInDays < range; } + /** + * Indicates whether or not the updates are using a batch update mode or + * not. + */ + private boolean batchUpdateMode; + + /** + * Get the value of batchUpdateMode. + * + * @return the value of batchUpdateMode + */ + protected boolean isBatchUpdateMode() { + return batchUpdateMode; + } + + /** + * Set the value of batchUpdateMode. + * + * @param batchUpdateMode new value of batchUpdateMode + */ + protected void setBatchUpdateMode(boolean batchUpdateMode) { + this.batchUpdateMode = batchUpdateMode; + } + //flag indicating whether or not the batch update should be performed. + protected boolean doBatchUpdate; + + /** + * Get the value of doBatchUpdate + * + * @return the value of doBatchUpdate + */ + protected boolean isDoBatchUpdate() { + return doBatchUpdate; + } + + /** + * Set the value of doBatchUpdate + * + * @param doBatchUpdate new value of doBatchUpdate + */ + protected void setDoBatchUpdate(boolean doBatchUpdate) { + this.doBatchUpdate = doBatchUpdate; + } /** * Retrieves the timestamps from the NVD CVE meta data file. @@ -520,18 +589,21 @@ public class DatabaseUpdater implements CachedWebDataSource { item.timestamp = Downloader.getLastModified(new URL(retrieveUrl)); map.put(MODIFIED, item); - final int start = Settings.getInt(Settings.KEYS.CVE_START_YEAR); - final int end = Calendar.getInstance().get(Calendar.YEAR); - final String baseUrl20 = Settings.getString(Settings.KEYS.CVE_SCHEMA_2_0); - final String baseUrl12 = Settings.getString(Settings.KEYS.CVE_SCHEMA_1_2); - for (int i = start; i <= end; i++) { - retrieveUrl = String.format(baseUrl20, i); - item = new NvdCveUrl(); - item.setId(Integer.toString(i)); - item.setUrl(retrieveUrl); - item.setOldSchemaVersionUrl(String.format(baseUrl12, i)); - item.setTimestamp(Downloader.getLastModified(new URL(retrieveUrl))); - map.put(item.id, item); + //only add these urls if we are not in batch mode + if (!isBatchUpdateMode()) { + final int start = Settings.getInt(Settings.KEYS.CVE_START_YEAR); + final int end = Calendar.getInstance().get(Calendar.YEAR); + final String baseUrl20 = Settings.getString(Settings.KEYS.CVE_SCHEMA_2_0); + final String baseUrl12 = Settings.getString(Settings.KEYS.CVE_SCHEMA_1_2); + for (int i = start; i <= end; i++) { + retrieveUrl = String.format(baseUrl20, i); + item = new NvdCveUrl(); + item.setId(Integer.toString(i)); + item.setUrl(retrieveUrl); + item.setOldSchemaVersionUrl(String.format(baseUrl12, i)); + item.setTimestamp(Downloader.getLastModified(new URL(retrieveUrl))); + map.put(item.id, item); + } } return map; } @@ -550,6 +622,33 @@ public class DatabaseUpdater implements CachedWebDataSource { } } + /** + * Deletes the existing data directories. + * + * @throws IOException thrown if the directory cannot be deleted + */ + private void deleteExistingData() throws IOException { + Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.INFO, "The database version is old. Rebuilding the database."); + + final File cveDir = CveDB.getDataDirectory(); + FileUtils.delete(cveDir); + + final File cpeDir = Index.getDataDirectory(); + FileUtils.delete(cpeDir); + } + + private void performBatchUpdate() throws IOException { + if (batchUpdateMode && doBatchUpdate) { + deleteExistingData(); + String batchSrc = Settings.getString(Settings.KEYS.BATCH_UPDATE_URL); + File dataDirectory = CveDB.getDataDirectory().getParentFile(); + URL batchUrl = new URL(batchSrc); + File tmp = File.createTempFile("batch_", ".zip"); + Downloader.fetchFile(batchUrl, tmp); + FileUtils.extractFiles(tmp, dataDirectory); + } + } + /** * A pojo that contains the Url and timestamp of the current NvdCve XML * files. From ebabc1117ed6b5806d81a2a048c997830777b800 Mon Sep 17 00:00:00 2001 From: Jeremy Long Date: Sun, 4 Aug 2013 14:48:21 -0400 Subject: [PATCH 02/50] added additional tests Former-commit-id: b377007cf39b3f828fb336e336804b7db56ff923 --- .../dependencycheck/utils/DownloaderIntegrationTest.java | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/dependency-check-core/src/test/java/org/owasp/dependencycheck/utils/DownloaderIntegrationTest.java b/dependency-check-core/src/test/java/org/owasp/dependencycheck/utils/DownloaderIntegrationTest.java index 55bd4abe5..2726ea4fe 100644 --- a/dependency-check-core/src/test/java/org/owasp/dependencycheck/utils/DownloaderIntegrationTest.java +++ b/dependency-check-core/src/test/java/org/owasp/dependencycheck/utils/DownloaderIntegrationTest.java @@ -82,4 +82,12 @@ public class DownloaderIntegrationTest { long timestamp = Downloader.getLastModified(url); assertTrue("timestamp equal to zero?", timestamp > 0); } + + @Test + public void testGetLastModified_file() throws Exception { + File f = new File("target/test-classes/nvdcve-2.0-2012.xml"); + URL url = new URL("file://" + f.getCanonicalPath()); + long timestamp = Downloader.getLastModified(url); + assertTrue("timestamp equal to zero?", timestamp > 0); + } } From 43583bbc2eb54987d7a122c73ac99b97e9d4f5ee Mon Sep 17 00:00:00 2001 From: Jeremy Long Date: Tue, 6 Aug 2013 19:33:37 -0400 Subject: [PATCH 05/50] updated tests Former-commit-id: f290ba61239a0a5beee9522e036100753ddef2f7 --- .../dependencycheck/utils/DownloaderTest.java | 60 +++++++++++++++++++ 1 file changed, 60 insertions(+) create mode 100644 dependency-check-core/src/test/java/org/owasp/dependencycheck/utils/DownloaderTest.java diff --git a/dependency-check-core/src/test/java/org/owasp/dependencycheck/utils/DownloaderTest.java b/dependency-check-core/src/test/java/org/owasp/dependencycheck/utils/DownloaderTest.java new file mode 100644 index 000000000..b9624ec9b --- /dev/null +++ b/dependency-check-core/src/test/java/org/owasp/dependencycheck/utils/DownloaderTest.java @@ -0,0 +1,60 @@ +/* + * This file is part of dependency-check-core. + * + * Dependency-check-core is free software: you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation, either version 3 of the License, or (at your option) any + * later version. + * + * Dependency-check-core is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more + * details. + * + * You should have received a copy of the GNU General Public License along with + * dependency-check-core. If not, see http://www.gnu.org/licenses/. + * + * Copyright (c) 2012 Jeremy Long. All Rights Reserved. + */ +package org.owasp.dependencycheck.utils; + +import java.io.File; +import org.owasp.dependencycheck.utils.Downloader; +import java.net.URL; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; +import static org.junit.Assert.*; + +/** + * + * @author Jeremy Long (jeremy.long@owasp.org) + */ +public class DownloaderTest { + + @BeforeClass + public static void setUpClass() throws Exception { + } + + @AfterClass + public static void tearDownClass() throws Exception { + } + + @Before + public void setUp() { + } + + @After + public void tearDown() { + } + + @Test + public void testGetLastModified_file() throws Exception { + File f = new File("target/test-classes/nvdcve-2.0-2012.xml"); + URL url = new URL("file:///" + f.getCanonicalPath()); + long timestamp = Downloader.getLastModified(url); + assertTrue("timestamp equal to zero?", timestamp > 0); + } +} From 09f07902ef7c41f7858253bbe1f7b594a8047d03 Mon Sep 17 00:00:00 2001 From: Jeremy Long Date: Tue, 6 Aug 2013 19:34:11 -0400 Subject: [PATCH 06/50] updated batch update functionality Former-commit-id: f62347bd25b61f048f18fb8cb23b8de7c053659e --- .../dependencycheck/data/nvdcve/CveDB.java | 2 +- .../data/nvdcve/xml/DatabaseUpdater.java | 49 ++++++++--- .../dependencycheck/utils/Downloader.java | 27 +++--- .../dependencycheck/utils/FileUtils.java | 2 +- .../main/resources/dependencycheck.properties | 2 +- .../xml/DatabaseUpdaterIntegrationTest.java | 17 ++++ .../data/nvdcve/xml/DatabaseUpdaterTest.java | 85 +++++++++++++++++++ .../utils/DownloaderIntegrationTest.java | 8 -- 8 files changed, 156 insertions(+), 36 deletions(-) create mode 100644 dependency-check-core/src/test/java/org/owasp/dependencycheck/data/nvdcve/xml/DatabaseUpdaterTest.java diff --git a/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/nvdcve/CveDB.java b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/nvdcve/CveDB.java index cdf85925a..82ccce2e8 100644 --- a/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/nvdcve/CveDB.java +++ b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/nvdcve/CveDB.java @@ -176,7 +176,7 @@ public class CveDB { final File f = new File(fileName, "cve." + DB_SCHEMA_VERSION); final File check = new File(f.getAbsolutePath() + ".h2.db"); final boolean createTables = !check.exists(); - final String connStr = "jdbc:h2:file:" + f.getAbsolutePath(); + final String connStr = String.format("jdbc:h2:file:%s;AUTO_SERVER=TRUE", f.getAbsolutePath()); Class.forName("org.h2.Driver"); conn = DriverManager.getConnection(connStr, "sa", ""); if (createTables) { diff --git a/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/nvdcve/xml/DatabaseUpdater.java b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/nvdcve/xml/DatabaseUpdater.java index 38256e4bb..af160e3ce 100644 --- a/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/nvdcve/xml/DatabaseUpdater.java +++ b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/nvdcve/xml/DatabaseUpdater.java @@ -29,6 +29,7 @@ import java.io.OutputStreamWriter; import javax.xml.parsers.ParserConfigurationException; import org.owasp.dependencycheck.data.CachedWebDataSource; import java.net.MalformedURLException; +import java.net.URISyntaxException; import java.net.URL; import java.sql.SQLException; import java.util.Calendar; @@ -114,13 +115,14 @@ public class DatabaseUpdater implements CachedWebDataSource { Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.INFO, "NVD CVE requires several updates; this could take a couple of minutes."); } - if (maxUpdates > 0) { + if (maxUpdates > 0 && !isDoBatchUpdate()) { openDataStores(); } if (isBatchUpdateMode() && isDoBatchUpdate()) { try { performBatchUpdate(); + openDataStores(); } catch (IOException ex) { throw new UpdateException("Unable to perform batch update", ex); } @@ -498,7 +500,12 @@ public class DatabaseUpdater implements CachedWebDataSource { } } } + } else { + //properties file does not exist - check about batch update + setDoBatchUpdate(isBatchUpdateMode()); } + } else { //this condition will likely never exist - but just in case we need to handle batch updates + setDoBatchUpdate(isBatchUpdateMode()); } return currentlyPublished; } @@ -627,7 +634,7 @@ public class DatabaseUpdater implements CachedWebDataSource { * * @throws IOException thrown if the directory cannot be deleted */ - private void deleteExistingData() throws IOException { + protected void deleteExistingData() throws IOException { Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.INFO, "The database version is old. Rebuilding the database."); final File cveDir = CveDB.getDataDirectory(); @@ -637,15 +644,37 @@ public class DatabaseUpdater implements CachedWebDataSource { FileUtils.delete(cpeDir); } - private void performBatchUpdate() throws IOException { + private void performBatchUpdate() throws UpdateException { if (batchUpdateMode && doBatchUpdate) { - deleteExistingData(); - String batchSrc = Settings.getString(Settings.KEYS.BATCH_UPDATE_URL); - File dataDirectory = CveDB.getDataDirectory().getParentFile(); - URL batchUrl = new URL(batchSrc); - File tmp = File.createTempFile("batch_", ".zip"); - Downloader.fetchFile(batchUrl, tmp); - FileUtils.extractFiles(tmp, dataDirectory); + final String batchSrc = Settings.getString(Settings.KEYS.BATCH_UPDATE_URL); + File tmp = null; + try { + deleteExistingData(); + final File dataDirectory = CveDB.getDataDirectory().getParentFile(); + final URL batchUrl = new URL(batchSrc); + if ("file".equals(batchUrl.getProtocol())) { + try { + tmp = new File(batchUrl.toURI()); + } catch (URISyntaxException ex) { + final String msg = String.format("Invalid batch update URI: %s", batchSrc); + throw new UpdateException(msg, ex); + } + } else if ("http".equals(batchUrl.getProtocol()) + || "https".equals(batchUrl.getProtocol())) { + tmp = File.createTempFile("batch_", ".zip"); + Downloader.fetchFile(batchUrl, tmp); + } + //TODO add FTP? + FileUtils.extractFiles(tmp, dataDirectory); + + } catch (IOException ex) { + final String msg = String.format("IO Exception Occured performing batch update using: %s", batchSrc); + throw new UpdateException(msg, ex); + } finally { + if (tmp != null && !tmp.delete()) { + tmp.deleteOnExit(); + } + } } } diff --git a/dependency-check-core/src/main/java/org/owasp/dependencycheck/utils/Downloader.java b/dependency-check-core/src/main/java/org/owasp/dependencycheck/utils/Downloader.java index 263f3b6ff..fb3f0568f 100644 --- a/dependency-check-core/src/main/java/org/owasp/dependencycheck/utils/Downloader.java +++ b/dependency-check-core/src/main/java/org/owasp/dependencycheck/utils/Downloader.java @@ -131,23 +131,23 @@ public final class Downloader { long timestamp = 0; //TODO add the FPR protocol? if ("file".equalsIgnoreCase(url.getProtocol())) { - File f; + File lastModifiedFile; try { - if (System.getProperty("os.name").toLowerCase().startsWith("windows")) { - String filePath = url.toString(); - if (filePath.matches("file://[a-zA-Z]:.*")) { - f = new File(filePath.substring(7)); - } else { - f = new File(url.toURI()); - } - } else { - f = new File(url.toURI()); - } +// if (System.getProperty("os.name").toLowerCase().startsWith("windows")) { +// String filePath = url.toString(); +// if (filePath.matches("file://[a-zA-Z]:.*")) { +// f = new File(filePath.substring(7)); +// } else { +// f = new File(url.toURI()); +// } +// } else { + lastModifiedFile = new File(url.toURI()); +// } } catch (URISyntaxException ex) { final String msg = String.format("Unable to locate '%s'; is the cve.url-2.0.modified property set correctly?", url.toString()); throw new DownloadFailedException(msg); } - timestamp = f.lastModified(); + timestamp = lastModifiedFile.lastModified(); } else { HttpURLConnection conn = null; try { @@ -192,11 +192,8 @@ public final class Downloader { } else { conn = (HttpURLConnection) url.openConnection(); } - //added a default timeout of 20000 - //if (Settings.getString(Settings.KEYS.CONNECTION_TIMEOUT) != null) { final int timeout = Settings.getInt(Settings.KEYS.CONNECTION_TIMEOUT, 60000); conn.setConnectTimeout(timeout); - //} } catch (IOException ex) { if (conn != null) { try { diff --git a/dependency-check-core/src/main/java/org/owasp/dependencycheck/utils/FileUtils.java b/dependency-check-core/src/main/java/org/owasp/dependencycheck/utils/FileUtils.java index 8c8ae9813..3958b1a90 100644 --- a/dependency-check-core/src/main/java/org/owasp/dependencycheck/utils/FileUtils.java +++ b/dependency-check-core/src/main/java/org/owasp/dependencycheck/utils/FileUtils.java @@ -200,7 +200,7 @@ public final class FileUtils { while ((entry = zis.getNextEntry()) != null) { if (entry.isDirectory()) { final File d = new File(extractTo, entry.getName()); - if (!d.mkdirs()) { + if (!d.exists() && !d.mkdirs()) { final String msg = String.format("Unable to create '%s'.", d.getAbsolutePath()); throw new ExtractionException(msg); } diff --git a/dependency-check-core/src/main/resources/dependencycheck.properties b/dependency-check-core/src/main/resources/dependencycheck.properties index 38d9a0108..27c31e0a1 100644 --- a/dependency-check-core/src/main/resources/dependencycheck.properties +++ b/dependency-check-core/src/main/resources/dependencycheck.properties @@ -24,7 +24,7 @@ cve.url.modified.validfordays=7 # into the specified "data" directory. Additionally, after pulling the data the # system will attempt to update the modified. Thus, if one were maintaining an # internal copy of the data one would not need to update it nightly. -batch.update.url=file://C:/Users/jeremy/Desktop/demo/cli/data/data.zip +#batch.update.url=file:///C:/path/to/data.zip # the path to the modified nvd cve xml file. cve.url-1.2.modified=http://nvd.nist.gov/download/nvdcve-modified.xml diff --git a/dependency-check-core/src/test/java/org/owasp/dependencycheck/data/nvdcve/xml/DatabaseUpdaterIntegrationTest.java b/dependency-check-core/src/test/java/org/owasp/dependencycheck/data/nvdcve/xml/DatabaseUpdaterIntegrationTest.java index 4ada94f14..2f66e43b6 100644 --- a/dependency-check-core/src/test/java/org/owasp/dependencycheck/data/nvdcve/xml/DatabaseUpdaterIntegrationTest.java +++ b/dependency-check-core/src/test/java/org/owasp/dependencycheck/data/nvdcve/xml/DatabaseUpdaterIntegrationTest.java @@ -18,12 +18,15 @@ */ package org.owasp.dependencycheck.data.nvdcve.xml; +import java.io.File; +import java.net.URL; import org.owasp.dependencycheck.data.nvdcve.xml.DatabaseUpdater; import org.junit.After; import org.junit.AfterClass; import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; +import org.owasp.dependencycheck.utils.Settings; /** * @@ -60,4 +63,18 @@ public class DatabaseUpdaterIntegrationTest { DatabaseUpdater instance = new DatabaseUpdater(); instance.update(); } + + /** + * Test of update method (when in batch mode), of class DatabaseUpdater. + * + * @throws Exception + */ + @Test + public void testBatchUpdate() throws Exception { + File file = new File("target/test-classes/nvdcve-2.0-2012.xml"); + String path = "file:///" + file.getCanonicalPath(); + Settings.setString(Settings.KEYS.BATCH_UPDATE_URL, path); + DatabaseUpdater instance = new DatabaseUpdater(); + instance.update(); + } } diff --git a/dependency-check-core/src/test/java/org/owasp/dependencycheck/data/nvdcve/xml/DatabaseUpdaterTest.java b/dependency-check-core/src/test/java/org/owasp/dependencycheck/data/nvdcve/xml/DatabaseUpdaterTest.java new file mode 100644 index 000000000..ab0ab2e3b --- /dev/null +++ b/dependency-check-core/src/test/java/org/owasp/dependencycheck/data/nvdcve/xml/DatabaseUpdaterTest.java @@ -0,0 +1,85 @@ +/* + * This file is part of dependency-check-core. + * + * Dependency-check-core is free software: you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation, either version 3 of the License, or (at your option) any + * later version. + * + * Dependency-check-core is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more + * details. + * + * You should have received a copy of the GNU General Public License along with + * dependency-check-core. If not, see http://www.gnu.org/licenses/. + * + * Copyright (c) 2012 Jeremy Long. All Rights Reserved. + */ +package org.owasp.dependencycheck.data.nvdcve.xml; + +import org.owasp.dependencycheck.data.nvdcve.xml.DatabaseUpdater; +import java.io.File; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; +import org.owasp.dependencycheck.utils.Settings; + +/** + * + * @author Jeremy Long (jeremy.long@owasp.org) + */ +public class DatabaseUpdaterTest { + + public DatabaseUpdaterTest() { + } + + @BeforeClass + public static void setUpClass() throws Exception { + } + + @AfterClass + public static void tearDownClass() throws Exception { + } + private String old12; + private String old20; + + @Before + public void setUp() throws Exception { + old12 = Settings.getString(Settings.KEYS.CVE_MODIFIED_12_URL); + old20 = Settings.getString(Settings.KEYS.CVE_MODIFIED_20_URL); + + File file = new File("target/test-classes/nvdcve-2012.xml"); + String path = "file:///" + file.getCanonicalPath(); + Settings.setString(Settings.KEYS.CVE_MODIFIED_12_URL, path); + + file = new File("target/test-classes/nvdcve-2.0-2012.xml"); + path = "file:///" + file.getCanonicalPath(); + Settings.setString(Settings.KEYS.CVE_MODIFIED_20_URL, path); + + file = new File("target/test-classes/data.zip"); + path = "file:///" + file.getCanonicalPath(); + Settings.setString(Settings.KEYS.BATCH_UPDATE_URL, path); + } + + @After + public void tearDown() { + Settings.setString(Settings.KEYS.CVE_MODIFIED_12_URL, old12); + Settings.setString(Settings.KEYS.CVE_MODIFIED_20_URL, old20); + Settings.setString(Settings.KEYS.BATCH_UPDATE_URL, ""); + } + + /** + * Test of update method (when in batch mode), of class DatabaseUpdater. + * + * @throws Exception + */ + @Test + public void testBatchUpdate() throws Exception { + DatabaseUpdater instance = new DatabaseUpdater(); + instance.deleteExistingData(); + instance.update(); + } +} diff --git a/dependency-check-core/src/test/java/org/owasp/dependencycheck/utils/DownloaderIntegrationTest.java b/dependency-check-core/src/test/java/org/owasp/dependencycheck/utils/DownloaderIntegrationTest.java index 2726ea4fe..55bd4abe5 100644 --- a/dependency-check-core/src/test/java/org/owasp/dependencycheck/utils/DownloaderIntegrationTest.java +++ b/dependency-check-core/src/test/java/org/owasp/dependencycheck/utils/DownloaderIntegrationTest.java @@ -82,12 +82,4 @@ public class DownloaderIntegrationTest { long timestamp = Downloader.getLastModified(url); assertTrue("timestamp equal to zero?", timestamp > 0); } - - @Test - public void testGetLastModified_file() throws Exception { - File f = new File("target/test-classes/nvdcve-2.0-2012.xml"); - URL url = new URL("file://" + f.getCanonicalPath()); - long timestamp = Downloader.getLastModified(url); - assertTrue("timestamp equal to zero?", timestamp > 0); - } } From d81206fe2e99311ac623f5faa1fcc6dc738f85ed Mon Sep 17 00:00:00 2001 From: Jeremy Long Date: Sun, 18 Aug 2013 05:54:11 -0400 Subject: [PATCH 07/50] Added an implementation of a spin lock that can be used to lock a directory. Former-commit-id: 121a3d5d026f524698762b377c3582fbc9303bf2 --- .../concurrency/DirectoryLockException.java | 64 +++++ .../concurrency/DirectorySpinLock.java | 266 ++++++++++++++++++ .../InvalidDirectoryException.java | 64 +++++ .../concurrency/DirectorySpinLockTest.java | 116 ++++++++ .../concurrency/SpinLockTask.java | 82 ++++++ 5 files changed, 592 insertions(+) create mode 100644 dependency-check-core/src/main/java/org/owasp/dependencycheck/concurrency/DirectoryLockException.java create mode 100644 dependency-check-core/src/main/java/org/owasp/dependencycheck/concurrency/DirectorySpinLock.java create mode 100644 dependency-check-core/src/main/java/org/owasp/dependencycheck/concurrency/InvalidDirectoryException.java create mode 100644 dependency-check-core/src/test/java/org/owasp/dependencycheck/concurrency/DirectorySpinLockTest.java create mode 100644 dependency-check-core/src/test/java/org/owasp/dependencycheck/concurrency/SpinLockTask.java diff --git a/dependency-check-core/src/main/java/org/owasp/dependencycheck/concurrency/DirectoryLockException.java b/dependency-check-core/src/main/java/org/owasp/dependencycheck/concurrency/DirectoryLockException.java new file mode 100644 index 000000000..5985152a1 --- /dev/null +++ b/dependency-check-core/src/main/java/org/owasp/dependencycheck/concurrency/DirectoryLockException.java @@ -0,0 +1,64 @@ +/* + * This file is part of dependency-check-core. + * + * Dependency-check-core is free software: you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation, either version 3 of the License, or (at your option) any + * later version. + * + * Dependency-check-core is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more + * details. + * + * You should have received a copy of the GNU General Public License along with + * dependency-check-core. If not, see http://www.gnu.org/licenses/. + * + * Copyright (c) 2013 Jeremy Long. All Rights Reserved. + */ +package org.owasp.dependencycheck.concurrency; + +/** + * If thrown, indicates that a problem occurred when locking a directory. + * + * @author Jeremy Long (jeremy.long@owasp.org) + */ +public class DirectoryLockException extends Exception { + + private static final long serialVersionUID = 1L; + + /** + * Constructs a new Directory Lock Exception. + */ + public DirectoryLockException() { + super(); + } + + /** + * Constructs a new Directory Lock Exception. + * + * @param msg the message describing the exception + */ + public DirectoryLockException(String msg) { + super(msg); + } + + /** + * Constructs a new Directory Lock Exception. + * + * @param ex the cause of the exception + */ + public DirectoryLockException(Throwable ex) { + super(ex); + } + + /** + * Constructs a new Directory Lock Exception. + * + * @param msg the message describing the exception + * @param ex the cause of the exception + */ + public DirectoryLockException(String msg, Throwable ex) { + super(msg, ex); + } +} diff --git a/dependency-check-core/src/main/java/org/owasp/dependencycheck/concurrency/DirectorySpinLock.java b/dependency-check-core/src/main/java/org/owasp/dependencycheck/concurrency/DirectorySpinLock.java new file mode 100644 index 000000000..60de62900 --- /dev/null +++ b/dependency-check-core/src/main/java/org/owasp/dependencycheck/concurrency/DirectorySpinLock.java @@ -0,0 +1,266 @@ +/* + * This file is part of dependency-check-core. + * + * Dependency-check-core is free software: you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation, either version 3 of the License, or (at your option) any + * later version. + * + * Dependency-check-core is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more + * details. + * + * You should have received a copy of the GNU General Public License along with + * dependency-check-core. If not, see http://www.gnu.org/licenses/. + * + * Copyright (c) 2013 Jeremy Long. All Rights Reserved. + */ +package org.owasp.dependencycheck.concurrency; + +import java.io.Closeable; +import java.io.File; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.RandomAccessFile; +import java.nio.channels.AsynchronousCloseException; +import java.nio.channels.ClosedChannelException; +import java.nio.channels.FileChannel; +import java.nio.channels.FileLock; +import java.nio.channels.FileLockInterruptionException; +import java.nio.channels.NonWritableChannelException; +import java.nio.channels.OverlappingFileLockException; +import java.util.logging.Level; +import java.util.logging.Logger; + +/** + * Implements a spin lock on a given directory. If the lock cannot be obtained, + * the process will "spin" waiting for an opportunity to obtain the lock + * requested. + * + * @author Jeremy Long (jeremy.long@owasp.org) + */ +public class DirectorySpinLock implements Closeable, AutoCloseable { + + /** + * The name of the lock file. + */ + public static final String LOCK_NAME = "data.lock"; + /** + * The maximum wait period used when attempting to obtain a lock. + */ + public static final int MAX_SPIN = 100; + /** + * The file channel used to perform the lock. + */ + private FileChannel channel = null; + /** + * The file used to perform the lock. + */ + private File lockFile = null; + /** + * The lock object. + */ + private FileLock lock = null; + /** + * The maximum number of seconds that the spin lock will wait while trying + * to obtain a lock. + */ + private long maxWait = MAX_SPIN; + + /** + * Get the maximum wait time, in seconds, that the spin lock will wait while + * trying to obtain a lock. + * + * @return the number of seconds the spin lock will wait + */ + public long getMaxWait() { + return maxWait / 2; //sleep is for 500, so / 2 + } + + /** + * Set the maximum wait time, in seconds, that the spin lock will wait while + * trying to obtain a lock. + * + * @param maxWait the number of seconds the spin lock will wait + */ + public void setMaxWait(long maxWait) { + this.maxWait = maxWait * 2; //sleep is for 500, so * 2 + } + + /** + * Constructs a new spin lock on the given directory. + * + * @param directory the directory to monitor/lock + * @throws InvalidDirectoryException thrown if there is an issue with the + * directory provided + * @throws DirectoryLockException thrown there is an issue obtaining a + * handle to the lock file + */ + public DirectorySpinLock(File directory) throws InvalidDirectoryException, DirectoryLockException { + checkDirectory(directory); + lockFile = new File(directory, LOCK_NAME); + RandomAccessFile file = null; + try { + file = new RandomAccessFile(lockFile, "rw"); + } catch (FileNotFoundException ex) { + throw new DirectoryLockException("Lock file not found", ex); + } + channel = file.getChannel(); + } + + /** + * Attempts to obtain an exclusive lock; an exception is thrown if the lock + * could not be obtained. This method may block for a few seconds if a lock + * cannot be obtained. + * + * @throws DirectoryLockException thrown if there is an exception obtaining + * the lock + */ + public void obtainSharedLock() throws DirectoryLockException { + obtainLock(true); + } + + /** + * Attempts to obtain an exclusive lock; an exception is thrown if the lock + * could not be obtained. This method may block for a few seconds if a lock + * cannot be obtained. + * + * @throws DirectoryLockException thrown if there is an exception obtaining + * the lock + */ + public void obtainExclusiveLock() throws DirectoryLockException { + obtainLock(false); + } + + /** + * Attempts to obtain a lock; an exception is thrown if the lock could not + * be obtained. This method may block for a few seconds if a lock cannot be + * obtained. + * + * @param shared true if the lock is shared, otherwise false + * @param maxWait the maximum time to wait, in seconds, while trying to + * obtain the lock + * @throws DirectoryLockException thrown if there is an exception obtaining + * the lock + */ + protected void obtainLock(boolean shared, long maxWait) throws DirectoryLockException { + setMaxWait(maxWait); + obtainLock(shared); + } + + /** + * Attempts to obtain a lock; an exception is thrown if the lock could not + * be obtained. This method may block for a few seconds if a lock cannot be + * obtained. + * + * @param shared true if the lock is shared, otherwise false + * @throws DirectoryLockException thrown if there is an exception obtaining + * the lock + */ + protected void obtainLock(boolean shared) throws DirectoryLockException { + if (lock != null) { + release(); + } + if (channel == null) { + throw new DirectoryLockException("Unable to create lock, no file channel exists"); + } + int count = 0; + Exception lastException = null; + while (lock == null && count++ < maxWait) { + try { + lock = channel.lock(0, Long.MAX_VALUE, shared); + } catch (AsynchronousCloseException ex) { + lastException = ex; + } catch (ClosedChannelException ex) { + lastException = ex; + } catch (FileLockInterruptionException ex) { + lastException = ex; + } catch (OverlappingFileLockException ex) { + lastException = ex; + } catch (NonWritableChannelException ex) { + lastException = ex; + } catch (IOException ex) { + lastException = ex; + } + try { + Thread.sleep(500); + } catch (InterruptedException ex) { + Thread.currentThread().interrupt(); + } + } + if (lock == null) { + if (lastException == null) { + throw new DirectoryLockException("Unable to obtain lock"); + } else { + throw new DirectoryLockException("Unable to obtain lock", lastException); + } + } + } + + /** + * Performs a few simple rudimentary checks on the specified directory. + * Specifically, does the file exist and is it a directory. + * + * @param directory the File object to inspect + * @throws InvalidDirectoryException thrown if the directory is null or is + * not a directory + */ + private void checkDirectory(File directory) throws InvalidDirectoryException { + if (directory == null) { + throw new InvalidDirectoryException("Unable to obtain lock on a null File"); + } + if (!directory.isDirectory()) { + final String msg = String.format("File, '%s', does not exist or is not a directory", directory.getAbsolutePath()); + throw new InvalidDirectoryException(msg); + } + } + + /** + * Releases any locks and closes the underlying channel. + * + * @throws IOException + */ + @Override + public void close() throws IOException { + release(); + if (lock != null) { + try { + lock.close(); + } catch (IOException ex) { + Logger.getLogger(DirectorySpinLock.class.getName()).log(Level.FINEST, "Unable to close file lock due to IO Exception", ex); + } + } + if (channel != null) { + try { + channel.close(); + } catch (IOException ex) { + Logger.getLogger(DirectorySpinLock.class.getName()).log(Level.FINEST, "Unable to close the channel for the file lock", ex); + } + } + if (lockFile != null) { + if (lockFile.exists()) { + /* yes, this delete could fail which is totally fine. The other + * thread holding the lock while delete it. + */ + lockFile.delete(); + } + } + } + + /** + * Releases the lock. Any exceptions that are thrown by the underlying lock + * during the release are ignored. + */ + public void release() { + if (lock != null) { + try { + lock.release(); + } catch (ClosedChannelException ex) { + Logger.getLogger(DirectorySpinLock.class.getName()).log(Level.FINEST, "Uable to release file lock", ex); + } catch (IOException ex) { + Logger.getLogger(DirectorySpinLock.class.getName()).log(Level.FINEST, "Unable to release file lock due to IO Exception", ex); + } + } + } +} diff --git a/dependency-check-core/src/main/java/org/owasp/dependencycheck/concurrency/InvalidDirectoryException.java b/dependency-check-core/src/main/java/org/owasp/dependencycheck/concurrency/InvalidDirectoryException.java new file mode 100644 index 000000000..3faa54ffa --- /dev/null +++ b/dependency-check-core/src/main/java/org/owasp/dependencycheck/concurrency/InvalidDirectoryException.java @@ -0,0 +1,64 @@ +/* + * This file is part of dependency-check-core. + * + * Dependency-check-core is free software: you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation, either version 3 of the License, or (at your option) any + * later version. + * + * Dependency-check-core is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more + * details. + * + * You should have received a copy of the GNU General Public License along with + * dependency-check-core. If not, see http://www.gnu.org/licenses/. + * + * Copyright (c) 2013 Jeremy Long. All Rights Reserved. + */ +package org.owasp.dependencycheck.concurrency; + +/** + * If thrown, indicates that there is a problem with a directory. + * + * @author Jeremy Long (jeremy.long@owasp.org) + */ +public class InvalidDirectoryException extends Exception { + + private static final long serialVersionUID = 1L; + + /** + * Constructs a new Invalid Directory Exception. + */ + public InvalidDirectoryException() { + super(); + } + + /** + * Constructs a new Invalid Directory Exception. + * + * @param msg the message describing the exception + */ + public InvalidDirectoryException(String msg) { + super(msg); + } + + /** + * Constructs a new Invalid Directory Exception. + * + * @param ex the cause of the exception + */ + public InvalidDirectoryException(Throwable ex) { + super(ex); + } + + /** + * Constructs a new Invalid Directory Exception. + * + * @param msg the message describing the exception + * @param ex the cause of the exception + */ + public InvalidDirectoryException(String msg, Throwable ex) { + super(msg, ex); + } +} diff --git a/dependency-check-core/src/test/java/org/owasp/dependencycheck/concurrency/DirectorySpinLockTest.java b/dependency-check-core/src/test/java/org/owasp/dependencycheck/concurrency/DirectorySpinLockTest.java new file mode 100644 index 000000000..ac1c95285 --- /dev/null +++ b/dependency-check-core/src/test/java/org/owasp/dependencycheck/concurrency/DirectorySpinLockTest.java @@ -0,0 +1,116 @@ +/* + * This file is part of dependency-check-core. + * + * Dependency-check-core is free software: you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation, either version 3 of the License, or (at your option) any + * later version. + * + * Dependency-check-core is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more + * details. + * + * You should have received a copy of the GNU General Public License along with + * dependency-check-core. If not, see http://www.gnu.org/licenses/. + * + * Copyright (c) 2013 Jeremy Long. All Rights Reserved. + */ +package org.owasp.dependencycheck.concurrency; + +import java.io.File; +import java.net.URL; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; +import static org.junit.Assert.*; + +/** + * + * @author Jeremy Long (jeremy.long@owasp.org) + */ +public class DirectorySpinLockTest { + + public DirectorySpinLockTest() { + } + + @BeforeClass + public static void setUpClass() { + } + + @AfterClass + public static void tearDownClass() { + } + + @Before + public void setUp() { + } + + @After + public void tearDown() { + } + + /** + * Test of obtainSharedLock method, of class DirectorySpinLock. + * Specifically, this test uses the SpinLockTask to obtain an exclusive lock + * that is held for 5 seconds. We then try to obtain a shared lock while + * that task is running. It should take longer then 5 seconds to obtain the + * shared lock. + */ + @Test + public void testObtainSharedLock_withContention() throws Exception { + URL location = this.getClass().getProtectionDomain().getCodeSource().getLocation(); + File directory = new File(location.getFile()); + DirectorySpinLock instance = new DirectorySpinLock(directory); + SpinLockTask task = new SpinLockTask(directory, 5000, false, 2); + long start = System.currentTimeMillis(); + task.run(); + instance.obtainSharedLock(); + long end = System.currentTimeMillis(); + instance.close(); + if (task.getException() != null) { + throw task.getException(); + } + long timeElapsed = end - start; + assertTrue("no lock contention occured?", timeElapsed >= 5000); + //no exceptions means everything worked. + } + + /** + * Test of obtainSharedLock method, of class DirectorySpinLock. This method + * obtains two shared locks by using the SpinLockTask to obtain a lock in + * another thread. + */ + @Test + public void testObtainSharedLock() throws Exception { + URL location = this.getClass().getProtectionDomain().getCodeSource().getLocation(); + File directory = new File(location.getFile()); + DirectorySpinLock instance = new DirectorySpinLock(directory); + SpinLockTask task = new SpinLockTask(directory, 1000, true, 2); + task.run(); + instance.obtainSharedLock(); + instance.close(); + if (task.getException() != null) { + throw task.getException(); + } + //no exceptions means everything worked. + } + + /** + * Test of obtainExclusiveLock method, of class DirectorySpinLock. + */ + @Test + public void testObtainExclusiveLock() throws Exception { + URL location = this.getClass().getProtectionDomain().getCodeSource().getLocation(); + File directory = new File(location.getFile()); + DirectorySpinLock instance = new DirectorySpinLock(directory); + SpinLockTask task = new SpinLockTask(directory, 1000, true, 1); + instance.obtainExclusiveLock(); + task.run(); + instance.close(); + assertNotNull("No exception thrown due to exclusive lock failure?", task.getException()); + assertEquals("Incorrect exception when obtaining exclusive lock", "Unable to obtain lock", task.getException().getMessage()); + } +} diff --git a/dependency-check-core/src/test/java/org/owasp/dependencycheck/concurrency/SpinLockTask.java b/dependency-check-core/src/test/java/org/owasp/dependencycheck/concurrency/SpinLockTask.java new file mode 100644 index 000000000..7e6ce2a35 --- /dev/null +++ b/dependency-check-core/src/test/java/org/owasp/dependencycheck/concurrency/SpinLockTask.java @@ -0,0 +1,82 @@ +/* + * This file is part of dependency-check-core. + * + * Dependency-check-core is free software: you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation, either version 3 of the License, or (at your option) any + * later version. + * + * Dependency-check-core is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more + * details. + * + * You should have received a copy of the GNU General Public License along with + * dependency-check-core. If not, see http://www.gnu.org/licenses/. + * + * Copyright (c) 2013 Jeremy Long. All Rights Reserved. + */ +package org.owasp.dependencycheck.concurrency; + +import java.io.File; +import java.io.IOException; +import java.util.logging.Level; +import java.util.logging.Logger; + +/** + * + * @author Jeremy Long (jeremy.long@owasp.org) + */ +public class SpinLockTask implements Runnable { + + DirectorySpinLock lock = null; + int holdLockFor; + long maxWait; + boolean shared; + private Exception exception = null; + + /** + * Get the value of exception + * + * @return the value of exception + */ + public Exception getException() { + return exception; + } + + /** + * Set the value of exception + * + * @param exception new value of exception + */ + public void setException(Exception exception) { + this.exception = exception; + } + + public SpinLockTask(File directory, int holdLockFor, boolean shared, long maxWait) throws InvalidDirectoryException, DirectoryLockException { + this.holdLockFor = holdLockFor; + this.shared = shared; + this.maxWait = maxWait; + lock = new DirectorySpinLock(directory); + } + + @Override + public void run() { + try { + lock.obtainLock(shared, maxWait); + Thread.sleep(holdLockFor); + } catch (DirectoryLockException ex) { + exception = ex; + } catch (InterruptedException ex) { + exception = ex; + } finally { + if (lock != null) { + try { + lock.close(); + } catch (IOException ex) { + exception = ex; + } + } + } + } +} From 05f40f345169c8a0d5c6256d5ad66cc85dfdd05d Mon Sep 17 00:00:00 2001 From: Jeremy Long Date: Sun, 18 Aug 2013 05:55:31 -0400 Subject: [PATCH 08/50] updated javadoc Former-commit-id: 21268a3107bf50b84d02f8ed68dd567bba7d6de9 --- .../org/owasp/dependencycheck/concurrency/SpinLockTask.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/dependency-check-core/src/test/java/org/owasp/dependencycheck/concurrency/SpinLockTask.java b/dependency-check-core/src/test/java/org/owasp/dependencycheck/concurrency/SpinLockTask.java index 7e6ce2a35..096f67db5 100644 --- a/dependency-check-core/src/test/java/org/owasp/dependencycheck/concurrency/SpinLockTask.java +++ b/dependency-check-core/src/test/java/org/owasp/dependencycheck/concurrency/SpinLockTask.java @@ -24,6 +24,8 @@ import java.util.logging.Level; import java.util.logging.Logger; /** + * A simple task that obtains a lock on a directory. This is used in testing of + * the shared and exclusive locks. * * @author Jeremy Long (jeremy.long@owasp.org) */ From 09f416efdf36549427c4bdebef256c60731be385 Mon Sep 17 00:00:00 2001 From: Jeremy Long Date: Sat, 24 Aug 2013 16:21:32 -0400 Subject: [PATCH 09/50] modified CPE Index to seperate writer/reader in prep for adding locking Former-commit-id: baa48c8e70b36b9b14591d581bc1180b6d30d686 --- .../dependencycheck/analyzer/CPEAnalyzer.java | 790 ++++++++++++++++++ .../dependencycheck/data/cpe/BaseIndex.java | 108 +++ .../data/cpe/CpeIndexReader.java | 179 ++++ .../data/cpe/CpeIndexWriter.java | 149 ++++ .../analyzer/CPEAnalyzerTest.java | 256 ++++++ .../data/cpe/BaseIndexTest.java | 63 ++ 6 files changed, 1545 insertions(+) create mode 100644 dependency-check-core/src/main/java/org/owasp/dependencycheck/analyzer/CPEAnalyzer.java create mode 100644 dependency-check-core/src/main/java/org/owasp/dependencycheck/data/cpe/BaseIndex.java create mode 100644 dependency-check-core/src/main/java/org/owasp/dependencycheck/data/cpe/CpeIndexReader.java create mode 100644 dependency-check-core/src/main/java/org/owasp/dependencycheck/data/cpe/CpeIndexWriter.java create mode 100644 dependency-check-core/src/test/java/org/owasp/dependencycheck/analyzer/CPEAnalyzerTest.java create mode 100644 dependency-check-core/src/test/java/org/owasp/dependencycheck/data/cpe/BaseIndexTest.java diff --git a/dependency-check-core/src/main/java/org/owasp/dependencycheck/analyzer/CPEAnalyzer.java b/dependency-check-core/src/main/java/org/owasp/dependencycheck/analyzer/CPEAnalyzer.java new file mode 100644 index 000000000..ab1dd665c --- /dev/null +++ b/dependency-check-core/src/main/java/org/owasp/dependencycheck/analyzer/CPEAnalyzer.java @@ -0,0 +1,790 @@ +/* + * This file is part of dependency-check-core. + * + * Dependency-check-core is free software: you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation, either version 3 of the License, or (at your option) any + * later version. + * + * Dependency-check-core is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more + * details. + * + * You should have received a copy of the GNU General Public License along with + * dependency-check-core. If not, see http://www.gnu.org/licenses/. + * + * Copyright (c) 2012 Jeremy Long. All Rights Reserved. + */ +package org.owasp.dependencycheck.analyzer; + +import java.io.IOException; +import java.io.UnsupportedEncodingException; +import java.net.URLEncoder; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Set; +import java.util.StringTokenizer; +import java.util.logging.Level; +import java.util.logging.Logger; +import org.apache.lucene.document.Document; +import org.apache.lucene.index.CorruptIndexException; +import org.apache.lucene.queryparser.classic.ParseException; +import org.apache.lucene.search.ScoreDoc; +import org.apache.lucene.search.TopDocs; +import org.owasp.dependencycheck.Engine; +import org.owasp.dependencycheck.data.lucene.LuceneUtils; +import org.owasp.dependencycheck.dependency.Dependency; +import org.owasp.dependencycheck.dependency.Evidence; +import org.owasp.dependencycheck.dependency.Evidence.Confidence; +import org.owasp.dependencycheck.dependency.EvidenceCollection; +import org.owasp.dependencycheck.data.cpe.CpeIndexReader; +import org.owasp.dependencycheck.data.cpe.Fields; +import org.owasp.dependencycheck.data.cpe.IndexEntry; +import org.owasp.dependencycheck.data.nvdcve.CveDB; +import org.owasp.dependencycheck.data.nvdcve.DatabaseException; +import org.owasp.dependencycheck.dependency.Identifier; +import org.owasp.dependencycheck.dependency.VulnerableSoftware; +import org.owasp.dependencycheck.utils.DependencyVersion; +import org.owasp.dependencycheck.utils.DependencyVersionUtil; + +/** + * CPEAnalyzer is a utility class that takes a project dependency and attempts + * to discern if there is an associated CPE. It uses the evidence contained + * within the dependency to search the Lucene index. + * + * @author Jeremy Long (jeremy.long@owasp.org) + */ +public class CPEAnalyzer implements Analyzer { + + /** + * The maximum number of query results to return. + */ + static final int MAX_QUERY_RESULTS = 25; + /** + * The weighting boost to give terms when constructing the Lucene query. + */ + static final String WEIGHTING_BOOST = "^5"; + /** + * A string representation of a regular expression defining characters + * utilized within the CPE Names. + */ + static final String CLEANSE_CHARACTER_RX = "[^A-Za-z0-9 ._-]"; + /** + * A string representation of a regular expression used to remove all but + * alpha characters. + */ + static final String CLEANSE_NONALPHA_RX = "[^A-Za-z]*"; + /** + * The additional size to add to a new StringBuilder to account for extra + * data that will be written into the string. + */ + static final int STRING_BUILDER_BUFFER = 20; + /** + * The CPE Index Reader. + */ + private CpeIndexReader cpe; + /** + * The CVE Database. + */ + private CveDB cve; + + /** + * Opens the data source. + * + * @throws IOException when the Lucene directory to be queried does not + * exist or is corrupt. + * @throws DatabaseException when the database throws an exception. This + * usually occurs when the database is in use by another process. + */ + public void open() throws IOException, DatabaseException { + cpe = new CpeIndexReader(); + cpe.open(); + cve = new CveDB(); + try { + cve.open(); + } catch (SQLException ex) { + Logger.getLogger(CPEAnalyzer.class.getName()).log(Level.FINE, null, ex); + throw new DatabaseException("Unable to open the cve db", ex); + } catch (ClassNotFoundException ex) { + Logger.getLogger(CPEAnalyzer.class.getName()).log(Level.FINE, null, ex); + throw new DatabaseException("Unable to open the cve db", ex); + } + } + + /** + * Closes the data source. + */ + @Override + public void close() { + if (cpe != null) { + cpe.close(); + } + if (cve != null) { + cve.close(); + } + } + + /** + * Returns the status of the data source - is the index open. + * + * @return true or false. + */ + public boolean isOpen() { + return (cpe != null) && cpe.isOpen(); + } + + /** + * Ensures that the Lucene index is closed. + * + * @throws Throwable when a throwable is thrown. + */ + @Override + protected void finalize() throws Throwable { + super.finalize(); + if (isOpen()) { + close(); + } + } + + /** + * Searches the data store of CPE entries, trying to identify the CPE for + * the given dependency based on the evidence contained within. The + * dependency passed in is updated with any identified CPE values. + * + * @param dependency the dependency to search for CPE entries on. + * @throws CorruptIndexException is thrown when the Lucene index is corrupt. + * @throws IOException is thrown when an IOException occurs. + * @throws ParseException is thrown when the Lucene query cannot be parsed. + */ + protected void determineCPE(Dependency dependency) throws CorruptIndexException, IOException, ParseException { + Confidence vendorConf = Confidence.HIGHEST; + Confidence productConf = Confidence.HIGHEST; + + String vendors = addEvidenceWithoutDuplicateTerms("", dependency.getVendorEvidence(), vendorConf); + String products = addEvidenceWithoutDuplicateTerms("", dependency.getProductEvidence(), productConf); + + int ctr = 0; + do { + if (!vendors.isEmpty() && !products.isEmpty()) { + final List entries = searchCPE(vendors, products, dependency.getProductEvidence().getWeighting(), + dependency.getVendorEvidence().getWeighting()); + + for (IndexEntry e : entries) { + if (verifyEntry(e, dependency)) { + final String vendor = e.getVendor(); + final String product = e.getProduct(); + determineIdentifiers(dependency, vendor, product); + } + } + } + vendorConf = reduceConfidence(vendorConf); + if (dependency.getVendorEvidence().contains(vendorConf)) { + vendors = addEvidenceWithoutDuplicateTerms(vendors, dependency.getVendorEvidence(), vendorConf); + } + productConf = reduceConfidence(productConf); + if (dependency.getProductEvidence().contains(productConf)) { + products = addEvidenceWithoutDuplicateTerms(products, dependency.getProductEvidence(), productConf); + } + } while ((++ctr) < 4); + } + + /** + * Returns the text created by concatenating the text and the values from + * the EvidenceCollection (filtered for a specific confidence). This + * attempts to prevent duplicate terms from being added.
Note, if + * the evidence is longer then 200 characters it will be truncated. + * + * @param text the base text. + * @param ec an EvidenceCollection + * @param confidenceFilter a Confidence level to filter the evidence by. + * @return the new evidence text + */ + private String addEvidenceWithoutDuplicateTerms(final String text, final EvidenceCollection ec, Confidence confidenceFilter) { + final String txt = (text == null) ? "" : text; + final StringBuilder sb = new StringBuilder(txt.length() + (20 * ec.size())); + sb.append(' ').append(txt).append(' '); + for (Evidence e : ec.iterator(confidenceFilter)) { + String value = e.getValue(); + + //hack to get around the fact that lucene does a really good job of recognizing domains and not + // splitting them. TODO - put together a better lucene analyzer specific to the domain. + if (value.startsWith("http://")) { + value = value.substring(7).replaceAll("\\.", " "); + } + if (value.startsWith("https://")) { + value = value.substring(8).replaceAll("\\.", " "); + } + if (sb.indexOf(" " + value + " ") < 0) { + sb.append(value).append(' '); + } + } + return sb.toString().trim(); + } + + /** + * Reduces the given confidence by one level. This returns LOW if the + * confidence passed in is not HIGH. + * + * @param c the confidence to reduce. + * @return One less then the confidence passed in. + */ + private Confidence reduceConfidence(final Confidence c) { + if (c == Confidence.HIGHEST) { + return Confidence.HIGH; + } else if (c == Confidence.HIGH) { + return Confidence.MEDIUM; + } else { + return Confidence.LOW; + } + } + + /** + *

Searches the Lucene CPE index to identify possible CPE entries + * associated with the supplied vendor, product, and version.

+ * + *

If either the vendorWeightings or productWeightings lists have been + * populated this data is used to add weighting factors to the search.

+ * + * @param vendor the text used to search the vendor field + * @param product the text used to search the product field + * @param vendorWeightings a list of strings to use to add weighting factors + * to the vendor field + * @param productWeightings Adds a list of strings that will be used to add + * weighting factors to the product search + * @return a list of possible CPE values + * @throws CorruptIndexException when the Lucene index is corrupt + * @throws IOException when the Lucene index is not found + * @throws ParseException when the generated query is not valid + */ + protected List searchCPE(String vendor, String product, + Set vendorWeightings, Set productWeightings) + throws CorruptIndexException, IOException, ParseException { + final ArrayList ret = new ArrayList(MAX_QUERY_RESULTS); + + final String searchString = buildSearch(vendor, product, vendorWeightings, productWeightings); + if (searchString == null) { + return ret; + } + + final TopDocs docs = cpe.search(searchString, MAX_QUERY_RESULTS); + for (ScoreDoc d : docs.scoreDocs) { + final Document doc = cpe.getDocument(d.doc); + final IndexEntry entry = new IndexEntry(); + entry.setVendor(doc.get(Fields.VENDOR)); + entry.setProduct(doc.get(Fields.PRODUCT)); + entry.setSearchScore(d.score); + if (!ret.contains(entry)) { + ret.add(entry); + } + } + return ret; + } + + /** + *

Builds a Lucene search string by properly escaping data and + * constructing a valid search query.

+ * + *

If either the possibleVendor or possibleProducts lists have been + * populated this data is used to add weighting factors to the search string + * generated.

+ * + * @param vendor text to search the vendor field + * @param product text to search the product field + * @param vendorWeighting a list of strings to apply to the vendor to boost + * the terms weight + * @param productWeightings a list of strings to apply to the product to + * boost the terms weight + * @return the Lucene query + */ + protected String buildSearch(String vendor, String product, + Set vendorWeighting, Set productWeightings) { + final String v = vendor; //.replaceAll("[^\\w\\d]", " "); + final String p = product; //.replaceAll("[^\\w\\d]", " "); + final StringBuilder sb = new StringBuilder(v.length() + p.length() + + Fields.PRODUCT.length() + Fields.VENDOR.length() + STRING_BUILDER_BUFFER); + + if (!appendWeightedSearch(sb, Fields.PRODUCT, p, productWeightings)) { + return null; + } + sb.append(" AND "); + if (!appendWeightedSearch(sb, Fields.VENDOR, v, vendorWeighting)) { + return null; + } + return sb.toString(); + } + + /** + * This method constructs a Lucene query for a given field. The searchText + * is split into separate words and if the word is within the list of + * weighted words then an additional weighting is applied to the term as it + * is appended into the query. + * + * @param sb a StringBuilder that the query text will be appended to. + * @param field the field within the Lucene index that the query is + * searching. + * @param searchText text used to construct the query. + * @param weightedText a list of terms that will be considered higher + * importance when searching. + * @return if the append was successful. + */ + private boolean appendWeightedSearch(StringBuilder sb, String field, String searchText, Set weightedText) { + sb.append(" ").append(field).append(":( "); + + final String cleanText = cleanseText(searchText); + + if ("".equals(cleanText)) { + return false; + } + + if (weightedText == null || weightedText.isEmpty()) { + LuceneUtils.appendEscapedLuceneQuery(sb, cleanText); + } else { + final StringTokenizer tokens = new StringTokenizer(cleanText); + while (tokens.hasMoreElements()) { + final String word = tokens.nextToken(); + String temp = null; + for (String weighted : weightedText) { + final String weightedStr = cleanseText(weighted); + if (equalsIgnoreCaseAndNonAlpha(word, weightedStr)) { + temp = LuceneUtils.escapeLuceneQuery(word) + WEIGHTING_BOOST; + if (!word.equalsIgnoreCase(weightedStr)) { + temp += " " + LuceneUtils.escapeLuceneQuery(weightedStr) + WEIGHTING_BOOST; + } + } + } + if (temp == null) { + temp = LuceneUtils.escapeLuceneQuery(word); + } + sb.append(" ").append(temp); + } + } + sb.append(" ) "); + return true; + } + + /** + * Removes characters from the input text that are not used within the CPE + * index. + * + * @param text is the text to remove the characters from. + * @return the text having removed some characters. + */ + private String cleanseText(String text) { + return text.replaceAll(CLEANSE_CHARACTER_RX, " "); + } + + /** + * Compares two strings after lower casing them and removing the non-alpha + * characters. + * + * @param l string one to compare. + * @param r string two to compare. + * @return whether or not the two strings are similar. + */ + private boolean equalsIgnoreCaseAndNonAlpha(String l, String r) { + if (l == null || r == null) { + return false; + } + + final String left = l.replaceAll(CLEANSE_NONALPHA_RX, ""); + final String right = r.replaceAll(CLEANSE_NONALPHA_RX, ""); + return left.equalsIgnoreCase(right); + } + + /** + * Ensures that the CPE Identified matches the dependency. This validates + * that the product, vendor, and version information for the CPE are + * contained within the dependencies evidence. + * + * @param entry a CPE entry. + * @param dependency the dependency that the CPE entries could be for. + * @return whether or not the entry is valid. + */ + private boolean verifyEntry(final IndexEntry entry, final Dependency dependency) { + boolean isValid = false; + + if (collectionContainsString(dependency.getProductEvidence(), entry.getProduct()) + && collectionContainsString(dependency.getVendorEvidence(), entry.getVendor())) { + //&& collectionContainsVersion(dependency.getVersionEvidence(), entry.getVersion()) + isValid = true; + } + return isValid; + } + + /** + * Used to determine if the EvidenceCollection contains a specific string. + * + * @param ec an EvidenceCollection + * @param text the text to search for + * @return whether or not the EvidenceCollection contains the string + */ + private boolean collectionContainsString(EvidenceCollection ec, String text) { + + // + // String[] splitText = text.split("[\\s_-]"); + // + // for (String search : splitText) { + // //final String search = text.replaceAll("[\\s_-]", "").toLowerCase(); + // if (ec.containsUsedString(search)) { + // return true; + // } + // } + // + + //TODO - likely need to change the split... not sure if this will work for CPE with special chars + final String[] words = text.split("[\\s_-]"); + final List list = new ArrayList(); + String tempWord = null; + for (String word : words) { + //single letter words should be concatonated with the next word. + // so { "m", "core", "sample" } -> { "mcore", "sample" } + if (tempWord != null) { + list.add(tempWord + word); + tempWord = null; + } else if (word.length() <= 2) { + tempWord = word; + } else { + list.add(word); + } + } +// if (tempWord != null) { +// //for now ignore any last single letter words... +// } + boolean contains = true; + for (String word : list) { + contains &= ec.containsUsedString(word); + } + return contains; + } + + /** + * Analyzes a dependency and attempts to determine if there are any CPE + * identifiers for this dependency. + * + * @param dependency The Dependency to analyze. + * @param engine The analysis engine + * @throws AnalysisException is thrown if there is an issue analyzing the + * dependency. + */ + @Override + public void analyze(Dependency dependency, Engine engine) throws AnalysisException { + try { + determineCPE(dependency); + } catch (CorruptIndexException ex) { + throw new AnalysisException("CPE Index is corrupt.", ex); + } catch (IOException ex) { + throw new AnalysisException("Failure opening the CPE Index.", ex); + } catch (ParseException ex) { + throw new AnalysisException("Unable to parse the generated Lucene query for this dependency.", ex); + } + } + + /** + * Returns true because this analyzer supports all dependency types. + * + * @return true. + */ + @Override + public Set getSupportedExtensions() { + return null; + } + + /** + * Returns the name of this analyzer. + * + * @return the name of this analyzer. + */ + @Override + public String getName() { + return "CPE Analyzer"; + } + + /** + * Returns true because this analyzer supports all dependency types. + * + * @param extension the file extension of the dependency being analyzed. + * @return true. + */ + @Override + public boolean supportsExtension(String extension) { + return true; + } + + /** + * Returns the analysis phase that this analyzer should run in. + * + * @return the analysis phase that this analyzer should run in. + */ + @Override + public AnalysisPhase getAnalysisPhase() { + return AnalysisPhase.IDENTIFIER_ANALYSIS; + } + + /** + * Opens the CPE Lucene Index. + * + * @throws Exception is thrown if there is an issue opening the index. + */ + @Override + public void initialize() throws Exception { + this.open(); + } + + /** + * Retrieves a list of CPE values from the CveDB based on the vendor and + * product passed in. The list is then validated to find only CPEs that are + * valid for the given dependency. It is possible that the CPE identified is + * a best effort "guess" based on the vendor, product, and version + * information. + * + * @param dependency the Dependency being analyzed + * @param vendor the vendor for the CPE being analyzed + * @param product the product for the CPE being analyzed + * @throws UnsupportedEncodingException is thrown if UTF-8 is not supported + */ + private void determineIdentifiers(Dependency dependency, String vendor, String product) throws UnsupportedEncodingException { + final Set cpes = cve.getCPEs(vendor, product); + DependencyVersion bestGuess = new DependencyVersion("-"); + Confidence bestGuessConf = null; + final List collected = new ArrayList(); + for (Confidence conf : Confidence.values()) { + for (Evidence evidence : dependency.getVersionEvidence().iterator(conf)) { + final DependencyVersion evVer = DependencyVersionUtil.parseVersion(evidence.getValue()); + if (evVer == null) { + continue; + } + for (VulnerableSoftware vs : cpes) { + DependencyVersion dbVer; + if (vs.getRevision() != null && !vs.getRevision().isEmpty()) { + dbVer = DependencyVersionUtil.parseVersion(vs.getVersion() + "." + vs.getRevision()); + } else { + dbVer = DependencyVersionUtil.parseVersion(vs.getVersion()); + } + if (dbVer == null //special case, no version specified - everything is vulnerable + || evVer.equals(dbVer)) { //woot exect match + final String url = String.format("http://web.nvd.nist.gov/view/vuln/search?cpe=%s", URLEncoder.encode(vs.getName(), "UTF-8")); + final IdentifierMatch match = new IdentifierMatch("cpe", vs.getName(), url, IdentifierConfidence.EXACT_MATCH, conf); + collected.add(match); + } else { + //TODO the following isn't quite right is it? need to think about this guessing game a bit more. + if (evVer.getVersionParts().size() <= dbVer.getVersionParts().size() + && evVer.matchesAtLeastThreeLevels(dbVer)) { + if (bestGuessConf == null || bestGuessConf.compareTo(conf) > 0) { + if (bestGuess.getVersionParts().size() < dbVer.getVersionParts().size()) { + bestGuess = dbVer; + bestGuessConf = conf; + } + } + } + } + } + if (bestGuessConf == null || bestGuessConf.compareTo(conf) > 0) { + if (bestGuess.getVersionParts().size() < evVer.getVersionParts().size()) { + bestGuess = evVer; + bestGuessConf = conf; + } + } + } + } + final String cpeName = String.format("cpe:/a:%s:%s:%s", vendor, product, bestGuess.toString()); + final String url = null; //String.format("http://web.nvd.nist.gov/view/vuln/search?cpe=%s", URLEncoder.encode(cpeName, "UTF-8")); + if (bestGuessConf == null) { + bestGuessConf = Confidence.LOW; + } + final IdentifierMatch match = new IdentifierMatch("cpe", cpeName, url, IdentifierConfidence.BEST_GUESS, bestGuessConf); + collected.add(match); + + Collections.sort(collected); + final IdentifierConfidence bestIdentifierQuality = collected.get(0).getConfidence(); + final Confidence bestEvidenceQuality = collected.get(0).getEvidenceConfidence(); + for (IdentifierMatch m : collected) { + if (bestIdentifierQuality.equals(m.getConfidence()) + && bestEvidenceQuality.equals(m.getEvidenceConfidence())) { + dependency.addIdentifier(m.getIdentifier()); + } + } + } + + /** + * The confidence whether the identifier is an exact match, or a best guess. + */ + private enum IdentifierConfidence { + + /** + * An exact match for the CPE. + */ + EXACT_MATCH, + /** + * A best guess for the CPE. + */ + BEST_GUESS + } + + /** + * A simple object to hold an identifier and carry information about the + * confidence in the identifier. + */ + private static class IdentifierMatch implements Comparable { + + /** + * Constructs an IdentiferMatch. + * + * @param type the type of identifier (such as CPE) + * @param value the value of the identifier + * @param url the URL of the identifier + * @param identifierConfidence the confidence in the identifier: best + * guess or exact match + * @param evidenceConfidence the confidence of the evidence used to find + * the identifier + */ + IdentifierMatch(String type, String value, String url, IdentifierConfidence identifierConfidence, Confidence evidenceConfidence) { + this.identifier = new Identifier(type, value, url); + this.confidence = identifierConfidence; + this.evidenceConfidence = evidenceConfidence; + } + // + /** + * The confidence in the evidence used to identify this match. + */ + private Confidence evidenceConfidence; + + /** + * Get the value of evidenceConfidence + * + * @return the value of evidenceConfidence + */ + public Confidence getEvidenceConfidence() { + return evidenceConfidence; + } + + /** + * Set the value of evidenceConfidence + * + * @param evidenceConfidence new value of evidenceConfidence + */ + public void setEvidenceConfidence(Confidence evidenceConfidence) { + this.evidenceConfidence = evidenceConfidence; + } + /** + * The confidence whether this is an exact match, or a best guess. + */ + private IdentifierConfidence confidence; + + /** + * Get the value of confidence. + * + * @return the value of confidence + */ + public IdentifierConfidence getConfidence() { + return confidence; + } + + /** + * Set the value of confidence. + * + * @param confidence new value of confidence + */ + public void setConfidence(IdentifierConfidence confidence) { + this.confidence = confidence; + } + /** + * The CPE identifier. + */ + private Identifier identifier; + + /** + * Get the value of identifier. + * + * @return the value of identifier + */ + public Identifier getIdentifier() { + return identifier; + } + + /** + * Set the value of identifier. + * + * @param identifier new value of identifier + */ + public void setIdentifier(Identifier identifier) { + this.identifier = identifier; + } + // + // + + /** + * Standard toString() implementation. + * + * @return the string representation of the object + */ + @Override + public String toString() { + return "IdentifierMatch{" + "evidenceConfidence=" + evidenceConfidence + + ", confidence=" + confidence + ", identifier=" + identifier + '}'; + } + + /** + * Standard hashCode() implementation. + * + * @return the hashCode + */ + @Override + public int hashCode() { + int hash = 5; + hash = 97 * hash + (this.evidenceConfidence != null ? this.evidenceConfidence.hashCode() : 0); + hash = 97 * hash + (this.confidence != null ? this.confidence.hashCode() : 0); + hash = 97 * hash + (this.identifier != null ? this.identifier.hashCode() : 0); + return hash; + } + + /** + * Standard equals implementation. + * + * @param obj the object to compare + * @return true if the objects are equal, otherwise false + */ + @Override + public boolean equals(Object obj) { + if (obj == null) { + return false; + } + if (getClass() != obj.getClass()) { + return false; + } + final IdentifierMatch other = (IdentifierMatch) obj; + if (this.evidenceConfidence != other.evidenceConfidence) { + return false; + } + if (this.confidence != other.confidence) { + return false; + } + if (this.identifier != other.identifier && (this.identifier == null || !this.identifier.equals(other.identifier))) { + return false; + } + return true; + } + // + + /** + * Standard implementation of compareTo that compares identifier + * confidence, evidence confidence, and then the identifier. + * + * @param o the IdentifierMatch to compare to + * @return the natural ordering of IdentifierMatch + */ + @Override + public int compareTo(IdentifierMatch o) { + int conf = this.confidence.compareTo(o.confidence); + if (conf == 0) { + conf = this.evidenceConfidence.compareTo(o.evidenceConfidence); + if (conf == 0) { + conf = identifier.compareTo(o.identifier); + } + } + return conf; + } + } +} diff --git a/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/cpe/BaseIndex.java b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/cpe/BaseIndex.java new file mode 100644 index 000000000..1f6e84d91 --- /dev/null +++ b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/cpe/BaseIndex.java @@ -0,0 +1,108 @@ +/* + * This file is part of dependency-check-core. + * + * Dependency-check-core is free software: you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation, either version 3 of the License, or (at your option) any + * later version. + * + * Dependency-check-core is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more + * details. + * + * You should have received a copy of the GNU General Public License along with + * dependency-check-core. If not, see http://www.gnu.org/licenses/. + * + * Copyright (c) 2013 Jeremy Long. All Rights Reserved. + */ +package org.owasp.dependencycheck.data.cpe; + +import java.io.File; +import java.io.IOException; +import java.util.logging.Level; +import java.util.logging.Logger; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.FSDirectory; +import org.owasp.dependencycheck.utils.Settings; + +/** + * The Base Index class used to access the CPE Index. + * + * @author Jeremy Long (jeremy.long@owasp.org) + */ +public abstract class BaseIndex { + + /** + * The Lucene directory containing the index. + */ + protected Directory directory; + /** + * Indicates whether or not the Lucene Index is open. + */ + protected boolean indexOpen = false; + + /** + * Opens the CPE Index. + * + * @throws IOException is thrown if an IOException occurs opening the index. + */ + public void open() throws IOException { + directory = this.openDirectory(); + indexOpen = true; + } + + /** + * Closes the CPE Index. + */ + public void close() { + try { + directory.close(); + } catch (IOException ex) { + final String msg = "Unable to update database due to an IO error."; + Logger.getLogger(BaseIndex.class.getName()).log(Level.SEVERE, msg); + Logger.getLogger(BaseIndex.class.getName()).log(Level.FINE, null, ex); + } finally { + directory = null; + } + indexOpen = false; + + } + + /** + * Returns the status of the data source - is the index open. + * + * @return true or false. + */ + public boolean isOpen() { + return indexOpen; + } + + /** + * Returns the Lucene directory object for the CPE Index. + * + * @return the Lucene Directory object for the CPE Index. + * @throws IOException is thrown if an IOException occurs. + */ + protected Directory openDirectory() throws IOException { + final File path = getDataDirectory(); + return FSDirectory.open(path); + } + + /** + * Retrieves the directory that the JAR file exists in so that we can ensure + * we always use a common data directory. + * + * @return the data directory for this index. + * @throws IOException is thrown if an IOException occurs of course... + */ + public static File getDataDirectory() throws IOException { + final File path = Settings.getFile(Settings.KEYS.CPE_DATA_DIRECTORY); + if (!path.exists()) { + if (!path.mkdirs()) { + throw new IOException("Unable to create CPE Data directory"); + } + } + return path; + } +} diff --git a/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/cpe/CpeIndexReader.java b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/cpe/CpeIndexReader.java new file mode 100644 index 000000000..dfee343e5 --- /dev/null +++ b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/cpe/CpeIndexReader.java @@ -0,0 +1,179 @@ +/* + * This file is part of dependency-check-core. + * + * Dependency-check-core is free software: you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation, either version 3 of the License, or (at your option) any + * later version. + * + * Dependency-check-core is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more + * details. + * + * You should have received a copy of the GNU General Public License along with + * dependency-check-core. If not, see http://www.gnu.org/licenses/. + * + * Copyright (c) 2013 Jeremy Long. All Rights Reserved. + */ +package org.owasp.dependencycheck.data.cpe; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; +import java.util.logging.Level; +import java.util.logging.Logger; +import org.apache.lucene.analysis.Analyzer; +import org.apache.lucene.analysis.core.KeywordAnalyzer; +import org.apache.lucene.analysis.miscellaneous.PerFieldAnalyzerWrapper; +import org.apache.lucene.document.Document; +import org.apache.lucene.index.CorruptIndexException; +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.queryparser.classic.ParseException; +import org.apache.lucene.queryparser.classic.QueryParser; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.TopDocs; +import org.apache.lucene.util.Version; +import org.owasp.dependencycheck.data.lucene.FieldAnalyzer; +import org.owasp.dependencycheck.data.lucene.SearchFieldAnalyzer; + +/** + * + * @author Jeremy Long (jeremy.long@owasp.org) + */ +public class CpeIndexReader extends BaseIndex { + + /** + * The Lucene IndexReader. + */ + private IndexReader indexReader; + /** + * The Lucene IndexSearcher. + */ + private IndexSearcher indexSearcher; + /** + * The Lucene Analyzer used for Searching. + */ + private Analyzer searchingAnalyzer; + /** + * The Lucene QueryParser used for Searching. + */ + private QueryParser queryParser; + /** + * The search field analyzer for the product field. + */ + private SearchFieldAnalyzer productSearchFieldAnalyzer; + /** + * The search field analyzer for the vendor field. + */ + private SearchFieldAnalyzer vendorSearchFieldAnalyzer; + + /** + * Opens the CPE Index. + * + * @throws IOException is thrown if an IOException occurs opening the index. + */ + @Override + public void open() throws IOException { + //TODO add spinlock (shared) + super.open(); + indexReader = DirectoryReader.open(directory); + indexSearcher = new IndexSearcher(indexReader); + searchingAnalyzer = createSearchingAnalyzer(); + queryParser = new QueryParser(Version.LUCENE_43, Fields.DOCUMENT_KEY, searchingAnalyzer); + } + + /** + * Closes the CPE Index. + */ + @Override + public void close() { + //TODO remove spinlock (shared) + if (searchingAnalyzer != null) { + searchingAnalyzer.close(); + searchingAnalyzer = null; + } + if (indexReader != null) { + try { + indexReader.close(); + } catch (IOException ex) { + Logger.getLogger(CpeIndexReader.class.getName()).log(Level.FINEST, null, ex); + } + indexReader = null; + } + queryParser = null; + indexSearcher = null; + super.close(); + } + + /** + * Searches the index using the given search string. + * + * @param searchString the query text + * @param maxQueryResults the maximum number of documents to return + * @return the TopDocs found by the search + * @throws ParseException thrown when the searchString is invalid + * @throws IOException is thrown if there is an issue with the underlying + * Index + */ + public TopDocs search(String searchString, int maxQueryResults) throws ParseException, IOException { + final Query query = queryParser.parse(searchString); + return indexSearcher.search(query, maxQueryResults); + } + + /** + * Searches the index using the given query. + * + * @param query the query used to search the index + * @param maxQueryResults the max number of results to return + * @return the TopDocs found be the query + * @throws CorruptIndexException thrown if the Index is corrupt + * @throws IOException thrown if there is an IOException + */ + public TopDocs search(Query query, int maxQueryResults) throws CorruptIndexException, IOException { + resetSearchingAnalyzer(); + return indexSearcher.search(query, maxQueryResults); + } + + /** + * Retrieves a document from the Index. + * + * @param documentId the id of the document to retrieve + * @return the Document + * @throws IOException thrown if there is an IOException + */ + public Document getDocument(int documentId) throws IOException { + return indexSearcher.doc(documentId); + } + + /** + * Creates an Analyzer for searching the CPE Index. + * + * @return the CPE Analyzer. + */ + @SuppressWarnings("unchecked") + private Analyzer createSearchingAnalyzer() { + final Map fieldAnalyzers = new HashMap(); + fieldAnalyzers.put(Fields.DOCUMENT_KEY, new KeywordAnalyzer()); + productSearchFieldAnalyzer = new SearchFieldAnalyzer(Version.LUCENE_43); + vendorSearchFieldAnalyzer = new SearchFieldAnalyzer(Version.LUCENE_43); + fieldAnalyzers.put(Fields.PRODUCT, productSearchFieldAnalyzer); + fieldAnalyzers.put(Fields.VENDOR, vendorSearchFieldAnalyzer); + + return new PerFieldAnalyzerWrapper(new FieldAnalyzer(Version.LUCENE_43), fieldAnalyzers); + } + + /** + * Resets the searching analyzers + */ + private void resetSearchingAnalyzer() { + if (productSearchFieldAnalyzer != null) { + productSearchFieldAnalyzer.clear(); + } + if (vendorSearchFieldAnalyzer != null) { + vendorSearchFieldAnalyzer.clear(); + } + } +} diff --git a/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/cpe/CpeIndexWriter.java b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/cpe/CpeIndexWriter.java new file mode 100644 index 000000000..cc4c194c4 --- /dev/null +++ b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/cpe/CpeIndexWriter.java @@ -0,0 +1,149 @@ +/* + * This file is part of dependency-check-core. + * + * Dependency-check-core is free software: you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation, either version 3 of the License, or (at your option) any + * later version. + * + * Dependency-check-core is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more + * details. + * + * You should have received a copy of the GNU General Public License along with + * dependency-check-core. If not, see http://www.gnu.org/licenses/. + * + * Copyright (c) 2013 Jeremy Long. All Rights Reserved. + */ +package org.owasp.dependencycheck.data.cpe; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; +import java.util.logging.Level; +import java.util.logging.Logger; +import org.apache.lucene.analysis.Analyzer; +import org.apache.lucene.analysis.core.KeywordAnalyzer; +import org.apache.lucene.analysis.miscellaneous.PerFieldAnalyzerWrapper; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; +import org.apache.lucene.document.StringField; +import org.apache.lucene.document.TextField; +import org.apache.lucene.index.CorruptIndexException; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.IndexWriterConfig; +import org.apache.lucene.index.Term; +import org.apache.lucene.util.Version; +import org.owasp.dependencycheck.data.lucene.FieldAnalyzer; + +/** + * + * @author Jeremy Long (jeremy.long@owasp.org) + */ +public class CpeIndexWriter extends BaseIndex { + + /** + * The IndexWriter for the Lucene index. + */ + private IndexWriter indexWriter; + /** + * The Lucene Analyzer used for Indexing. + */ + private Analyzer indexingAnalyzer; + + /** + * Opens the CPE Index. + * + * @throws IOException is thrown if an IOException occurs opening the index. + */ + @Override + public void open() throws IOException { + //TODO add spinlock + super.open(); + indexingAnalyzer = createIndexingAnalyzer(); + final IndexWriterConfig conf = new IndexWriterConfig(Version.LUCENE_43, indexingAnalyzer); + indexWriter = new IndexWriter(directory, conf); + } + + /** + * Closes the CPE Index. + */ + @Override + public void close() { + //TODO remove spinlock + if (indexWriter != null) { + commit(); + try { + indexWriter.close(true); + } catch (CorruptIndexException ex) { + final String msg = "Unable to update database, there is a corrupt index."; + Logger.getLogger(CpeIndexWriter.class.getName()).log(Level.SEVERE, msg); + Logger.getLogger(CpeIndexWriter.class.getName()).log(Level.FINE, null, ex); + } catch (IOException ex) { + final String msg = "Unable to update database due to an IO error."; + Logger.getLogger(CpeIndexWriter.class.getName()).log(Level.SEVERE, msg); + Logger.getLogger(CpeIndexWriter.class.getName()).log(Level.FINE, null, ex); + } finally { + indexWriter = null; + } + } + if (indexingAnalyzer != null) { + indexingAnalyzer.close(); + indexingAnalyzer = null; + } + super.close(); + } + + /** + * Commits any pending changes. + */ + public void commit() { + if (indexWriter != null) { + try { + indexWriter.forceMerge(1); + indexWriter.commit(); + } catch (CorruptIndexException ex) { + final String msg = "Unable to update database, there is a corrupt index."; + Logger.getLogger(CpeIndexWriter.class.getName()).log(Level.SEVERE, msg); + Logger.getLogger(CpeIndexWriter.class.getName()).log(Level.FINE, null, ex); + } catch (IOException ex) { + final String msg = "Unable to update database due to an IO error."; + Logger.getLogger(CpeIndexWriter.class.getName()).log(Level.SEVERE, msg); + Logger.getLogger(CpeIndexWriter.class.getName()).log(Level.FINE, null, ex); + } + } + } + + /** + * Creates the indexing analyzer for the CPE Index. + * + * @return the CPE Analyzer. + */ + @SuppressWarnings("unchecked") + private Analyzer createIndexingAnalyzer() { + final Map fieldAnalyzers = new HashMap(); + fieldAnalyzers.put(Fields.DOCUMENT_KEY, new KeywordAnalyzer()); + return new PerFieldAnalyzerWrapper(new FieldAnalyzer(Version.LUCENE_43), fieldAnalyzers); + } + + /** + * Saves a CPE IndexEntry into the Lucene index. + * + * @param entry a CPE entry. + * @throws CorruptIndexException is thrown if the index is corrupt. + * @throws IOException is thrown if an IOException occurs. + */ + public void saveEntry(IndexEntry entry) throws CorruptIndexException, IOException { + final Document doc = new Document(); + final Field documentKey = new StringField(Fields.DOCUMENT_KEY, entry.getDocumentId(), Field.Store.NO); + final Field vendor = new TextField(Fields.VENDOR, entry.getVendor(), Field.Store.YES); + final Field product = new TextField(Fields.PRODUCT, entry.getProduct(), Field.Store.YES); + doc.add(documentKey); + doc.add(vendor); + doc.add(product); + + final Term term = new Term(Fields.DOCUMENT_KEY, entry.getDocumentId()); + indexWriter.updateDocument(term, doc); + } +} diff --git a/dependency-check-core/src/test/java/org/owasp/dependencycheck/analyzer/CPEAnalyzerTest.java b/dependency-check-core/src/test/java/org/owasp/dependencycheck/analyzer/CPEAnalyzerTest.java new file mode 100644 index 000000000..1b8f805cb --- /dev/null +++ b/dependency-check-core/src/test/java/org/owasp/dependencycheck/analyzer/CPEAnalyzerTest.java @@ -0,0 +1,256 @@ +/* + * This file is part of dependency-check-core. + * + * Dependency-check-core is free software: you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation, either version 3 of the License, or (at your option) any + * later version. + * + * Dependency-check-core is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more + * details. + * + * You should have received a copy of the GNU General Public License along with + * dependency-check-core. If not, see http://www.gnu.org/licenses/. + * + * Copyright (c) 2012 Jeremy Long. All Rights Reserved. + */ +package org.owasp.dependencycheck.analyzer; + +import org.owasp.dependencycheck.data.cpe.IndexEntry; +import org.owasp.dependencycheck.analyzer.CPEAnalyzer; +import java.io.File; +import java.io.IOException; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import org.apache.lucene.index.CorruptIndexException; +import org.apache.lucene.queryparser.classic.ParseException; +import org.junit.After; +import org.junit.AfterClass; +import org.owasp.dependencycheck.dependency.Dependency; +import org.owasp.dependencycheck.analyzer.JarAnalyzer; +import org.junit.Assert; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; +import org.owasp.dependencycheck.analyzer.FalsePositiveAnalyzer; +import org.owasp.dependencycheck.analyzer.FileNameAnalyzer; +import org.owasp.dependencycheck.analyzer.HintAnalyzer; +import org.owasp.dependencycheck.data.cpe.BaseIndexTestCase; +import org.owasp.dependencycheck.data.cpe.IndexEntry; +import static org.owasp.dependencycheck.data.cpe.BaseIndexTestCase.ensureIndexExists; +import org.owasp.dependencycheck.dependency.Identifier; + +/** + * + * @author Jeremy Long (jeremy.long@owasp.org) + */ +public class CPEAnalyzerTest extends BaseIndexTestCase { + + @BeforeClass + public static void setUpClass() throws Exception { + } + + @AfterClass + public static void tearDownClass() throws Exception { + } + + @Before + public void setUp() throws Exception { + super.setUp(); + } + + @After + public void tearDown() throws Exception { + super.tearDown(); + } + + /** + * Tests of buildSearch of class CPEAnalyzer. + * + * @throws IOException is thrown when an IO Exception occurs. + * @throws CorruptIndexException is thrown when the index is corrupt. + * @throws ParseException is thrown when a parse exception occurs + */ + @Test + public void testBuildSearch() throws IOException, CorruptIndexException, ParseException { + Set productWeightings = new HashSet(1); + productWeightings.add("struts2"); + + Set vendorWeightings = new HashSet(1); + vendorWeightings.add("apache"); + + String vendor = "apache software foundation"; + String product = "struts 2 core"; + String version = "2.1.2"; + CPEAnalyzer instance = new CPEAnalyzer(); + + String queryText = instance.buildSearch(vendor, product, null, null); + String expResult = " product:( struts 2 core ) AND vendor:( apache software foundation ) "; + Assert.assertTrue(expResult.equals(queryText)); + + queryText = instance.buildSearch(vendor, product, null, productWeightings); + expResult = " product:( struts^5 struts2^5 2 core ) AND vendor:( apache software foundation ) "; + Assert.assertTrue(expResult.equals(queryText)); + + queryText = instance.buildSearch(vendor, product, vendorWeightings, null); + expResult = " product:( struts 2 core ) AND vendor:( apache^5 software foundation ) "; + Assert.assertTrue(expResult.equals(queryText)); + + queryText = instance.buildSearch(vendor, product, vendorWeightings, productWeightings); + expResult = " product:( struts^5 struts2^5 2 core ) AND vendor:( apache^5 software foundation ) "; + Assert.assertTrue(expResult.equals(queryText)); + } + + /** + * Test of open method, of class CPEAnalyzer. + * + * @throws Exception is thrown when an exception occurs + */ + @Test + public void testOpen() throws Exception { + CPEAnalyzer instance = new CPEAnalyzer(); + Assert.assertFalse(instance.isOpen()); + instance.open(); + Assert.assertTrue(instance.isOpen()); + instance.close(); + Assert.assertFalse(instance.isOpen()); + } + + /** + * Test of determineCPE method, of class CPEAnalyzer. + * + * @throws Exception is thrown when an exception occurs + */ + @Test + public void testDetermineCPE_full() throws Exception { + callDetermineCPE_full("spring-context-support-2.5.5.jar", "cpe:/a:vmware:springsource_spring_framework:2.5.5"); + callDetermineCPE_full("spring-core-3.0.0.RELEASE.jar", "cpe:/a:vmware:springsource_spring_framework:3.0.0"); + callDetermineCPE_full("org.mortbay.jetty.jar", "cpe:/a:mortbay_jetty:jetty:4.2"); + callDetermineCPE_full("jaxb-xercesImpl-1.5.jar", null); + callDetermineCPE_full("ehcache-core-2.2.0.jar", null); + } + + /** + * Test of determineCPE method, of class CPEAnalyzer. + * + * @throws Exception is thrown when an exception occurs + */ + public void callDetermineCPE_full(String depName, String expResult) throws Exception { + + File file = new File(this.getClass().getClassLoader().getResource(depName).getPath()); + Dependency dep = new Dependency(file); + + FileNameAnalyzer fnAnalyzer = new FileNameAnalyzer(); + fnAnalyzer.analyze(dep, null); + + JarAnalyzer jarAnalyzer = new JarAnalyzer(); + jarAnalyzer.analyze(dep, null); + HintAnalyzer hAnalyzer = new HintAnalyzer(); + hAnalyzer.analyze(dep, null); + + + CPEAnalyzer instance = new CPEAnalyzer(); + instance.open(); + instance.analyze(dep, null); + instance.close(); + FalsePositiveAnalyzer fp = new FalsePositiveAnalyzer(); + fp.analyze(dep, null); + +// for (Identifier i : dep.getIdentifiers()) { +// System.out.println(i.getValue()); +// } + if (expResult != null) { + Identifier expIdentifier = new Identifier("cpe", expResult, expResult); + Assert.assertTrue("Incorrect match: { dep:'" + dep.getFileName() + "' }", dep.getIdentifiers().contains(expIdentifier)); + } else { + Assert.assertTrue("Match found when an Identifier should not have been found: { dep:'" + dep.getFileName() + "' }", dep.getIdentifiers().isEmpty()); + } + } + + /** + * Test of determineCPE method, of class CPEAnalyzer. + * + * @throws Exception is thrown when an exception occurs + */ + @Test + public void testDetermineCPE() throws Exception { + File file = new File(this.getClass().getClassLoader().getResource("struts2-core-2.1.2.jar").getPath()); + //File file = new File(this.getClass().getClassLoader().getResource("axis2-adb-1.4.1.jar").getPath()); + Dependency struts = new Dependency(file); + + FileNameAnalyzer fnAnalyzer = new FileNameAnalyzer(); + fnAnalyzer.analyze(struts, null); + + JarAnalyzer jarAnalyzer = new JarAnalyzer(); + jarAnalyzer.analyze(struts, null); + + + File fileCommonValidator = new File(this.getClass().getClassLoader().getResource("commons-validator-1.4.0.jar").getPath()); + Dependency commonValidator = new Dependency(fileCommonValidator); + jarAnalyzer.analyze(commonValidator, null); + + File fileSpring = new File(this.getClass().getClassLoader().getResource("spring-core-2.5.5.jar").getPath()); + Dependency spring = new Dependency(fileSpring); + jarAnalyzer.analyze(spring, null); + + File fileSpring3 = new File(this.getClass().getClassLoader().getResource("spring-core-3.0.0.RELEASE.jar").getPath()); + Dependency spring3 = new Dependency(fileSpring3); + jarAnalyzer.analyze(spring3, null); + + CPEAnalyzer instance = new CPEAnalyzer(); + instance.open(); + instance.determineCPE(commonValidator); + instance.determineCPE(struts); + instance.determineCPE(spring); + instance.determineCPE(spring3); + instance.close(); + + String expResult = "cpe:/a:apache:struts:2.1.2"; + Identifier expIdentifier = new Identifier("cpe", expResult, expResult); + String expResultSpring = "cpe:/a:springsource:spring_framework:2.5.5"; + String expResultSpring3 = "cpe:/a:vmware:springsource_spring_framework:3.0.0"; + + Assert.assertTrue("Apache Common Validator - found an identifier?", commonValidator.getIdentifiers().isEmpty()); + Assert.assertTrue("Incorrect match size - struts", struts.getIdentifiers().size() >= 1); + Assert.assertTrue("Incorrect match - struts", struts.getIdentifiers().contains(expIdentifier)); + Assert.assertTrue("Incorrect match size - spring3 - " + spring3.getIdentifiers().size(), spring3.getIdentifiers().size() >= 1); + + //the following two only work if the HintAnalyzer is used. + //Assert.assertTrue("Incorrect match size - spring", spring.getIdentifiers().size() == 1); + //Assert.assertTrue("Incorrect match - spring", spring.getIdentifiers().get(0).getValue().equals(expResultSpring)); + + } + + /** + * Test of searchCPE method, of class CPEAnalyzer. + * + * @throws Exception is thrown when an exception occurs + */ + @Test + public void testSearchCPE() throws Exception { + String vendor = "apache software foundation"; + String product = "struts 2 core"; + String version = "2.1.2"; + String expResult = "cpe:/a:apache:struts:2.1.2"; + + CPEAnalyzer instance = new CPEAnalyzer(); + instance.open(); + + //TODO - yeah, not a very good test as the results are the same with or without weighting... + Set productWeightings = new HashSet(1); + productWeightings.add("struts2"); + + Set vendorWeightings = new HashSet(1); + vendorWeightings.add("apache"); + + List result = instance.searchCPE(vendor, product, productWeightings, vendorWeightings); + //TODO fix this assert + //Assert.assertEquals(expResult, result.get(0).getName()); + + + instance.close(); + } +} diff --git a/dependency-check-core/src/test/java/org/owasp/dependencycheck/data/cpe/BaseIndexTest.java b/dependency-check-core/src/test/java/org/owasp/dependencycheck/data/cpe/BaseIndexTest.java new file mode 100644 index 000000000..cf3c78eff --- /dev/null +++ b/dependency-check-core/src/test/java/org/owasp/dependencycheck/data/cpe/BaseIndexTest.java @@ -0,0 +1,63 @@ +/* + * This file is part of dependency-check-core. + * + * Dependency-check-core is free software: you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation, either version 3 of the License, or (at your option) any + * later version. + * + * Dependency-check-core is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more + * details. + * + * You should have received a copy of the GNU General Public License along with + * dependency-check-core. If not, see http://www.gnu.org/licenses/. + * + * Copyright (c) 2012 Jeremy Long. All Rights Reserved. + */ +package org.owasp.dependencycheck.data.cpe; + +import org.owasp.dependencycheck.data.cpe.BaseIndex; +import java.io.File; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; +import static org.junit.Assert.*; + +/** + * + * @author Jeremy Long (jeremy.long@owasp.org) + */ +public class BaseIndexTest { + + @BeforeClass + public static void setUpClass() throws Exception { + } + + @AfterClass + public static void tearDownClass() throws Exception { + } + + @Before + public void setUp() { + } + + @After + public void tearDown() { + } + + /** + * Test of getDataDirectory method, of class BaseIndex. + * + * @throws Exception + */ + @Test + public void testGetDataDirectory() throws Exception { + String file = BaseIndex.getDataDirectory().getPath(); + String exp = File.separatorChar + "target" + File.separatorChar + "data" + File.separatorChar + "cpe"; + assertTrue(file.contains(exp)); + } +} From f90b168fddc47814357ff0a203312dd5de6416e7 Mon Sep 17 00:00:00 2001 From: Jeremy Long Date: Sat, 24 Aug 2013 16:23:57 -0400 Subject: [PATCH 10/50] commiting deletions and no-ops Former-commit-id: 920f16fc7fb20d5536c62e1ccc180fa248f76802 --- .../dependencycheck/data/cpe/CPEAnalyzer.java | 794 ------------------ .../owasp/dependencycheck/data/cpe/Index.java | 170 ---- .../data/lucene/AbstractIndex.java | 341 -------- .../data/nvdcve/xml/DatabaseUpdater.java | 11 +- .../data/nvdcve/xml/NvdCve20Handler.java | 8 +- .../data/cpe/CPEAnalyzerTest.java | 254 ------ .../dependencycheck/data/cpe/IndexTest.java | 103 --- 7 files changed, 10 insertions(+), 1671 deletions(-) delete mode 100644 dependency-check-core/src/main/java/org/owasp/dependencycheck/data/cpe/CPEAnalyzer.java delete mode 100644 dependency-check-core/src/main/java/org/owasp/dependencycheck/data/cpe/Index.java delete mode 100644 dependency-check-core/src/main/java/org/owasp/dependencycheck/data/lucene/AbstractIndex.java delete mode 100644 dependency-check-core/src/test/java/org/owasp/dependencycheck/data/cpe/CPEAnalyzerTest.java delete mode 100644 dependency-check-core/src/test/java/org/owasp/dependencycheck/data/cpe/IndexTest.java diff --git a/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/cpe/CPEAnalyzer.java b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/cpe/CPEAnalyzer.java deleted file mode 100644 index dbd4b9f0f..000000000 --- a/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/cpe/CPEAnalyzer.java +++ /dev/null @@ -1,794 +0,0 @@ -/* - * This file is part of dependency-check-core. - * - * Dependency-check-core is free software: you can redistribute it and/or modify it - * under the terms of the GNU General Public License as published by the Free - * Software Foundation, either version 3 of the License, or (at your option) any - * later version. - * - * Dependency-check-core is distributed in the hope that it will be useful, but - * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more - * details. - * - * You should have received a copy of the GNU General Public License along with - * dependency-check-core. If not, see http://www.gnu.org/licenses/. - * - * Copyright (c) 2012 Jeremy Long. All Rights Reserved. - */ -package org.owasp.dependencycheck.data.cpe; - -import java.io.IOException; -import java.io.UnsupportedEncodingException; -import java.net.URLEncoder; -import java.sql.SQLException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.Set; -import java.util.StringTokenizer; -import java.util.logging.Level; -import java.util.logging.Logger; -import org.apache.lucene.document.Document; -import org.apache.lucene.index.CorruptIndexException; -import org.apache.lucene.queryparser.classic.ParseException; -import org.apache.lucene.search.ScoreDoc; -import org.apache.lucene.search.TopDocs; -import org.owasp.dependencycheck.Engine; -import org.owasp.dependencycheck.analyzer.AnalysisException; -import org.owasp.dependencycheck.analyzer.AnalysisPhase; -import org.owasp.dependencycheck.data.lucene.LuceneUtils; -import org.owasp.dependencycheck.dependency.Dependency; -import org.owasp.dependencycheck.dependency.Evidence; -import org.owasp.dependencycheck.dependency.Evidence.Confidence; -import org.owasp.dependencycheck.dependency.EvidenceCollection; -import org.owasp.dependencycheck.analyzer.Analyzer; -import org.owasp.dependencycheck.data.nvdcve.CveDB; -import org.owasp.dependencycheck.data.nvdcve.DatabaseException; -import org.owasp.dependencycheck.dependency.Identifier; -import org.owasp.dependencycheck.dependency.VulnerableSoftware; -import org.owasp.dependencycheck.utils.DependencyVersion; -import org.owasp.dependencycheck.utils.DependencyVersionUtil; - -/** - * CPEAnalyzer is a utility class that takes a project dependency and attempts - * to discern if there is an associated CPE. It uses the evidence contained - * within the dependency to search the Lucene index. - * - * @author Jeremy Long (jeremy.long@owasp.org) - */ -public class CPEAnalyzer implements Analyzer { - - /** - * The maximum number of query results to return. - */ - static final int MAX_QUERY_RESULTS = 25; - /** - * The weighting boost to give terms when constructing the Lucene query. - */ - static final String WEIGHTING_BOOST = "^5"; - /** - * A string representation of a regular expression defining characters - * utilized within the CPE Names. - */ - static final String CLEANSE_CHARACTER_RX = "[^A-Za-z0-9 ._-]"; - /** - * A string representation of a regular expression used to remove all but - * alpha characters. - */ - static final String CLEANSE_NONALPHA_RX = "[^A-Za-z]*"; - /** - * The additional size to add to a new StringBuilder to account for extra - * data that will be written into the string. - */ - static final int STRING_BUILDER_BUFFER = 20; - /** - * The CPE Index. - */ - private Index cpe; - /** - * The CVE Database. - */ - private CveDB cve; - - /** - * Opens the data source. - * - * @throws IOException when the Lucene directory to be queried does not - * exist or is corrupt. - * @throws DatabaseException when the database throws an exception. This - * usually occurs when the database is in use by another process. - */ - public void open() throws IOException, DatabaseException { - cpe = new Index(); - cpe.open(); - cve = new CveDB(); - try { - cve.open(); - } catch (SQLException ex) { - Logger.getLogger(CPEAnalyzer.class.getName()).log(Level.FINE, null, ex); - throw new DatabaseException("Unable to open the cve db", ex); - } catch (ClassNotFoundException ex) { - Logger.getLogger(CPEAnalyzer.class.getName()).log(Level.FINE, null, ex); - throw new DatabaseException("Unable to open the cve db", ex); - } - } - - /** - * Closes the data source. - */ - @Override - public void close() { - cpe.close(); - cve.close(); - } - - /** - * Returns the status of the data source - is the index open. - * - * @return true or false. - */ - public boolean isOpen() { - return (cpe != null) && cpe.isOpen(); - } - - /** - * Ensures that the Lucene index is closed. - * - * @throws Throwable when a throwable is thrown. - */ - @Override - protected void finalize() throws Throwable { - super.finalize(); - if (isOpen()) { - close(); - } - } - - /** - * Searches the data store of CPE entries, trying to identify the CPE for - * the given dependency based on the evidence contained within. The - * dependency passed in is updated with any identified CPE values. - * - * @param dependency the dependency to search for CPE entries on. - * @throws CorruptIndexException is thrown when the Lucene index is corrupt. - * @throws IOException is thrown when an IOException occurs. - * @throws ParseException is thrown when the Lucene query cannot be parsed. - */ - protected void determineCPE(Dependency dependency) throws CorruptIndexException, IOException, ParseException { - Confidence vendorConf = Confidence.HIGHEST; - Confidence productConf = Confidence.HIGHEST; - - String vendors = addEvidenceWithoutDuplicateTerms("", dependency.getVendorEvidence(), vendorConf); - String products = addEvidenceWithoutDuplicateTerms("", dependency.getProductEvidence(), productConf); - - //boolean found = false; - int ctr = 0; - do { - if (!vendors.isEmpty() && !products.isEmpty()) { - final List entries = searchCPE(vendors, products, dependency.getProductEvidence().getWeighting(), - dependency.getVendorEvidence().getWeighting()); - - for (IndexEntry e : entries) { - if (verifyEntry(e, dependency)) { - //found = true; // we found a vendor/product pair. Now find version from the cve db. - final String vendor = e.getVendor(); - final String product = e.getProduct(); - // cve.getVersions(vendor, product); - determineIdentifiers(dependency, vendor, product); - } - } - } - //if (!found) { - vendorConf = reduceConfidence(vendorConf); - if (dependency.getVendorEvidence().contains(vendorConf)) { - //vendors += " " + dependency.getVendorEvidence().toString(vendorConf); - vendors = addEvidenceWithoutDuplicateTerms(vendors, dependency.getVendorEvidence(), vendorConf); - } - productConf = reduceConfidence(productConf); - if (dependency.getProductEvidence().contains(productConf)) { - //products += " " + dependency.getProductEvidence().toString(productConf); - products = addEvidenceWithoutDuplicateTerms(products, dependency.getProductEvidence(), productConf); - } - //} - //} while (!found && (++ctr) < 4); - } while ((++ctr) < 4); - } - - /** - * Returns the text created by concatenating the text and the values from - * the EvidenceCollection (filtered for a specific confidence). This - * attempts to prevent duplicate terms from being added.
Note, if - * the evidence is longer then 200 characters it will be truncated. - * - * @param text the base text. - * @param ec an EvidenceCollection - * @param confidenceFilter a Confidence level to filter the evidence by. - * @return the new evidence text - */ - private String addEvidenceWithoutDuplicateTerms(final String text, final EvidenceCollection ec, Confidence confidenceFilter) { - final String txt = (text == null) ? "" : text; - final StringBuilder sb = new StringBuilder(txt.length() + (20 * ec.size())); - sb.append(' ').append(txt).append(' '); - for (Evidence e : ec.iterator(confidenceFilter)) { - String value = e.getValue(); - - //hack to get around the fact that lucene does a really good job of recognizing domains and not - // splitting them. TODO - put together a better lucene analyzer specific to the domain. - if (value.startsWith("http://")) { - value = value.substring(7).replaceAll("\\.", " "); - } - if (value.startsWith("https://")) { - value = value.substring(8).replaceAll("\\.", " "); - } - if (sb.indexOf(" " + value + " ") < 0) { - sb.append(value).append(' '); - } - } - return sb.toString().trim(); - } - - /** - * Reduces the given confidence by one level. This returns LOW if the - * confidence passed in is not HIGH. - * - * @param c the confidence to reduce. - * @return One less then the confidence passed in. - */ - private Confidence reduceConfidence(final Confidence c) { - if (c == Confidence.HIGHEST) { - return Confidence.HIGH; - } else if (c == Confidence.HIGH) { - return Confidence.MEDIUM; - } else { - return Confidence.LOW; - } - } - - /** - *

Searches the Lucene CPE index to identify possible CPE entries - * associated with the supplied vendor, product, and version.

- * - *

If either the vendorWeightings or productWeightings lists have been - * populated this data is used to add weighting factors to the search.

- * - * @param vendor the text used to search the vendor field - * @param product the text used to search the product field - * @param vendorWeightings a list of strings to use to add weighting factors - * to the vendor field - * @param productWeightings Adds a list of strings that will be used to add - * weighting factors to the product search - * @return a list of possible CPE values - * @throws CorruptIndexException when the Lucene index is corrupt - * @throws IOException when the Lucene index is not found - * @throws ParseException when the generated query is not valid - */ - protected List searchCPE(String vendor, String product, - Set vendorWeightings, Set productWeightings) - throws CorruptIndexException, IOException, ParseException { - final ArrayList ret = new ArrayList(MAX_QUERY_RESULTS); - - final String searchString = buildSearch(vendor, product, vendorWeightings, productWeightings); - if (searchString == null) { - return ret; - } - - final TopDocs docs = cpe.search(searchString, MAX_QUERY_RESULTS); - for (ScoreDoc d : docs.scoreDocs) { - final Document doc = cpe.getDocument(d.doc); - final IndexEntry entry = new IndexEntry(); - entry.setVendor(doc.get(Fields.VENDOR)); - entry.setProduct(doc.get(Fields.PRODUCT)); - entry.setSearchScore(d.score); - if (!ret.contains(entry)) { - ret.add(entry); - } - } - return ret; - } - - /** - *

Builds a Lucene search string by properly escaping data and - * constructing a valid search query.

- * - *

If either the possibleVendor or possibleProducts lists have been - * populated this data is used to add weighting factors to the search string - * generated.

- * - * @param vendor text to search the vendor field - * @param product text to search the product field - * @param vendorWeighting a list of strings to apply to the vendor to boost - * the terms weight - * @param productWeightings a list of strings to apply to the product to - * boost the terms weight - * @return the Lucene query - */ - protected String buildSearch(String vendor, String product, - Set vendorWeighting, Set productWeightings) { - final String v = vendor; //.replaceAll("[^\\w\\d]", " "); - final String p = product; //.replaceAll("[^\\w\\d]", " "); - final StringBuilder sb = new StringBuilder(v.length() + p.length() - + Fields.PRODUCT.length() + Fields.VENDOR.length() + STRING_BUILDER_BUFFER); - - if (!appendWeightedSearch(sb, Fields.PRODUCT, p, productWeightings)) { - return null; - } - sb.append(" AND "); - if (!appendWeightedSearch(sb, Fields.VENDOR, v, vendorWeighting)) { - return null; - } - return sb.toString(); - } - - /** - * This method constructs a Lucene query for a given field. The searchText - * is split into separate words and if the word is within the list of - * weighted words then an additional weighting is applied to the term as it - * is appended into the query. - * - * @param sb a StringBuilder that the query text will be appended to. - * @param field the field within the Lucene index that the query is - * searching. - * @param searchText text used to construct the query. - * @param weightedText a list of terms that will be considered higher - * importance when searching. - * @return if the append was successful. - */ - private boolean appendWeightedSearch(StringBuilder sb, String field, String searchText, Set weightedText) { - sb.append(" ").append(field).append(":( "); - - final String cleanText = cleanseText(searchText); - - if ("".equals(cleanText)) { - return false; - } - - if (weightedText == null || weightedText.isEmpty()) { - LuceneUtils.appendEscapedLuceneQuery(sb, cleanText); - } else { - final StringTokenizer tokens = new StringTokenizer(cleanText); - while (tokens.hasMoreElements()) { - final String word = tokens.nextToken(); - String temp = null; - for (String weighted : weightedText) { - final String weightedStr = cleanseText(weighted); - if (equalsIgnoreCaseAndNonAlpha(word, weightedStr)) { - temp = LuceneUtils.escapeLuceneQuery(word) + WEIGHTING_BOOST; - if (!word.equalsIgnoreCase(weightedStr)) { - temp += " " + LuceneUtils.escapeLuceneQuery(weightedStr) + WEIGHTING_BOOST; - } - } - } - if (temp == null) { - temp = LuceneUtils.escapeLuceneQuery(word); - } - sb.append(" ").append(temp); - } - } - sb.append(" ) "); - return true; - } - - /** - * Removes characters from the input text that are not used within the CPE - * index. - * - * @param text is the text to remove the characters from. - * @return the text having removed some characters. - */ - private String cleanseText(String text) { - return text.replaceAll(CLEANSE_CHARACTER_RX, " "); - } - - /** - * Compares two strings after lower casing them and removing the non-alpha - * characters. - * - * @param l string one to compare. - * @param r string two to compare. - * @return whether or not the two strings are similar. - */ - private boolean equalsIgnoreCaseAndNonAlpha(String l, String r) { - if (l == null || r == null) { - return false; - } - - final String left = l.replaceAll(CLEANSE_NONALPHA_RX, ""); - final String right = r.replaceAll(CLEANSE_NONALPHA_RX, ""); - return left.equalsIgnoreCase(right); - } - - /** - * Ensures that the CPE Identified matches the dependency. This validates - * that the product, vendor, and version information for the CPE are - * contained within the dependencies evidence. - * - * @param entry a CPE entry. - * @param dependency the dependency that the CPE entries could be for. - * @return whether or not the entry is valid. - */ - private boolean verifyEntry(final IndexEntry entry, final Dependency dependency) { - boolean isValid = false; - - if (collectionContainsString(dependency.getProductEvidence(), entry.getProduct()) - && collectionContainsString(dependency.getVendorEvidence(), entry.getVendor())) { - //&& collectionContainsVersion(dependency.getVersionEvidence(), entry.getVersion()) - isValid = true; - } - return isValid; - } - - /** - * Used to determine if the EvidenceCollection contains a specific string. - * - * @param ec an EvidenceCollection - * @param text the text to search for - * @return whether or not the EvidenceCollection contains the string - */ - private boolean collectionContainsString(EvidenceCollection ec, String text) { - - // - // String[] splitText = text.split("[\\s_-]"); - // - // for (String search : splitText) { - // //final String search = text.replaceAll("[\\s_-]", "").toLowerCase(); - // if (ec.containsUsedString(search)) { - // return true; - // } - // } - // - - //TODO - likely need to change the split... not sure if this will work for CPE with special chars - final String[] words = text.split("[\\s_-]"); - final List list = new ArrayList(); - String tempWord = null; - for (String word : words) { - //single letter words should be concatonated with the next word. - // so { "m", "core", "sample" } -> { "mcore", "sample" } - if (tempWord != null) { - list.add(tempWord + word); - tempWord = null; - } else if (word.length() <= 2) { - tempWord = word; - } else { - list.add(word); - } - } -// if (tempWord != null) { -// //for now ignore any last single letter words... -// } - boolean contains = true; - for (String word : list) { - contains &= ec.containsUsedString(word); - } - return contains; - } - - /** - * Analyzes a dependency and attempts to determine if there are any CPE - * identifiers for this dependency. - * - * @param dependency The Dependency to analyze. - * @param engine The analysis engine - * @throws AnalysisException is thrown if there is an issue analyzing the - * dependency. - */ - @Override - public void analyze(Dependency dependency, Engine engine) throws AnalysisException { - try { - determineCPE(dependency); - } catch (CorruptIndexException ex) { - throw new AnalysisException("CPE Index is corrupt.", ex); - } catch (IOException ex) { - throw new AnalysisException("Failure opening the CPE Index.", ex); - } catch (ParseException ex) { - throw new AnalysisException("Unable to parse the generated Lucene query for this dependency.", ex); - } - } - - /** - * Returns true because this analyzer supports all dependency types. - * - * @return true. - */ - @Override - public Set getSupportedExtensions() { - return null; - } - - /** - * Returns the name of this analyzer. - * - * @return the name of this analyzer. - */ - @Override - public String getName() { - return "CPE Analyzer"; - } - - /** - * Returns true because this analyzer supports all dependency types. - * - * @param extension the file extension of the dependency being analyzed. - * @return true. - */ - @Override - public boolean supportsExtension(String extension) { - return true; - } - - /** - * Returns the analysis phase that this analyzer should run in. - * - * @return the analysis phase that this analyzer should run in. - */ - @Override - public AnalysisPhase getAnalysisPhase() { - return AnalysisPhase.IDENTIFIER_ANALYSIS; - } - - /** - * Opens the CPE Lucene Index. - * - * @throws Exception is thrown if there is an issue opening the index. - */ - @Override - public void initialize() throws Exception { - this.open(); - } - - /** - * Retrieves a list of CPE values from the CveDB based on the vendor and - * product passed in. The list is then validated to find only CPEs that are - * valid for the given dependency. It is possible that the CPE identified is - * a best effort "guess" based on the vendor, product, and version - * information. - * - * @param dependency the Dependency being analyzed - * @param vendor the vendor for the CPE being analyzed - * @param product the product for the CPE being analyzed - * @throws UnsupportedEncodingException is thrown if UTF-8 is not supported - */ - private void determineIdentifiers(Dependency dependency, String vendor, String product) throws UnsupportedEncodingException { - final Set cpes = cve.getCPEs(vendor, product); - DependencyVersion bestGuess = new DependencyVersion("-"); - Confidence bestGuessConf = null; - final List collected = new ArrayList(); - for (Confidence conf : Confidence.values()) { - for (Evidence evidence : dependency.getVersionEvidence().iterator(conf)) { - final DependencyVersion evVer = DependencyVersionUtil.parseVersion(evidence.getValue()); - if (evVer == null) { - continue; - } - for (VulnerableSoftware vs : cpes) { - DependencyVersion dbVer; - if (vs.getRevision() != null && !vs.getRevision().isEmpty()) { - dbVer = DependencyVersionUtil.parseVersion(vs.getVersion() + "." + vs.getRevision()); - } else { - dbVer = DependencyVersionUtil.parseVersion(vs.getVersion()); - } - if (dbVer == null //special case, no version specified - everything is vulnerable - || evVer.equals(dbVer)) { //woot exect match - final String url = String.format("http://web.nvd.nist.gov/view/vuln/search?cpe=%s", URLEncoder.encode(vs.getName(), "UTF-8")); - final IdentifierMatch match = new IdentifierMatch("cpe", vs.getName(), url, IdentifierConfidence.EXACT_MATCH, conf); - collected.add(match); - } else { - //TODO the following isn't quite right is it? need to think about this guessing game a bit more. - if (evVer.getVersionParts().size() <= dbVer.getVersionParts().size() - && evVer.matchesAtLeastThreeLevels(dbVer)) { - if (bestGuessConf == null || bestGuessConf.compareTo(conf) > 0) { - if (bestGuess.getVersionParts().size() < dbVer.getVersionParts().size()) { - bestGuess = dbVer; - bestGuessConf = conf; - } - } - } - } - } - if (bestGuessConf == null || bestGuessConf.compareTo(conf) > 0) { - if (bestGuess.getVersionParts().size() < evVer.getVersionParts().size()) { - bestGuess = evVer; - bestGuessConf = conf; - } - } - } - } - final String cpeName = String.format("cpe:/a:%s:%s:%s", vendor, product, bestGuess.toString()); - final String url = null; //String.format("http://web.nvd.nist.gov/view/vuln/search?cpe=%s", URLEncoder.encode(cpeName, "UTF-8")); - if (bestGuessConf == null) { - bestGuessConf = Confidence.LOW; - } - final IdentifierMatch match = new IdentifierMatch("cpe", cpeName, url, IdentifierConfidence.BEST_GUESS, bestGuessConf); - collected.add(match); - - Collections.sort(collected); - final IdentifierConfidence bestIdentifierQuality = collected.get(0).getConfidence(); - final Confidence bestEvidenceQuality = collected.get(0).getEvidenceConfidence(); - for (IdentifierMatch m : collected) { - if (bestIdentifierQuality.equals(m.getConfidence()) - && bestEvidenceQuality.equals(m.getEvidenceConfidence())) { - dependency.addIdentifier(m.getIdentifier()); - } - } - } - - /** - * The confidence whether the identifier is an exact match, or a best guess. - */ - private enum IdentifierConfidence { - - /** - * An exact match for the CPE. - */ - EXACT_MATCH, - /** - * A best guess for the CPE. - */ - BEST_GUESS - } - - /** - * A simple object to hold an identifier and carry information about the - * confidence in the identifier. - */ - private static class IdentifierMatch implements Comparable { - - /** - * Constructs an IdentiferMatch. - * - * @param type the type of identifier (such as CPE) - * @param value the value of the identifier - * @param url the URL of the identifier - * @param identifierConfidence the confidence in the identifier: best - * guess or exact match - * @param evidenceConfidence the confidence of the evidence used to find - * the identifier - */ - IdentifierMatch(String type, String value, String url, IdentifierConfidence identifierConfidence, Confidence evidenceConfidence) { - this.identifier = new Identifier(type, value, url); - this.confidence = identifierConfidence; - this.evidenceConfidence = evidenceConfidence; - } - // - /** - * The confidence in the evidence used to identify this match. - */ - private Confidence evidenceConfidence; - - /** - * Get the value of evidenceConfidence - * - * @return the value of evidenceConfidence - */ - public Confidence getEvidenceConfidence() { - return evidenceConfidence; - } - - /** - * Set the value of evidenceConfidence - * - * @param evidenceConfidence new value of evidenceConfidence - */ - public void setEvidenceConfidence(Confidence evidenceConfidence) { - this.evidenceConfidence = evidenceConfidence; - } - /** - * The confidence whether this is an exact match, or a best guess. - */ - private IdentifierConfidence confidence; - - /** - * Get the value of confidence. - * - * @return the value of confidence - */ - public IdentifierConfidence getConfidence() { - return confidence; - } - - /** - * Set the value of confidence. - * - * @param confidence new value of confidence - */ - public void setConfidence(IdentifierConfidence confidence) { - this.confidence = confidence; - } - /** - * The CPE identifier. - */ - private Identifier identifier; - - /** - * Get the value of identifier. - * - * @return the value of identifier - */ - public Identifier getIdentifier() { - return identifier; - } - - /** - * Set the value of identifier. - * - * @param identifier new value of identifier - */ - public void setIdentifier(Identifier identifier) { - this.identifier = identifier; - } - // - // - - /** - * Standard toString() implementation. - * - * @return the string representation of the object - */ - @Override - public String toString() { - return "IdentifierMatch{" + "evidenceConfidence=" + evidenceConfidence - + ", confidence=" + confidence + ", identifier=" + identifier + '}'; - } - - /** - * Standard hashCode() implementation. - * - * @return the hashCode - */ - @Override - public int hashCode() { - int hash = 5; - hash = 97 * hash + (this.evidenceConfidence != null ? this.evidenceConfidence.hashCode() : 0); - hash = 97 * hash + (this.confidence != null ? this.confidence.hashCode() : 0); - hash = 97 * hash + (this.identifier != null ? this.identifier.hashCode() : 0); - return hash; - } - - /** - * Standard equals implementation. - * - * @param obj the object to compare - * @return true if the objects are equal, otherwise false - */ - @Override - public boolean equals(Object obj) { - if (obj == null) { - return false; - } - if (getClass() != obj.getClass()) { - return false; - } - final IdentifierMatch other = (IdentifierMatch) obj; - if (this.evidenceConfidence != other.evidenceConfidence) { - return false; - } - if (this.confidence != other.confidence) { - return false; - } - if (this.identifier != other.identifier && (this.identifier == null || !this.identifier.equals(other.identifier))) { - return false; - } - return true; - } - // - - /** - * Standard implementation of compareTo that compares identifier - * confidence, evidence confidence, and then the identifier. - * - * @param o the IdentifierMatch to compare to - * @return the natural ordering of IdentifierMatch - */ - @Override - public int compareTo(IdentifierMatch o) { - int conf = this.confidence.compareTo(o.confidence); - if (conf == 0) { - conf = this.evidenceConfidence.compareTo(o.evidenceConfidence); - if (conf == 0) { - conf = identifier.compareTo(o.identifier); - } - } - return conf; - } - } -} diff --git a/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/cpe/Index.java b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/cpe/Index.java deleted file mode 100644 index d63a7a189..000000000 --- a/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/cpe/Index.java +++ /dev/null @@ -1,170 +0,0 @@ -/* - * This file is part of dependency-check-core. - * - * Dependency-check-core is free software: you can redistribute it and/or modify it - * under the terms of the GNU General Public License as published by the Free - * Software Foundation, either version 3 of the License, or (at your option) any - * later version. - * - * Dependency-check-core is distributed in the hope that it will be useful, but - * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more - * details. - * - * You should have received a copy of the GNU General Public License along with - * dependency-check-core. If not, see http://www.gnu.org/licenses/. - * - * Copyright (c) 2012 Jeremy Long. All Rights Reserved. - */ -package org.owasp.dependencycheck.data.cpe; - -import java.io.File; -import java.io.IOException; -import java.util.HashMap; -import java.util.Map; -import org.apache.lucene.analysis.Analyzer; -import org.apache.lucene.analysis.core.KeywordAnalyzer; -import org.apache.lucene.analysis.miscellaneous.PerFieldAnalyzerWrapper; -import org.apache.lucene.document.Document; -import org.apache.lucene.document.Field; -import org.apache.lucene.document.TextField; -import org.apache.lucene.index.CorruptIndexException; -import org.apache.lucene.index.Term; -import org.apache.lucene.queryparser.classic.QueryParser; -import org.apache.lucene.store.Directory; -import org.apache.lucene.store.FSDirectory; -import org.apache.lucene.util.Version; -import org.owasp.dependencycheck.data.lucene.AbstractIndex; -import org.owasp.dependencycheck.utils.Settings; -import org.owasp.dependencycheck.data.lucene.FieldAnalyzer; -import org.owasp.dependencycheck.data.lucene.SearchFieldAnalyzer; - -/** - * The Index class is used to utilize and maintain the CPE Index. - * - * @author Jeremy Long (jeremy.long@owasp.org) - */ -public class Index extends AbstractIndex { - - /** - * Returns the directory that holds the CPE Index. - * - * @return the Directory containing the CPE Index. - * @throws IOException is thrown if an IOException occurs. - */ - @Override - public Directory getDirectory() throws IOException { - final File path = getDataDirectory(); - return FSDirectory.open(path); - } - - /** - * Retrieves the directory that the JAR file exists in so that we can ensure - * we always use a common data directory. - * - * @return the data directory for this index. - * @throws IOException is thrown if an IOException occurs of course... - */ - public static File getDataDirectory() throws IOException { - final File path = Settings.getFile(Settings.KEYS.CPE_DATA_DIRECTORY); - if (!path.exists()) { - if (!path.mkdirs()) { - throw new IOException("Unable to create CPE Data directory"); - } - } - return path; - } - - /** - * Creates an Analyzer for the CPE Index. - * - * @return the CPE Analyzer. - */ - @SuppressWarnings("unchecked") - @Override - public Analyzer createIndexingAnalyzer() { - final Map fieldAnalyzers = new HashMap(); - fieldAnalyzers.put(Fields.DOCUMENT_KEY, new KeywordAnalyzer()); - return new PerFieldAnalyzerWrapper(new FieldAnalyzer(Version.LUCENE_43), fieldAnalyzers); - } - /** - * The search field analyzer for the product field. - */ - private SearchFieldAnalyzer productSearchFieldAnalyzer; - /** - * The search field analyzer for the vendor field. - */ - private SearchFieldAnalyzer vendorSearchFieldAnalyzer; - - /** - * Creates an Analyzer for searching the CPE Index. - * - * @return the CPE Analyzer. - */ - @SuppressWarnings("unchecked") - @Override - public Analyzer createSearchingAnalyzer() { - final Map fieldAnalyzers = new HashMap(); - - fieldAnalyzers.put(Fields.DOCUMENT_KEY, new KeywordAnalyzer()); - productSearchFieldAnalyzer = new SearchFieldAnalyzer(Version.LUCENE_43); - vendorSearchFieldAnalyzer = new SearchFieldAnalyzer(Version.LUCENE_43); - fieldAnalyzers.put(Fields.PRODUCT, productSearchFieldAnalyzer); - fieldAnalyzers.put(Fields.VENDOR, vendorSearchFieldAnalyzer); - - return new PerFieldAnalyzerWrapper(new FieldAnalyzer(Version.LUCENE_43), fieldAnalyzers); - } - - /** - * Creates the Lucene QueryParser used when querying the index. - * - * @return a QueryParser. - */ - @Override - public QueryParser createQueryParser() { - return new QueryParser(Version.LUCENE_43, Fields.DOCUMENT_KEY, getSearchingAnalyzer()); - } - - /** - * Resets the searching analyzers - */ - @Override - protected void resetSearchingAnalyzer() { - if (productSearchFieldAnalyzer != null) { - productSearchFieldAnalyzer.clear(); - } - if (vendorSearchFieldAnalyzer != null) { - vendorSearchFieldAnalyzer.clear(); - } - } - - /** - * Saves a CPE IndexEntry into the Lucene index. - * - * @param entry a CPE entry. - * @throws CorruptIndexException is thrown if the index is corrupt. - * @throws IOException is thrown if an IOException occurs. - */ - public void saveEntry(IndexEntry entry) throws CorruptIndexException, IOException { - final Document doc = convertEntryToDoc(entry); - final Term term = new Term(Fields.DOCUMENT_KEY, entry.getDocumentId()); - getIndexWriter().updateDocument(term, doc); - } - - /** - * Converts a CPE entry into a Lucene Document. - * - * @param entry a CPE IndexEntry. - * @return a Lucene Document containing a CPE IndexEntry. - */ - protected Document convertEntryToDoc(IndexEntry entry) { - final Document doc = new Document(); - - final Field vendor = new TextField(Fields.VENDOR, entry.getVendor(), Field.Store.YES); - doc.add(vendor); - - final Field product = new TextField(Fields.PRODUCT, entry.getProduct(), Field.Store.YES); - doc.add(product); - return doc; - } -} diff --git a/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/lucene/AbstractIndex.java b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/lucene/AbstractIndex.java deleted file mode 100644 index 2fb2ad66f..000000000 --- a/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/lucene/AbstractIndex.java +++ /dev/null @@ -1,341 +0,0 @@ -/* - * This file is part of dependency-check-core. - * - * Dependency-check-core is free software: you can redistribute it and/or modify it - * under the terms of the GNU General Public License as published by the Free - * Software Foundation, either version 3 of the License, or (at your option) any - * later version. - * - * Dependency-check-core is distributed in the hope that it will be useful, but - * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more - * details. - * - * You should have received a copy of the GNU General Public License along with - * dependency-check-core. If not, see http://www.gnu.org/licenses/. - * - * Copyright (c) 2012 Jeremy Long. All Rights Reserved. - */ -package org.owasp.dependencycheck.data.lucene; - -import java.io.IOException; -import java.util.logging.Level; -import java.util.logging.Logger; -import org.apache.lucene.analysis.Analyzer; -import org.apache.lucene.document.Document; -import org.apache.lucene.index.CorruptIndexException; -import org.apache.lucene.index.DirectoryReader; -import org.apache.lucene.index.IndexReader; -import org.apache.lucene.index.IndexWriter; -import org.apache.lucene.index.IndexWriterConfig; -import org.apache.lucene.queryparser.classic.ParseException; -import org.apache.lucene.queryparser.classic.QueryParser; -import org.apache.lucene.search.IndexSearcher; -import org.apache.lucene.search.Query; -import org.apache.lucene.search.TopDocs; -import org.apache.lucene.store.Directory; -import org.apache.lucene.store.LockObtainFailedException; -import org.apache.lucene.util.Version; - -/** - * The base Index for other index objects. Implements the open and close - * methods. - * - * @author Jeremy Long (jeremy.long@owasp.org) - */ -public abstract class AbstractIndex { - - /** - * The Lucene directory containing the index. - */ - private Directory directory; - /** - * The IndexWriter for the Lucene index. - */ - private IndexWriter indexWriter; - /** - * The Lucene IndexReader. - */ - private IndexReader indexReader; - /** - * The Lucene IndexSearcher. - */ - private IndexSearcher indexSearcher; - /** - * The Lucene Analyzer used for Indexing. - */ - private Analyzer indexingAnalyzer; - /** - * The Lucene Analyzer used for Searching. - */ - private Analyzer searchingAnalyzer; - /** - * The Lucene QueryParser used for Searching. - */ - private QueryParser queryParser; - /** - * Indicates whether or not the Lucene Index is open. - */ - private boolean indexOpen = false; - - /** - * Opens the CPE Index. - * - * @throws IOException is thrown if an IOException occurs opening the index. - */ - public void open() throws IOException { - directory = this.getDirectory(); - indexingAnalyzer = this.getIndexingAnalyzer(); - searchingAnalyzer = this.getSearchingAnalyzer(); - indexOpen = true; - } - - /** - * Commits any pending changes. - */ - public void commit() { - if (indexWriter != null) { - try { - indexWriter.commit(); - } catch (CorruptIndexException ex) { - final String msg = "Unable to update database, there is a corrupt index."; - Logger.getLogger(AbstractIndex.class.getName()).log(Level.SEVERE, msg); - Logger.getLogger(AbstractIndex.class.getName()).log(Level.FINE, null, ex); - } catch (IOException ex) { - final String msg = "Unable to update database due to an IO error."; - Logger.getLogger(AbstractIndex.class.getName()).log(Level.SEVERE, msg); - Logger.getLogger(AbstractIndex.class.getName()).log(Level.FINE, null, ex); - } - } - } - - /** - * Closes the CPE Index. - */ - public void close() { - if (indexWriter != null) { - commit(); - try { - indexWriter.close(true); - } catch (CorruptIndexException ex) { - final String msg = "Unable to update database, there is a corrupt index."; - Logger.getLogger(AbstractIndex.class.getName()).log(Level.SEVERE, msg); - Logger.getLogger(AbstractIndex.class.getName()).log(Level.FINE, null, ex); - } catch (IOException ex) { - final String msg = "Unable to update database due to an IO error."; - Logger.getLogger(AbstractIndex.class.getName()).log(Level.SEVERE, msg); - Logger.getLogger(AbstractIndex.class.getName()).log(Level.FINE, null, ex); - } finally { - indexWriter = null; - } - } - if (indexSearcher != null) { - indexSearcher = null; - } - - if (indexingAnalyzer != null) { - indexingAnalyzer.close(); - indexingAnalyzer = null; - } - - if (searchingAnalyzer != null) { - searchingAnalyzer.close(); - searchingAnalyzer = null; - } - - try { - directory.close(); - } catch (IOException ex) { - final String msg = "Unable to update database due to an IO error."; - Logger.getLogger(AbstractIndex.class.getName()).log(Level.SEVERE, msg); - Logger.getLogger(AbstractIndex.class.getName()).log(Level.FINE, null, ex); - } finally { - directory = null; - } - indexOpen = false; - } - - /** - * Returns the status of the data source - is the index open. - * - * @return true or false. - */ - public boolean isOpen() { - return indexOpen; - } - - /** - * Opens the Lucene Index Writer. - * - * @throws CorruptIndexException is thrown if the Lucene index is corrupt. - * @throws IOException is thrown if an IOException occurs opening the index. - */ - public void openIndexWriter() throws CorruptIndexException, IOException { - if (!isOpen()) { - open(); - } - final IndexWriterConfig conf = new IndexWriterConfig(Version.LUCENE_43, indexingAnalyzer); - indexWriter = new IndexWriter(directory, conf); - } - - /** - * Retrieves the IndexWriter for the Lucene Index. - * - * @return an IndexWriter. - * @throws CorruptIndexException is thrown if the Lucene Index is corrupt. - * @throws LockObtainFailedException is thrown if there is an exception - * obtaining a lock on the Lucene index. - * @throws IOException is thrown if an IOException occurs opening the index. - */ - public IndexWriter getIndexWriter() throws CorruptIndexException, LockObtainFailedException, IOException { - if (indexWriter == null) { - openIndexWriter(); - } - return indexWriter; - } - - /** - * Opens the Lucene Index for reading. - * - * @throws CorruptIndexException is thrown if the index is corrupt. - * @throws IOException is thrown if there is an exception reading the index. - */ - public void openIndexReader() throws CorruptIndexException, IOException { - if (!isOpen()) { - open(); - } - //indexReader = IndexReader.open(directory, true); - indexReader = DirectoryReader.open(directory); - } - - /** - * Returns an IndexSearcher for the Lucene Index. - * - * @return an IndexSearcher. - * @throws CorruptIndexException is thrown if the index is corrupt. - * @throws IOException is thrown if there is an exception reading the index. - */ - protected IndexSearcher getIndexSearcher() throws CorruptIndexException, IOException { - if (indexReader == null) { - openIndexReader(); - } - if (indexSearcher == null) { - indexSearcher = new IndexSearcher(indexReader); - } - return indexSearcher; - } - - /** - * Returns an Analyzer to be used when indexing. - * - * @return an Analyzer. - */ - public Analyzer getIndexingAnalyzer() { - if (indexingAnalyzer == null) { - indexingAnalyzer = createIndexingAnalyzer(); - } - return indexingAnalyzer; - } - - /** - * Returns an analyzer used for searching the index - * - * @return a lucene analyzer - */ - protected Analyzer getSearchingAnalyzer() { - if (searchingAnalyzer == null) { - searchingAnalyzer = createSearchingAnalyzer(); - } - return searchingAnalyzer; - } - - /** - * Gets a query parser - * - * @return a query parser - */ - protected QueryParser getQueryParser() { - if (queryParser == null) { - queryParser = createQueryParser(); - } - return queryParser; - } - - /** - * Searches the index using the given search string. - * - * @param searchString the query text - * @param maxQueryResults the maximum number of documents to return - * @return the TopDocs found by the search - * @throws ParseException thrown when the searchString is invalid - * @throws IOException is thrown if there is an issue with the underlying - * Index - */ - public TopDocs search(String searchString, int maxQueryResults) throws ParseException, IOException { - final QueryParser parser = getQueryParser(); - final Query query = parser.parse(searchString); - resetSearchingAnalyzer(); - final IndexSearcher is = getIndexSearcher(); - return is.search(query, maxQueryResults); - } - - /** - * Searches the index using the given query. - * - * @param query the query used to search the index - * @param maxQueryResults the max number of results to return - * @return the TopDocs found be the query - * @throws CorruptIndexException thrown if the Index is corrupt - * @throws IOException thrown if there is an IOException - */ - public TopDocs search(Query query, int maxQueryResults) throws CorruptIndexException, IOException { - final IndexSearcher is = getIndexSearcher(); - return is.search(query, maxQueryResults); - } - - /** - * Retrieves a document from the Index. - * - * @param documentId the id of the document to retrieve - * @return the Document - * @throws IOException thrown if there is an IOException - */ - public Document getDocument(int documentId) throws IOException { - final IndexSearcher is = getIndexSearcher(); - return is.doc(documentId); - } - - /** - * Gets the directory that contains the Lucene Index. - * - * @return a Lucene Directory - * @throws IOException is thrown when an IOException occurs - */ - public abstract Directory getDirectory() throws IOException; - - /** - * Creates the Lucene Analyzer used when indexing. - * - * @return a Lucene Analyzer - */ - public abstract Analyzer createIndexingAnalyzer(); - - /** - * Creates the Lucene Analyzer used when querying the index. - * - * @return a Lucene Analyzer - */ - public abstract Analyzer createSearchingAnalyzer(); - - /** - * Creates the Lucene QueryParser used when querying the index. - * - * @return a QueryParser - */ - public abstract QueryParser createQueryParser(); - - /** - * Resets the searching analyzers - */ - protected abstract void resetSearchingAnalyzer(); -} diff --git a/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/nvdcve/xml/DatabaseUpdater.java b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/nvdcve/xml/DatabaseUpdater.java index af160e3ce..d65fad8b4 100644 --- a/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/nvdcve/xml/DatabaseUpdater.java +++ b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/nvdcve/xml/DatabaseUpdater.java @@ -43,7 +43,8 @@ import java.util.logging.Logger; import javax.xml.parsers.SAXParser; import javax.xml.parsers.SAXParserFactory; import org.owasp.dependencycheck.data.UpdateException; -import org.owasp.dependencycheck.data.cpe.Index; +import org.owasp.dependencycheck.data.cpe.BaseIndex; +import org.owasp.dependencycheck.data.cpe.CpeIndexWriter; import org.owasp.dependencycheck.data.nvdcve.CveDB; import org.owasp.dependencycheck.dependency.VulnerableSoftware; import org.owasp.dependencycheck.utils.DownloadFailedException; @@ -87,7 +88,7 @@ public class DatabaseUpdater implements CachedWebDataSource { /** * Reference to the Cpe Index. */ - private Index cpeIndex = null; + private CpeIndexWriter cpeIndex = null; public DatabaseUpdater() { batchUpdateMode = !Settings.getString(Settings.KEYS.BATCH_UPDATE_URL, "").isEmpty(); @@ -273,8 +274,8 @@ public class DatabaseUpdater implements CachedWebDataSource { try { cveDB = new CveDB(); cveDB.open(); - cpeIndex = new Index(); - cpeIndex.openIndexWriter(); + cpeIndex = new CpeIndexWriter(); + cpeIndex.open(); } catch (IOException ex) { closeDataStores(); Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.FINE, "IO Error opening databases", ex); @@ -640,7 +641,7 @@ public class DatabaseUpdater implements CachedWebDataSource { final File cveDir = CveDB.getDataDirectory(); FileUtils.delete(cveDir); - final File cpeDir = Index.getDataDirectory(); + final File cpeDir = BaseIndex.getDataDirectory(); FileUtils.delete(cpeDir); } diff --git a/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/nvdcve/xml/NvdCve20Handler.java b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/nvdcve/xml/NvdCve20Handler.java index b1ee760ba..3f5808466 100644 --- a/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/nvdcve/xml/NvdCve20Handler.java +++ b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/nvdcve/xml/NvdCve20Handler.java @@ -24,7 +24,7 @@ import java.util.Map; import java.util.logging.Level; import java.util.logging.Logger; import org.apache.lucene.index.CorruptIndexException; -import org.owasp.dependencycheck.data.cpe.Index; +import org.owasp.dependencycheck.data.cpe.CpeIndexWriter; import org.owasp.dependencycheck.data.nvdcve.CveDB; import org.owasp.dependencycheck.data.nvdcve.DatabaseException; import org.owasp.dependencycheck.dependency.Reference; @@ -274,14 +274,14 @@ public class NvdCve20Handler extends DefaultHandler { /** * the cpe index. */ - private Index cpeIndex; + private CpeIndexWriter cpeIndex; /** - * Sets the cpe index. + * Sets the cpe index writer. * * @param index the CPE Lucene Index */ - void setCpeIndex(Index index) { + void setCpeIndex(CpeIndexWriter index) { cpeIndex = index; } diff --git a/dependency-check-core/src/test/java/org/owasp/dependencycheck/data/cpe/CPEAnalyzerTest.java b/dependency-check-core/src/test/java/org/owasp/dependencycheck/data/cpe/CPEAnalyzerTest.java deleted file mode 100644 index b3584a068..000000000 --- a/dependency-check-core/src/test/java/org/owasp/dependencycheck/data/cpe/CPEAnalyzerTest.java +++ /dev/null @@ -1,254 +0,0 @@ -/* - * This file is part of dependency-check-core. - * - * Dependency-check-core is free software: you can redistribute it and/or modify it - * under the terms of the GNU General Public License as published by the Free - * Software Foundation, either version 3 of the License, or (at your option) any - * later version. - * - * Dependency-check-core is distributed in the hope that it will be useful, but - * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more - * details. - * - * You should have received a copy of the GNU General Public License along with - * dependency-check-core. If not, see http://www.gnu.org/licenses/. - * - * Copyright (c) 2012 Jeremy Long. All Rights Reserved. - */ -package org.owasp.dependencycheck.data.cpe; - -import org.owasp.dependencycheck.data.cpe.IndexEntry; -import org.owasp.dependencycheck.data.cpe.CPEAnalyzer; -import java.io.File; -import java.io.IOException; -import java.util.HashSet; -import java.util.List; -import java.util.Set; -import org.apache.lucene.index.CorruptIndexException; -import org.apache.lucene.queryparser.classic.ParseException; -import org.junit.After; -import org.junit.AfterClass; -import org.owasp.dependencycheck.dependency.Dependency; -import org.owasp.dependencycheck.analyzer.JarAnalyzer; -import org.junit.Assert; -import org.junit.Before; -import org.junit.BeforeClass; -import org.junit.Test; -import org.owasp.dependencycheck.analyzer.FalsePositiveAnalyzer; -import org.owasp.dependencycheck.analyzer.FileNameAnalyzer; -import org.owasp.dependencycheck.analyzer.HintAnalyzer; -import static org.owasp.dependencycheck.data.cpe.BaseIndexTestCase.ensureIndexExists; -import org.owasp.dependencycheck.dependency.Identifier; - -/** - * - * @author Jeremy Long (jeremy.long@owasp.org) - */ -public class CPEAnalyzerTest extends BaseIndexTestCase { - - @BeforeClass - public static void setUpClass() throws Exception { - } - - @AfterClass - public static void tearDownClass() throws Exception { - } - - @Before - public void setUp() throws Exception { - super.setUp(); - } - - @After - public void tearDown() throws Exception { - super.tearDown(); - } - - /** - * Tests of buildSearch of class CPEAnalyzer. - * - * @throws IOException is thrown when an IO Exception occurs. - * @throws CorruptIndexException is thrown when the index is corrupt. - * @throws ParseException is thrown when a parse exception occurs - */ - @Test - public void testBuildSearch() throws IOException, CorruptIndexException, ParseException { - Set productWeightings = new HashSet(1); - productWeightings.add("struts2"); - - Set vendorWeightings = new HashSet(1); - vendorWeightings.add("apache"); - - String vendor = "apache software foundation"; - String product = "struts 2 core"; - String version = "2.1.2"; - CPEAnalyzer instance = new CPEAnalyzer(); - - String queryText = instance.buildSearch(vendor, product, null, null); - String expResult = " product:( struts 2 core ) AND vendor:( apache software foundation ) "; - Assert.assertTrue(expResult.equals(queryText)); - - queryText = instance.buildSearch(vendor, product, null, productWeightings); - expResult = " product:( struts^5 struts2^5 2 core ) AND vendor:( apache software foundation ) "; - Assert.assertTrue(expResult.equals(queryText)); - - queryText = instance.buildSearch(vendor, product, vendorWeightings, null); - expResult = " product:( struts 2 core ) AND vendor:( apache^5 software foundation ) "; - Assert.assertTrue(expResult.equals(queryText)); - - queryText = instance.buildSearch(vendor, product, vendorWeightings, productWeightings); - expResult = " product:( struts^5 struts2^5 2 core ) AND vendor:( apache^5 software foundation ) "; - Assert.assertTrue(expResult.equals(queryText)); - } - - /** - * Test of open method, of class CPEAnalyzer. - * - * @throws Exception is thrown when an exception occurs - */ - @Test - public void testOpen() throws Exception { - CPEAnalyzer instance = new CPEAnalyzer(); - Assert.assertFalse(instance.isOpen()); - instance.open(); - Assert.assertTrue(instance.isOpen()); - instance.close(); - Assert.assertFalse(instance.isOpen()); - } - - /** - * Test of determineCPE method, of class CPEAnalyzer. - * - * @throws Exception is thrown when an exception occurs - */ - @Test - public void testDetermineCPE_full() throws Exception { - callDetermineCPE_full("spring-context-support-2.5.5.jar", "cpe:/a:vmware:springsource_spring_framework:2.5.5"); - callDetermineCPE_full("spring-core-3.0.0.RELEASE.jar", "cpe:/a:vmware:springsource_spring_framework:3.0.0"); - callDetermineCPE_full("org.mortbay.jetty.jar", "cpe:/a:mortbay_jetty:jetty:4.2"); - callDetermineCPE_full("jaxb-xercesImpl-1.5.jar", null); - callDetermineCPE_full("ehcache-core-2.2.0.jar", null); - } - - /** - * Test of determineCPE method, of class CPEAnalyzer. - * - * @throws Exception is thrown when an exception occurs - */ - public void callDetermineCPE_full(String depName, String expResult) throws Exception { - - File file = new File(this.getClass().getClassLoader().getResource(depName).getPath()); - Dependency dep = new Dependency(file); - - FileNameAnalyzer fnAnalyzer = new FileNameAnalyzer(); - fnAnalyzer.analyze(dep, null); - - JarAnalyzer jarAnalyzer = new JarAnalyzer(); - jarAnalyzer.analyze(dep, null); - HintAnalyzer hAnalyzer = new HintAnalyzer(); - hAnalyzer.analyze(dep, null); - - - CPEAnalyzer instance = new CPEAnalyzer(); - instance.open(); - instance.analyze(dep, null); - instance.close(); - FalsePositiveAnalyzer fp = new FalsePositiveAnalyzer(); - fp.analyze(dep, null); - -// for (Identifier i : dep.getIdentifiers()) { -// System.out.println(i.getValue()); -// } - if (expResult != null) { - Identifier expIdentifier = new Identifier("cpe", expResult, expResult); - Assert.assertTrue("Incorrect match: { dep:'" + dep.getFileName() + "' }", dep.getIdentifiers().contains(expIdentifier)); - } else { - Assert.assertTrue("Match found when an Identifier should not have been found: { dep:'" + dep.getFileName() + "' }", dep.getIdentifiers().isEmpty()); - } - } - - /** - * Test of determineCPE method, of class CPEAnalyzer. - * - * @throws Exception is thrown when an exception occurs - */ - @Test - public void testDetermineCPE() throws Exception { - File file = new File(this.getClass().getClassLoader().getResource("struts2-core-2.1.2.jar").getPath()); - //File file = new File(this.getClass().getClassLoader().getResource("axis2-adb-1.4.1.jar").getPath()); - Dependency struts = new Dependency(file); - - FileNameAnalyzer fnAnalyzer = new FileNameAnalyzer(); - fnAnalyzer.analyze(struts, null); - - JarAnalyzer jarAnalyzer = new JarAnalyzer(); - jarAnalyzer.analyze(struts, null); - - - File fileCommonValidator = new File(this.getClass().getClassLoader().getResource("commons-validator-1.4.0.jar").getPath()); - Dependency commonValidator = new Dependency(fileCommonValidator); - jarAnalyzer.analyze(commonValidator, null); - - File fileSpring = new File(this.getClass().getClassLoader().getResource("spring-core-2.5.5.jar").getPath()); - Dependency spring = new Dependency(fileSpring); - jarAnalyzer.analyze(spring, null); - - File fileSpring3 = new File(this.getClass().getClassLoader().getResource("spring-core-3.0.0.RELEASE.jar").getPath()); - Dependency spring3 = new Dependency(fileSpring3); - jarAnalyzer.analyze(spring3, null); - - CPEAnalyzer instance = new CPEAnalyzer(); - instance.open(); - instance.determineCPE(commonValidator); - instance.determineCPE(struts); - instance.determineCPE(spring); - instance.determineCPE(spring3); - instance.close(); - - String expResult = "cpe:/a:apache:struts:2.1.2"; - Identifier expIdentifier = new Identifier("cpe", expResult, expResult); - String expResultSpring = "cpe:/a:springsource:spring_framework:2.5.5"; - String expResultSpring3 = "cpe:/a:vmware:springsource_spring_framework:3.0.0"; - - Assert.assertTrue("Apache Common Validator - found an identifier?", commonValidator.getIdentifiers().isEmpty()); - Assert.assertTrue("Incorrect match size - struts", struts.getIdentifiers().size() >= 1); - Assert.assertTrue("Incorrect match - struts", struts.getIdentifiers().contains(expIdentifier)); - Assert.assertTrue("Incorrect match size - spring3 - " + spring3.getIdentifiers().size(), spring3.getIdentifiers().size() >= 1); - - //the following two only work if the HintAnalyzer is used. - //Assert.assertTrue("Incorrect match size - spring", spring.getIdentifiers().size() == 1); - //Assert.assertTrue("Incorrect match - spring", spring.getIdentifiers().get(0).getValue().equals(expResultSpring)); - - } - - /** - * Test of searchCPE method, of class CPEAnalyzer. - * - * @throws Exception is thrown when an exception occurs - */ - @Test - public void testSearchCPE() throws Exception { - String vendor = "apache software foundation"; - String product = "struts 2 core"; - String version = "2.1.2"; - String expResult = "cpe:/a:apache:struts:2.1.2"; - - CPEAnalyzer instance = new CPEAnalyzer(); - instance.open(); - - //TODO - yeah, not a very good test as the results are the same with or without weighting... - Set productWeightings = new HashSet(1); - productWeightings.add("struts2"); - - Set vendorWeightings = new HashSet(1); - vendorWeightings.add("apache"); - - List result = instance.searchCPE(vendor, product, productWeightings, vendorWeightings); - //TODO fix this assert - //Assert.assertEquals(expResult, result.get(0).getName()); - - - instance.close(); - } -} diff --git a/dependency-check-core/src/test/java/org/owasp/dependencycheck/data/cpe/IndexTest.java b/dependency-check-core/src/test/java/org/owasp/dependencycheck/data/cpe/IndexTest.java deleted file mode 100644 index 0835469bf..000000000 --- a/dependency-check-core/src/test/java/org/owasp/dependencycheck/data/cpe/IndexTest.java +++ /dev/null @@ -1,103 +0,0 @@ -/* - * This file is part of dependency-check-core. - * - * Dependency-check-core is free software: you can redistribute it and/or modify it - * under the terms of the GNU General Public License as published by the Free - * Software Foundation, either version 3 of the License, or (at your option) any - * later version. - * - * Dependency-check-core is distributed in the hope that it will be useful, but - * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more - * details. - * - * You should have received a copy of the GNU General Public License along with - * dependency-check-core. If not, see http://www.gnu.org/licenses/. - * - * Copyright (c) 2012 Jeremy Long. All Rights Reserved. - */ -package org.owasp.dependencycheck.data.cpe; - -import org.owasp.dependencycheck.data.cpe.Index; -import java.io.File; -import java.io.IOException; -import java.util.logging.Level; -import java.util.logging.Logger; -import org.apache.lucene.document.Document; -import org.apache.lucene.queryparser.classic.ParseException; -import org.apache.lucene.search.ScoreDoc; -import org.apache.lucene.search.TopDocs; -import org.apache.lucene.store.Directory; -import org.junit.After; -import org.junit.AfterClass; -import org.junit.Before; -import org.junit.BeforeClass; -import org.junit.Test; -import static org.junit.Assert.*; - -/** - * - * @author Jeremy Long (jeremy.long@owasp.org) - */ -public class IndexTest { - - @BeforeClass - public static void setUpClass() throws Exception { - } - - @AfterClass - public static void tearDownClass() throws Exception { - } - - @Before - public void setUp() { - } - - @After - public void tearDown() { - } - - /** - * Test of open method, of class Index. - */ - @Test - public void testOpen() { - Index instance = new Index(); - try { - instance.open(); - //TODO research why are we getting multiple documents for the same documentId. is the update method not working? -// try { -// instance.createSearchingAnalyzer(); -// TopDocs docs = instance.search("product:( project\\-open )", 20); -// for (ScoreDoc d : docs.scoreDocs) { -// final Document doc = instance.getDocument(d.doc); -// String vendor = doc.getField(Fields.VENDOR).stringValue(); -// String product = doc.getField(Fields.PRODUCT).stringValue(); -// System.out.print(d.doc); -// System.out.print(" : "); -// System.out.print(vendor + ":"); -// System.out.println(product); -// } -// } catch (ParseException ex) { -// Logger.getLogger(IndexTest.class.getName()).log(Level.SEVERE, null, ex); -// } - } catch (IOException ex) { - assertNull(ex.getMessage(), ex); - } - instance.close(); - } - - /** - * Test of getDirectory method, of class Index. - * - * @throws Exception - */ - @Test - public void testGetDirectory() throws Exception { - Index index = new Index(); - Directory result = index.getDirectory(); - - String exp = File.separatorChar + "target" + File.separatorChar + "data" + File.separatorChar + "cpe"; - assertTrue(result.toString().contains(exp)); - } -} From d16123c2763385bddbd40075a3d072816da10254 Mon Sep 17 00:00:00 2001 From: Jeremy Long Date: Sat, 24 Aug 2013 17:01:39 -0400 Subject: [PATCH 11/50] updates to ensure backward compatability with 1.6 Former-commit-id: b98e3caf316ea880b6c0adb25f62338e8d16f40e --- .../concurrency/DirectorySpinLock.java | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/dependency-check-core/src/main/java/org/owasp/dependencycheck/concurrency/DirectorySpinLock.java b/dependency-check-core/src/main/java/org/owasp/dependencycheck/concurrency/DirectorySpinLock.java index 60de62900..001f344bf 100644 --- a/dependency-check-core/src/main/java/org/owasp/dependencycheck/concurrency/DirectorySpinLock.java +++ b/dependency-check-core/src/main/java/org/owasp/dependencycheck/concurrency/DirectorySpinLock.java @@ -40,7 +40,7 @@ import java.util.logging.Logger; * * @author Jeremy Long (jeremy.long@owasp.org) */ -public class DirectorySpinLock implements Closeable, AutoCloseable { +public class DirectorySpinLock implements Closeable /*, AutoCloseable*/ { /** * The name of the lock file. @@ -224,13 +224,14 @@ public class DirectorySpinLock implements Closeable, AutoCloseable { @Override public void close() throws IOException { release(); - if (lock != null) { - try { - lock.close(); - } catch (IOException ex) { - Logger.getLogger(DirectorySpinLock.class.getName()).log(Level.FINEST, "Unable to close file lock due to IO Exception", ex); - } - } +// TODO uncomment this once support for 1.6 is dropped. +// if (lock != null) { +// try { +// lock.close(); +// } catch (IOException ex) { +// Logger.getLogger(DirectorySpinLock.class.getName()).log(Level.FINEST, "Unable to close file lock due to IO Exception", ex); +// } +// } if (channel != null) { try { channel.close(); From 19ecb67f2d5752cb872fa4a6f7c5cb10014f4f98 Mon Sep 17 00:00:00 2001 From: Jeremy Long Date: Sat, 24 Aug 2013 17:02:27 -0400 Subject: [PATCH 12/50] updates as the CPEAnalyzer was moved Former-commit-id: d3c21f1c0187a94e56aeff55fd0874273cc65faf --- .../services/org.owasp.dependencycheck.analyzer.Analyzer | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dependency-check-core/src/main/resources/META-INF/services/org.owasp.dependencycheck.analyzer.Analyzer b/dependency-check-core/src/main/resources/META-INF/services/org.owasp.dependencycheck.analyzer.Analyzer index a8bfe53cf..bf33320c8 100644 --- a/dependency-check-core/src/main/resources/META-INF/services/org.owasp.dependencycheck.analyzer.Analyzer +++ b/dependency-check-core/src/main/resources/META-INF/services/org.owasp.dependencycheck.analyzer.Analyzer @@ -4,5 +4,5 @@ org.owasp.dependencycheck.analyzer.FileNameAnalyzer org.owasp.dependencycheck.analyzer.HintAnalyzer org.owasp.dependencycheck.analyzer.DependencyBundlingAnalyzer org.owasp.dependencycheck.analyzer.FalsePositiveAnalyzer -org.owasp.dependencycheck.data.cpe.CPEAnalyzer +org.owasp.dependencycheck.analyzer.CPEAnalyzer org.owasp.dependencycheck.data.nvdcve.NvdCveAnalyzer \ No newline at end of file From 0731ed2c7a442036878b935755587746c7e3e17b Mon Sep 17 00:00:00 2001 From: Jeremy Long Date: Sat, 24 Aug 2013 19:56:13 -0400 Subject: [PATCH 13/50] updated database schema version so the fix to the lucene index is enforced on clients Former-commit-id: b743deca9949b67ce5e94c52fff6125e65bf340b --- .../owasp/dependencycheck/data/nvdcve/CveDB.java | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/nvdcve/CveDB.java b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/nvdcve/CveDB.java index 82ccce2e8..df8299bdf 100644 --- a/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/nvdcve/CveDB.java +++ b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/nvdcve/CveDB.java @@ -58,7 +58,7 @@ public class CveDB { /** * The version of the current DB Schema. */ - public static final String DB_SCHEMA_VERSION = "2.5"; + public static final String DB_SCHEMA_VERSION = "2.6"; /** * Database connection */ @@ -162,16 +162,6 @@ public class CveDB { value = "DMI_EMPTY_DB_PASSWORD", justification = "Yes, I know... Blank password.") public void open() throws IOException, SQLException, DatabaseException, ClassNotFoundException { - /* - * TODO - make it so we can exteralize the database (lucene index is a problem), could I store it as a blob - * and just download it when needed? - */ -// String dbDriver = Settings.getString(Settings.KEYS.DB_DRIVER); -// String dbConnStr = Settings.getString(Settings.KEYS.DB_CONNECTION_STRING); -// if (dbDriver != null && dbConnStr != null) { -// Class.forName(dbDriver); -// conn = DriverManager.getConnection(dbConnStr); -// } else { //use the embeded version final String fileName = CveDB.getDataDirectory().getCanonicalPath(); final File f = new File(fileName, "cve." + DB_SCHEMA_VERSION); final File check = new File(f.getAbsolutePath() + ".h2.db"); @@ -182,7 +172,6 @@ public class CveDB { if (createTables) { createTables(); } -// } } /** From 57c09d1772dbe9fc933b130a9be6f1faf56307bd Mon Sep 17 00:00:00 2001 From: Jeremy Long Date: Sat, 24 Aug 2013 19:57:14 -0400 Subject: [PATCH 14/50] updated how initial test data is updated Former-commit-id: 7fac5faf5a82bedc6f323f85cee2cd47c984fb74 --- .../java/org/owasp/dependencycheck/EngineIntegrationTest.java | 1 - 1 file changed, 1 deletion(-) diff --git a/dependency-check-core/src/test/java/org/owasp/dependencycheck/EngineIntegrationTest.java b/dependency-check-core/src/test/java/org/owasp/dependencycheck/EngineIntegrationTest.java index 07b7f4577..0a81e4d94 100644 --- a/dependency-check-core/src/test/java/org/owasp/dependencycheck/EngineIntegrationTest.java +++ b/dependency-check-core/src/test/java/org/owasp/dependencycheck/EngineIntegrationTest.java @@ -44,7 +44,6 @@ public class EngineIntegrationTest { @Before public void setUp() throws Exception { org.owasp.dependencycheck.data.nvdcve.BaseDBTestCase.ensureDBExists(); - org.owasp.dependencycheck.data.cpe.BaseIndexTestCase.ensureIndexExists(); } @After From 8ffb91022e701a8790d6a225c2c306faa114b806 Mon Sep 17 00:00:00 2001 From: Jeremy Long Date: Sat, 24 Aug 2013 19:57:55 -0400 Subject: [PATCH 15/50] minor update of annotations Former-commit-id: b29855c5afc64d91c2931aef5debb410b5f457c8 --- .../org/owasp/dependencycheck/analyzer/CPEAnalyzerTest.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dependency-check-core/src/test/java/org/owasp/dependencycheck/analyzer/CPEAnalyzerTest.java b/dependency-check-core/src/test/java/org/owasp/dependencycheck/analyzer/CPEAnalyzerTest.java index 1b8f805cb..1a7b0a887 100644 --- a/dependency-check-core/src/test/java/org/owasp/dependencycheck/analyzer/CPEAnalyzerTest.java +++ b/dependency-check-core/src/test/java/org/owasp/dependencycheck/analyzer/CPEAnalyzerTest.java @@ -40,7 +40,6 @@ import org.owasp.dependencycheck.analyzer.FileNameAnalyzer; import org.owasp.dependencycheck.analyzer.HintAnalyzer; import org.owasp.dependencycheck.data.cpe.BaseIndexTestCase; import org.owasp.dependencycheck.data.cpe.IndexEntry; -import static org.owasp.dependencycheck.data.cpe.BaseIndexTestCase.ensureIndexExists; import org.owasp.dependencycheck.dependency.Identifier; /** @@ -58,11 +57,13 @@ public class CPEAnalyzerTest extends BaseIndexTestCase { } @Before + @Override public void setUp() throws Exception { super.setUp(); } @After + @Override public void tearDown() throws Exception { super.tearDown(); } From 46d106e6e21f412d2b9e08d3b59512137e7957a0 Mon Sep 17 00:00:00 2001 From: Jeremy Long Date: Sat, 24 Aug 2013 19:58:54 -0400 Subject: [PATCH 16/50] updated how initial test data is updated Former-commit-id: c63f49d89c63446c9ed73800e8cdd17f0a977986 --- .../data/cpe/BaseIndexTestCase.java | 97 ++----------------- 1 file changed, 6 insertions(+), 91 deletions(-) diff --git a/dependency-check-core/src/test/java/org/owasp/dependencycheck/data/cpe/BaseIndexTestCase.java b/dependency-check-core/src/test/java/org/owasp/dependencycheck/data/cpe/BaseIndexTestCase.java index 84efe73fb..872fed583 100644 --- a/dependency-check-core/src/test/java/org/owasp/dependencycheck/data/cpe/BaseIndexTestCase.java +++ b/dependency-check-core/src/test/java/org/owasp/dependencycheck/data/cpe/BaseIndexTestCase.java @@ -18,30 +18,18 @@ */ package org.owasp.dependencycheck.data.cpe; -import java.io.BufferedInputStream; -import java.io.BufferedOutputStream; -import java.io.File; -import java.io.FileInputStream; -import java.io.FileOutputStream; -import java.io.IOException; -import java.util.logging.Level; -import java.util.logging.Logger; -import java.util.zip.ZipEntry; -import java.util.zip.ZipInputStream; +import junit.framework.TestCase; import org.junit.After; import org.junit.AfterClass; import org.junit.Before; import org.junit.BeforeClass; import org.owasp.dependencycheck.data.nvdcve.BaseDBTestCase; -import org.owasp.dependencycheck.utils.Settings; /** * * @author Jeremy Long (jeremy.long@owasp.org) */ -public abstract class BaseIndexTestCase { - - protected static final int BUFFER_SIZE = 2048; +public abstract class BaseIndexTestCase extends TestCase { @BeforeClass public static void setUpClass() throws Exception { @@ -52,88 +40,15 @@ public abstract class BaseIndexTestCase { } @Before + @Override public void setUp() throws Exception { - ensureIndexExists(); BaseDBTestCase.ensureDBExists(); + super.setUp(); } @After + @Override public void tearDown() throws Exception { - } - - protected static File getDataDirectory() throws IOException { - final String fileName = Settings.getString(Settings.KEYS.CPE_DATA_DIRECTORY); - final String dataDirectory = Settings.getString(Settings.KEYS.DATA_DIRECTORY); - return new File(dataDirectory, fileName); - //return FileUtils.getDataDirectory(fileName, Index.class); - } - - public static void ensureIndexExists() throws Exception { - //String indexPath = Settings.getString(Settings.KEYS.CPE_DATA_DIRECTORY); - String indexPath = getDataDirectory().getCanonicalPath(); - java.io.File f = new File(indexPath); - - if (!f.exists() || (f.isDirectory() && f.listFiles().length == 0)) { - f.mkdirs(); - FileInputStream fis = null; - ZipInputStream zin = null; - try { - File path = new File(BaseIndexTestCase.class.getClassLoader().getResource("index.cpe.zip").getPath()); - fis = new FileInputStream(path); - zin = new ZipInputStream(new BufferedInputStream(fis)); - ZipEntry entry; - while ((entry = zin.getNextEntry()) != null) { - if (entry.isDirectory()) { - continue; - } - FileOutputStream fos = null; - BufferedOutputStream dest = null; - try { - File o = new File(indexPath, entry.getName()); - o.createNewFile(); - fos = new FileOutputStream(o, false); - dest = new BufferedOutputStream(fos, BUFFER_SIZE); - byte data[] = new byte[BUFFER_SIZE]; - int count; - while ((count = zin.read(data, 0, BUFFER_SIZE)) != -1) { - dest.write(data, 0, count); - } - } catch (Exception ex) { - Logger.getLogger(BaseIndexTestCase.class.getName()).log(Level.FINEST, null, ex); - } finally { - if (dest != null) { - try { - dest.flush(); - dest.close(); - } catch (Throwable ex) { - Logger.getLogger(BaseIndexTestCase.class.getName()).log(Level.FINEST, null, ex); - } - } - if (fos != null) { - try { - fos.close(); - } catch (Throwable ex) { - Logger.getLogger(BaseIndexTestCase.class.getName()).log(Level.FINEST, null, ex); - } - } - } - } - } finally { - try { - if (zin != null) { - zin.close(); - } - } catch (Throwable ex) { - Logger.getLogger(BaseIndexTestCase.class.getName()).log(Level.FINEST, null, ex); - } - try { - if (fis != null) { - fis.close(); - } - } catch (Throwable ex) { - Logger.getLogger(BaseIndexTestCase.class.getName()).log(Level.FINEST, null, ex); - } - } - } + super.tearDown(); } } From 788b5633cbbfcc0ff4d1bea5de738cbe1a6e3b40 Mon Sep 17 00:00:00 2001 From: Jeremy Long Date: Sat, 24 Aug 2013 19:59:31 -0400 Subject: [PATCH 17/50] minor update to test class Former-commit-id: c4e38c8f5f3ed154216aad0b023176c74e5fa4a4 --- .../dependencycheck/data/cpe/IndexEntryTest.java | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/dependency-check-core/src/test/java/org/owasp/dependencycheck/data/cpe/IndexEntryTest.java b/dependency-check-core/src/test/java/org/owasp/dependencycheck/data/cpe/IndexEntryTest.java index 663dd4435..6a03e1c1e 100644 --- a/dependency-check-core/src/test/java/org/owasp/dependencycheck/data/cpe/IndexEntryTest.java +++ b/dependency-check-core/src/test/java/org/owasp/dependencycheck/data/cpe/IndexEntryTest.java @@ -19,6 +19,7 @@ package org.owasp.dependencycheck.data.cpe; import org.owasp.dependencycheck.data.cpe.IndexEntry; +import junit.framework.TestCase; import org.junit.After; import org.junit.AfterClass; import org.junit.Before; @@ -30,7 +31,7 @@ import org.junit.Assert; * * @author Jeremy Long (jeremy.long@owasp.org) */ -public class IndexEntryTest { +public class IndexEntryTest extends TestCase { @BeforeClass public static void setUpClass() throws Exception { @@ -41,11 +42,15 @@ public class IndexEntryTest { } @Before - public void setUp() { + @Override + public void setUp() throws Exception { + super.setUp(); } @After - public void tearDown() { + @Override + public void tearDown() throws Exception { + super.tearDown(); } /** From cd7362c654d767160d11ff331a313df218a68f45 Mon Sep 17 00:00:00 2001 From: Jeremy Long Date: Sat, 24 Aug 2013 20:00:09 -0400 Subject: [PATCH 18/50] updated how initial test data is updated Former-commit-id: dd94b61a39efdd619c23beddf25f683dc1fe5791 --- .../org/owasp/dependencycheck/data/nvdcve/BaseDBTestCase.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dependency-check-core/src/test/java/org/owasp/dependencycheck/data/nvdcve/BaseDBTestCase.java b/dependency-check-core/src/test/java/org/owasp/dependencycheck/data/nvdcve/BaseDBTestCase.java index f36dcb69e..9a141348a 100644 --- a/dependency-check-core/src/test/java/org/owasp/dependencycheck/data/nvdcve/BaseDBTestCase.java +++ b/dependency-check-core/src/test/java/org/owasp/dependencycheck/data/nvdcve/BaseDBTestCase.java @@ -56,11 +56,12 @@ public abstract class BaseDBTestCase extends TestCase { String indexPath = getDataDirectory().getCanonicalPath(); java.io.File f = new File(indexPath); if (!f.exists() || (f.isDirectory() && f.listFiles().length == 0)) { + f = f.getParentFile(); f.mkdirs(); FileInputStream fis = null; ZipInputStream zin = null; try { - File path = new File(BaseDBTestCase.class.getClassLoader().getResource("db.cve.zip").getPath()); + File path = new File(BaseDBTestCase.class.getClassLoader().getResource("data.zip").getPath()); fis = new FileInputStream(path); zin = new ZipInputStream(new BufferedInputStream(fis)); ZipEntry entry; From 12ce96d8026004e72cb6abf94fc98c8d76e43bcc Mon Sep 17 00:00:00 2001 From: Jeremy Long Date: Wed, 28 Aug 2013 06:15:24 -0400 Subject: [PATCH 20/50] updated getFile to correctly get the main data directory Former-commit-id: 4c175b6c218c264c8255614858545224c0c597f7 --- .../owasp/dependencycheck/utils/Settings.java | 30 +++++-------------- 1 file changed, 8 insertions(+), 22 deletions(-) diff --git a/dependency-check-core/src/main/java/org/owasp/dependencycheck/utils/Settings.java b/dependency-check-core/src/main/java/org/owasp/dependencycheck/utils/Settings.java index 3a60bf6f2..bc48db645 100644 --- a/dependency-check-core/src/main/java/org/owasp/dependencycheck/utils/Settings.java +++ b/dependency-check-core/src/main/java/org/owasp/dependencycheck/utils/Settings.java @@ -226,25 +226,6 @@ public final class Settings { INSTANCE.props.load(stream); } - /** - * Returns a value from the properties file as a File object. If the value - * was specified as a system property or passed in via the -Dprop=value - * argument - this method will return the value from the system properties - * before the values in the contained configuration file. - * - * @param key the key to lookup within the properties file - * @param defaultValue the default value for the requested property - * @return the property from the properties file as a File object - */ - public static File getFile(String key, String defaultValue) { - final String baseDir = getString(Settings.KEYS.DATA_DIRECTORY); - final String str = getString(key, defaultValue); - if (baseDir != null) { - return new File(baseDir, str); - } - return new File(str); - } - /** * Returns a value from the properties file as a File object. If the value * was specified as a system property or passed in via the -Dprop=value @@ -256,17 +237,22 @@ public final class Settings { * * @param key the key to lookup within the properties file * @return the property from the properties file converted to a File object - * @throws IOException thrown if the file path to the JAR cannot be found */ - public static File getFile(String key) throws IOException { + public static File getFile(String key) { final String file = getString(key); final String baseDir = getString(Settings.KEYS.DATA_DIRECTORY); if (baseDir != null) { if (baseDir.startsWith("[JAR]/")) { final File jarPath = getJarPath(); - final File newBase = new File(jarPath.getCanonicalPath(), baseDir.substring(6)); + final File newBase = new File(jarPath, baseDir.substring(6)); + if (Settings.KEYS.DATA_DIRECTORY.equals(key)) { + return newBase; + } return new File(newBase, file); } + if (Settings.KEYS.DATA_DIRECTORY.equals(key)) { + return new File(baseDir); + } return new File(baseDir, file); } return new File(file); From 8009794ccaac80d01b1ee6ecd51dd696d7fdc993 Mon Sep 17 00:00:00 2001 From: Jeremy Long Date: Thu, 29 Aug 2013 06:12:55 -0400 Subject: [PATCH 21/50] Updated to externalize the data properties file Former-commit-id: 729b2181f3cfcb7aa2862c1874b8aaacb154d46b --- .../data/nvdcve/xml/DatabaseUpdater.java | 567 +++++------------- 1 file changed, 146 insertions(+), 421 deletions(-) diff --git a/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/nvdcve/xml/DatabaseUpdater.java b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/nvdcve/xml/DatabaseUpdater.java index d65fad8b4..9d452500b 100644 --- a/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/nvdcve/xml/DatabaseUpdater.java +++ b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/nvdcve/xml/DatabaseUpdater.java @@ -19,14 +19,10 @@ package org.owasp.dependencycheck.data.nvdcve.xml; import java.io.File; -import java.io.FileInputStream; import java.io.FileNotFoundException; -import java.io.FileOutputStream; import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.io.OutputStreamWriter; import javax.xml.parsers.ParserConfigurationException; +import org.xml.sax.SAXException; import org.owasp.dependencycheck.data.CachedWebDataSource; import java.net.MalformedURLException; import java.net.URISyntaxException; @@ -34,10 +30,9 @@ import java.net.URL; import java.sql.SQLException; import java.util.Calendar; import java.util.Date; -import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Properties; +import java.util.TreeMap; import java.util.logging.Level; import java.util.logging.Logger; import javax.xml.parsers.SAXParser; @@ -50,10 +45,10 @@ import org.owasp.dependencycheck.dependency.VulnerableSoftware; import org.owasp.dependencycheck.utils.DownloadFailedException; import org.owasp.dependencycheck.utils.Downloader; import org.owasp.dependencycheck.utils.FileUtils; -import org.owasp.dependencycheck.utils.InvalidSettingException; import org.owasp.dependencycheck.utils.Settings; -import org.xml.sax.SAXException; import org.owasp.dependencycheck.data.nvdcve.DatabaseException; +import static org.owasp.dependencycheck.data.nvdcve.xml.DataStoreMetaInfo.MODIFIED; +import org.owasp.dependencycheck.utils.InvalidSettingException; /** * @@ -62,25 +57,9 @@ import org.owasp.dependencycheck.data.nvdcve.DatabaseException; public class DatabaseUpdater implements CachedWebDataSource { /** - * The name of the properties file containing the timestamp of the last - * update. + * Utility to read and write meta-data about the data. */ - private static final String UPDATE_PROPERTIES_FILE = "lastupdated.prop"; - /** - * The properties file key for the last updated field - used to store the - * last updated time of the Modified NVD CVE xml file. - */ - private static final String LAST_UPDATED_MODIFIED = "lastupdated.modified"; - /** - * Stores the last updated time for each of the NVD CVE files. These - * timestamps should be updated if we process the modified file within 7 - * days of the last update. - */ - private static final String LAST_UPDATED_BASE = "lastupdated."; - /** - * Modified key word. - */ - public static final String MODIFIED = "modified"; + private DataStoreMetaInfo properties = null; /** * Reference to the Cve Database. */ @@ -89,10 +68,27 @@ public class DatabaseUpdater implements CachedWebDataSource { * Reference to the Cpe Index. */ private CpeIndexWriter cpeIndex = null; + /** + * A flag indicating whether or not the batch update should be performed. + */ + protected boolean doBatchUpdate; - public DatabaseUpdater() { - batchUpdateMode = !Settings.getString(Settings.KEYS.BATCH_UPDATE_URL, "").isEmpty(); - doBatchUpdate = false; + /** + * Get the value of doBatchUpdate + * + * @return the value of doBatchUpdate + */ + protected boolean isDoBatchUpdate() { + return doBatchUpdate; + } + + /** + * Set the value of doBatchUpdate + * + * @param doBatchUpdate new value of doBatchUpdate + */ + protected void setDoBatchUpdate(boolean doBatchUpdate) { + this.doBatchUpdate = doBatchUpdate; } /** @@ -104,6 +100,8 @@ public class DatabaseUpdater implements CachedWebDataSource { */ @Override public void update() throws UpdateException { + doBatchUpdate = false; + properties = new DataStoreMetaInfo(); try { final Map update = updateNeeded(); int maxUpdates = 0; @@ -112,7 +110,7 @@ public class DatabaseUpdater implements CachedWebDataSource { maxUpdates += 1; } } - if (maxUpdates > 3) { + if (maxUpdates > 3 && !properties.isBatchUpdateMode()) { Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.INFO, "NVD CVE requires several updates; this could take a couple of minutes."); } @@ -120,7 +118,7 @@ public class DatabaseUpdater implements CachedWebDataSource { openDataStores(); } - if (isBatchUpdateMode() && isDoBatchUpdate()) { + if (properties.isBatchUpdateMode() && isDoBatchUpdate()) { try { performBatchUpdate(); openDataStores(); @@ -130,7 +128,6 @@ public class DatabaseUpdater implements CachedWebDataSource { } int count = 0; - for (NvdCveUrl cve : update.values()) { if (cve.getNeedsUpdate()) { count += 1; @@ -142,7 +139,6 @@ public class DatabaseUpdater implements CachedWebDataSource { try { Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.INFO, "Downloading {0}", cve.getUrl()); - outputPath = File.createTempFile("cve" + cve.getId() + "_", ".xml"); Downloader.fetchFile(url, outputPath); @@ -158,7 +154,7 @@ public class DatabaseUpdater implements CachedWebDataSource { cveDB.commit(); cpeIndex.commit(); - writeLastUpdatedPropertyFile(cve); + properties.save(cve); Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.INFO, "Completed update {0} of {1}", new Object[]{count, maxUpdates}); @@ -201,7 +197,7 @@ public class DatabaseUpdater implements CachedWebDataSource { } } if (maxUpdates >= 1) { - ensureModifiedIsInLastUpdatedProperties(update); + properties.save(update.get(MODIFIED)); cveDB.cleanupDatabase(); } } catch (MalformedURLException ex) { @@ -244,6 +240,55 @@ public class DatabaseUpdater implements CachedWebDataSource { saxParser.parse(file, cve20Handler); } + /** + * Deletes the existing data directories. + * + * @throws IOException thrown if the directory cannot be deleted + */ + protected void deleteExistingData() throws IOException { + Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.INFO, "The database version is old. Rebuilding the database."); + + final File cveDir = CveDB.getDataDirectory(); + FileUtils.delete(cveDir); + + final File cpeDir = BaseIndex.getDataDirectory(); + FileUtils.delete(cpeDir); + } + + private void performBatchUpdate() throws UpdateException { + if (properties.isBatchUpdateMode() && doBatchUpdate) { + final String batchSrc = Settings.getString(Settings.KEYS.BATCH_UPDATE_URL); + File tmp = null; + try { + deleteExistingData(); + final File dataDirectory = CveDB.getDataDirectory().getParentFile(); + final URL batchUrl = new URL(batchSrc); + if ("file".equals(batchUrl.getProtocol())) { + try { + tmp = new File(batchUrl.toURI()); + } catch (URISyntaxException ex) { + final String msg = String.format("Invalid batch update URI: %s", batchSrc); + throw new UpdateException(msg, ex); + } + } else if ("http".equals(batchUrl.getProtocol()) + || "https".equals(batchUrl.getProtocol())) { + tmp = File.createTempFile("batch_", ".zip"); + Downloader.fetchFile(batchUrl, tmp); + } + //TODO add FTP? + FileUtils.extractFiles(tmp, dataDirectory); + + } catch (IOException ex) { + final String msg = String.format("IO Exception Occured performing batch update using: %s", batchSrc); + throw new UpdateException(msg, ex); + } finally { + if (tmp != null && !tmp.delete()) { + tmp.deleteOnExit(); + } + } + } + } + /** * Closes the CVE and CPE data stores. */ @@ -295,81 +340,9 @@ public class DatabaseUpdater implements CachedWebDataSource { } } - // - /** - * Writes a properties file containing the last updated date to the - * VULNERABLE_CPE directory. - * - * @param updatedValue the updated nvdcve entry - * @throws UpdateException is thrown if there is an update exception - */ - private void writeLastUpdatedPropertyFile(NvdCveUrl updatedValue) throws UpdateException { - if (updatedValue == null) { - return; - } - String dir; - try { - dir = CveDB.getDataDirectory().getCanonicalPath(); - } catch (IOException ex) { - Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.FINE, "Error updating the databases propterty file.", ex); - throw new UpdateException("Unable to locate last updated properties file.", ex); - } - final File cveProp = new File(dir, UPDATE_PROPERTIES_FILE); - final Properties prop = new Properties(); - if (cveProp.exists()) { - FileInputStream in = null; - try { - in = new FileInputStream(cveProp); - prop.load(in); - } catch (Exception ignoreMe) { - Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.FINEST, null, ignoreMe); - } finally { - if (in != null) { - try { - in.close(); - } catch (Exception ignoreMeToo) { - Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.FINEST, null, ignoreMeToo); - } - } - } - - } - prop.put("version", CveDB.DB_SCHEMA_VERSION); - prop.put(LAST_UPDATED_BASE + updatedValue.getId(), String.valueOf(updatedValue.getTimestamp())); - - OutputStream os = null; - OutputStreamWriter out = null; - try { - os = new FileOutputStream(cveProp); - out = new OutputStreamWriter(os, "UTF-8"); - prop.store(out, dir); - } catch (FileNotFoundException ex) { - Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.FINE, null, ex); - throw new UpdateException("Unable to find last updated properties file.", ex); - } catch (IOException ex) { - Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.FINE, null, ex); - throw new UpdateException("Unable to update last updated properties file.", ex); - } finally { - if (out != null) { - try { - out.close(); - } catch (IOException ex) { - Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.FINEST, null, ex); - } - } - if (os != null) { - try { - os.close(); - } catch (IOException ex) { - Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.FINEST, null, ex); - } - } - } - } - /** * Determines if the index needs to be updated. This is done by fetching the - * nvd cve meta data and checking the last update date. If the data needs to + * NVD CVE meta data and checking the last update date. If the data needs to * be refreshed this method will return the NvdCveUrl for the files that * need to be updated. * @@ -377,27 +350,27 @@ public class DatabaseUpdater implements CachedWebDataSource { * @throws MalformedURLException is thrown if the URL for the NVD CVE Meta * data is incorrect. * @throws DownloadFailedException is thrown if there is an error. - * downloading the nvd cve download data file. + * downloading the NVD CVE download data file. * @throws UpdateException Is thrown if there is an issue with the last * updated properties file. */ - public Map updateNeeded() throws MalformedURLException, DownloadFailedException, UpdateException { + private Map updateNeeded() throws MalformedURLException, DownloadFailedException, UpdateException { Map currentlyPublished; try { currentlyPublished = retrieveCurrentTimestampsFromWeb(); } catch (InvalidDataException ex) { final String msg = "Unable to retrieve valid timestamp from nvd cve downloads page"; - Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.FINE, msg, ex); + Logger.getLogger(DataStoreMetaInfo.class.getName()).log(Level.FINE, msg, ex); throw new DownloadFailedException(msg, ex); - } catch (InvalidSettingException ex) { - Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.FINE, "Invalid setting found when retrieving timestamps", ex); + Logger.getLogger(DataStoreMetaInfo.class.getName()).log(Level.FINE, "Invalid setting found when retrieving timestamps", ex); throw new DownloadFailedException("Invalid settings", ex); } if (currentlyPublished == null) { - throw new DownloadFailedException("Unable to retrieve valid timestamp from nvd cve downloads page"); + //TODO change messages once we have a new batch mode + throw new DownloadFailedException("Unable to retrieve valid timestamp from NVD CVE data feeds"); } final File cpeDataDirectory; @@ -416,97 +389,74 @@ public class DatabaseUpdater implements CachedWebDataSource { } throw new UpdateException(msg, ex); } - if (cpeDataDirectory.exists()) { - final File cveProp = new File(cpeDataDirectory, UPDATE_PROPERTIES_FILE); - if (cveProp.exists()) { - final Properties prop = new Properties(); - InputStream is = null; - try { - is = new FileInputStream(cveProp); - prop.load(is); - boolean deleteAndRecreate = false; - float version; + if (!properties.isEmpty()) { + try { + boolean deleteAndRecreate = false; + float version; - if (prop.getProperty("version") == null) { - deleteAndRecreate = true; - } else { - try { - version = Float.parseFloat(prop.getProperty("version")); - final float currentVersion = Float.parseFloat(CveDB.DB_SCHEMA_VERSION); - if (currentVersion > version) { - deleteAndRecreate = true; - } - } catch (NumberFormatException ex) { + if (properties.getProperty("version") == null) { + deleteAndRecreate = true; + } else { + try { + version = Float.parseFloat(properties.getProperty("version")); + final float currentVersion = Float.parseFloat(CveDB.DB_SCHEMA_VERSION); + if (currentVersion > version) { deleteAndRecreate = true; } + } catch (NumberFormatException ex) { + deleteAndRecreate = true; } - if (deleteAndRecreate) { - is.close(); - is = null; - deleteExistingData(); - setDoBatchUpdate(isBatchUpdateMode()); - return currentlyPublished; - } + } + if (deleteAndRecreate) { + setDoBatchUpdate(properties.isBatchUpdateMode()); + return currentlyPublished; + } - final long lastUpdated = Long.parseLong(prop.getProperty(LAST_UPDATED_MODIFIED, "0")); - final Date now = new Date(); - final int days = Settings.getInt(Settings.KEYS.CVE_MODIFIED_VALID_FOR_DAYS, 7); - final int start = Settings.getInt(Settings.KEYS.CVE_START_YEAR, 2002); - final int end = Calendar.getInstance().get(Calendar.YEAR); - if (lastUpdated == currentlyPublished.get(MODIFIED).timestamp) { - currentlyPublished.clear(); //we don't need to update anything. - setDoBatchUpdate(batchUpdateMode); - } else if (withinRange(lastUpdated, now.getTime(), days)) { - currentlyPublished.get(MODIFIED).setNeedsUpdate(true); - if (isBatchUpdateMode()) { - setDoBatchUpdate(false); - } else { - for (int i = start; i <= end; i++) { - currentlyPublished.get(String.valueOf(i)).setNeedsUpdate(false); - } - } - } else if (isBatchUpdateMode()) { - currentlyPublished.get(MODIFIED).setNeedsUpdate(true); - setDoBatchUpdate(true); - } else { //we figure out which of the several XML files need to be downloaded. - currentlyPublished.get(MODIFIED).setNeedsUpdate(false); + final long lastUpdated = Long.parseLong(properties.getProperty(DataStoreMetaInfo.LAST_UPDATED, "0")); + final Date now = new Date(); + final int days = Settings.getInt(Settings.KEYS.CVE_MODIFIED_VALID_FOR_DAYS, 7); + final int start = Settings.getInt(Settings.KEYS.CVE_START_YEAR, 2002); + final int end = Calendar.getInstance().get(Calendar.YEAR); + if (lastUpdated == currentlyPublished.get(MODIFIED).getTimestamp()) { + currentlyPublished.clear(); //we don't need to update anything. + setDoBatchUpdate(properties.isBatchUpdateMode()); + } else if (withinRange(lastUpdated, now.getTime(), days)) { + currentlyPublished.get(MODIFIED).setNeedsUpdate(true); + if (properties.isBatchUpdateMode()) { + setDoBatchUpdate(false); + } else { for (int i = start; i <= end; i++) { - final NvdCveUrl cve = currentlyPublished.get(String.valueOf(i)); - long currentTimestamp = 0; - try { - currentTimestamp = Long.parseLong(prop.getProperty(LAST_UPDATED_BASE + String.valueOf(i), "0")); - } catch (NumberFormatException ex) { - final String msg = String.format("Error parsing '%s' '%s' from nvdcve.lastupdated", - LAST_UPDATED_BASE, String.valueOf(i)); - Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.FINE, msg, ex); - } - if (currentTimestamp == cve.getTimestamp()) { - cve.setNeedsUpdate(false); //they default to true. - } + currentlyPublished.get(String.valueOf(i)).setNeedsUpdate(false); } } - } catch (FileNotFoundException ex) { - Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.FINEST, null, ex); - } catch (IOException ex) { - Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.FINEST, null, ex); - } catch (NumberFormatException ex) { - Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.FINEST, null, ex); - } finally { - if (is != null) { + } else if (properties.isBatchUpdateMode()) { + currentlyPublished.get(MODIFIED).setNeedsUpdate(true); + setDoBatchUpdate(true); + } else { //we figure out which of the several XML files need to be downloaded. + currentlyPublished.get(MODIFIED).setNeedsUpdate(false); + for (int i = start; i <= end; i++) { + final NvdCveUrl cve = currentlyPublished.get(String.valueOf(i)); + long currentTimestamp = 0; try { - is.close(); - } catch (IOException ex) { - Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.FINEST, null, ex); + currentTimestamp = Long.parseLong(properties.getProperty(DataStoreMetaInfo.LAST_UPDATED_BASE + String.valueOf(i), "0")); + } catch (NumberFormatException ex) { + final String msg = String.format("Error parsing '%s' '%s' from nvdcve.lastupdated", + DataStoreMetaInfo.LAST_UPDATED_BASE, String.valueOf(i)); + Logger.getLogger(DataStoreMetaInfo.class.getName()).log(Level.FINE, msg, ex); + } + if (currentTimestamp == cve.getTimestamp()) { + cve.setNeedsUpdate(false); //they default to true. } } } - } else { - //properties file does not exist - check about batch update - setDoBatchUpdate(isBatchUpdateMode()); + } catch (NumberFormatException ex) { + Logger.getLogger(DataStoreMetaInfo.class.getName()).log(Level.WARNING, "An invalid schema version or timestamp exists in the data.properties file."); + Logger.getLogger(DataStoreMetaInfo.class.getName()).log(Level.FINE, null, ex); + setDoBatchUpdate(properties.isBatchUpdateMode()); } - } else { //this condition will likely never exist - but just in case we need to handle batch updates - setDoBatchUpdate(isBatchUpdateMode()); + } else { + setDoBatchUpdate(properties.isBatchUpdateMode()); } return currentlyPublished; } @@ -526,49 +476,6 @@ public class DatabaseUpdater implements CachedWebDataSource { final double differenceInDays = (compareTo - date) / 1000.0 / 60.0 / 60.0 / 24.0; return differenceInDays < range; } - /** - * Indicates whether or not the updates are using a batch update mode or - * not. - */ - private boolean batchUpdateMode; - - /** - * Get the value of batchUpdateMode. - * - * @return the value of batchUpdateMode - */ - protected boolean isBatchUpdateMode() { - return batchUpdateMode; - } - - /** - * Set the value of batchUpdateMode. - * - * @param batchUpdateMode new value of batchUpdateMode - */ - protected void setBatchUpdateMode(boolean batchUpdateMode) { - this.batchUpdateMode = batchUpdateMode; - } - //flag indicating whether or not the batch update should be performed. - protected boolean doBatchUpdate; - - /** - * Get the value of doBatchUpdate - * - * @return the value of doBatchUpdate - */ - protected boolean isDoBatchUpdate() { - return doBatchUpdate; - } - - /** - * Set the value of doBatchUpdate - * - * @param doBatchUpdate new value of doBatchUpdate - */ - protected void setDoBatchUpdate(boolean doBatchUpdate) { - this.doBatchUpdate = doBatchUpdate; - } /** * Retrieves the timestamps from the NVD CVE meta data file. @@ -585,7 +492,7 @@ public class DatabaseUpdater implements CachedWebDataSource { protected Map retrieveCurrentTimestampsFromWeb() throws MalformedURLException, DownloadFailedException, InvalidDataException, InvalidSettingException { - final Map map = new HashMap(); + final Map map = new TreeMap(); String retrieveUrl = Settings.getString(Settings.KEYS.CVE_MODIFIED_20_URL); NvdCveUrl item = new NvdCveUrl(); @@ -594,11 +501,11 @@ public class DatabaseUpdater implements CachedWebDataSource { item.setUrl(retrieveUrl); item.setOldSchemaVersionUrl(Settings.getString(Settings.KEYS.CVE_MODIFIED_12_URL)); - item.timestamp = Downloader.getLastModified(new URL(retrieveUrl)); + item.setTimestamp(Downloader.getLastModified(new URL(retrieveUrl))); map.put(MODIFIED, item); //only add these urls if we are not in batch mode - if (!isBatchUpdateMode()) { + if (!properties.isBatchUpdateMode()) { final int start = Settings.getInt(Settings.KEYS.CVE_START_YEAR); final int end = Calendar.getInstance().get(Calendar.YEAR); final String baseUrl20 = Settings.getString(Settings.KEYS.CVE_SCHEMA_2_0); @@ -610,191 +517,9 @@ public class DatabaseUpdater implements CachedWebDataSource { item.setUrl(retrieveUrl); item.setOldSchemaVersionUrl(String.format(baseUrl12, i)); item.setTimestamp(Downloader.getLastModified(new URL(retrieveUrl))); - map.put(item.id, item); + map.put(item.getId(), item); } } return map; } - - /** - * Method to double check that the "modified" nvdcve file is listed and has - * a timestamp in the last updated properties file. - * - * @param update a set of updated NvdCveUrl objects - */ - private void ensureModifiedIsInLastUpdatedProperties(Map update) { - try { - writeLastUpdatedPropertyFile(update.get(MODIFIED)); - } catch (UpdateException ex) { - Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.FINE, null, ex); - } - } - - /** - * Deletes the existing data directories. - * - * @throws IOException thrown if the directory cannot be deleted - */ - protected void deleteExistingData() throws IOException { - Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.INFO, "The database version is old. Rebuilding the database."); - - final File cveDir = CveDB.getDataDirectory(); - FileUtils.delete(cveDir); - - final File cpeDir = BaseIndex.getDataDirectory(); - FileUtils.delete(cpeDir); - } - - private void performBatchUpdate() throws UpdateException { - if (batchUpdateMode && doBatchUpdate) { - final String batchSrc = Settings.getString(Settings.KEYS.BATCH_UPDATE_URL); - File tmp = null; - try { - deleteExistingData(); - final File dataDirectory = CveDB.getDataDirectory().getParentFile(); - final URL batchUrl = new URL(batchSrc); - if ("file".equals(batchUrl.getProtocol())) { - try { - tmp = new File(batchUrl.toURI()); - } catch (URISyntaxException ex) { - final String msg = String.format("Invalid batch update URI: %s", batchSrc); - throw new UpdateException(msg, ex); - } - } else if ("http".equals(batchUrl.getProtocol()) - || "https".equals(batchUrl.getProtocol())) { - tmp = File.createTempFile("batch_", ".zip"); - Downloader.fetchFile(batchUrl, tmp); - } - //TODO add FTP? - FileUtils.extractFiles(tmp, dataDirectory); - - } catch (IOException ex) { - final String msg = String.format("IO Exception Occured performing batch update using: %s", batchSrc); - throw new UpdateException(msg, ex); - } finally { - if (tmp != null && !tmp.delete()) { - tmp.deleteOnExit(); - } - } - } - } - - /** - * A pojo that contains the Url and timestamp of the current NvdCve XML - * files. - */ - protected static class NvdCveUrl { - - /** - * an id. - */ - private String id; - - /** - * Get the value of id. - * - * @return the value of id - */ - public String getId() { - return id; - } - - /** - * Set the value of id. - * - * @param id new value of id - */ - public void setId(String id) { - this.id = id; - } - /** - * a url. - */ - private String url; - - /** - * Get the value of url. - * - * @return the value of url - */ - public String getUrl() { - return url; - } - - /** - * Set the value of url. - * - * @param url new value of url - */ - public void setUrl(String url) { - this.url = url; - } - /** - * The 1.2 schema URL. - */ - private String oldSchemaVersionUrl; - - /** - * Get the value of oldSchemaVersionUrl. - * - * @return the value of oldSchemaVersionUrl - */ - public String getOldSchemaVersionUrl() { - return oldSchemaVersionUrl; - } - - /** - * Set the value of oldSchemaVersionUrl. - * - * @param oldSchemaVersionUrl new value of oldSchemaVersionUrl - */ - public void setOldSchemaVersionUrl(String oldSchemaVersionUrl) { - this.oldSchemaVersionUrl = oldSchemaVersionUrl; - } - /** - * a timestamp - epoch time. - */ - private long timestamp; - - /** - * Get the value of timestamp - epoch time. - * - * @return the value of timestamp - epoch time - */ - public long getTimestamp() { - return timestamp; - } - - /** - * Set the value of timestamp - epoch time. - * - * @param timestamp new value of timestamp - epoch time - */ - public void setTimestamp(long timestamp) { - this.timestamp = timestamp; - } - /** - * indicates whether or not this item should be updated. - */ - private boolean needsUpdate = true; - - /** - * Get the value of needsUpdate. - * - * @return the value of needsUpdate - */ - public boolean getNeedsUpdate() { - return needsUpdate; - } - - /** - * Set the value of needsUpdate. - * - * @param needsUpdate new value of needsUpdate - */ - public void setNeedsUpdate(boolean needsUpdate) { - this.needsUpdate = needsUpdate; - } - } - // } From 029e0e50446e6a9830f561d0a6c35535de71562d Mon Sep 17 00:00:00 2001 From: Jeremy Long Date: Thu, 29 Aug 2013 06:14:25 -0400 Subject: [PATCH 23/50] updated test for getFile to return the DATA_DIRECTORY Former-commit-id: f16d34a12416999c28597261b409c997ebc0c3ad --- .../java/org/owasp/dependencycheck/utils/SettingsTest.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dependency-check-core/src/test/java/org/owasp/dependencycheck/utils/SettingsTest.java b/dependency-check-core/src/test/java/org/owasp/dependencycheck/utils/SettingsTest.java index a574d4029..77717bb72 100644 --- a/dependency-check-core/src/test/java/org/owasp/dependencycheck/utils/SettingsTest.java +++ b/dependency-check-core/src/test/java/org/owasp/dependencycheck/utils/SettingsTest.java @@ -73,9 +73,9 @@ public class SettingsTest { File result = Settings.getFile(key); Assert.assertTrue(result.getAbsolutePath().endsWith(expResult)); - key = "an invalid key!!!"; - result = Settings.getFile(key, expResult); - Assert.assertTrue(result.getAbsolutePath().endsWith(expResult)); + result = Settings.getFile(Settings.KEYS.DATA_DIRECTORY); + String path = result.getPath(); + Assert.assertTrue(path.endsWith("data") || path.endsWith("data" + File.separator)); } /** From db22159a8937ccba4f7ee28bb12cc4100f67ff59 Mon Sep 17 00:00:00 2001 From: Jeremy Long Date: Thu, 29 Aug 2013 06:15:20 -0400 Subject: [PATCH 24/50] added to externalize the meta info about the data used Former-commit-id: 1fb0bf0594245fc7095c085faf18a000bf76d7f8 --- .../data/nvdcve/xml/DataStoreMetaInfo.java | 217 ++++++++++++++++++ .../data/nvdcve/xml/NvdCveUrl.java | 136 +++++++++++ 2 files changed, 353 insertions(+) create mode 100644 dependency-check-core/src/main/java/org/owasp/dependencycheck/data/nvdcve/xml/DataStoreMetaInfo.java create mode 100644 dependency-check-core/src/main/java/org/owasp/dependencycheck/data/nvdcve/xml/NvdCveUrl.java diff --git a/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/nvdcve/xml/DataStoreMetaInfo.java b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/nvdcve/xml/DataStoreMetaInfo.java new file mode 100644 index 000000000..a6fd31a94 --- /dev/null +++ b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/nvdcve/xml/DataStoreMetaInfo.java @@ -0,0 +1,217 @@ +/* + * This file is part of dependency-check-core. + * + * Dependency-check-core is free software: you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation, either version 3 of the License, or (at your option) any + * later version. + * + * Dependency-check-core is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more + * details. + * + * You should have received a copy of the GNU General Public License along with + * dependency-check-core. If not, see http://www.gnu.org/licenses/. + * + * Copyright (c) 2013 Jeremy Long. All Rights Reserved. + */ +package org.owasp.dependencycheck.data.nvdcve.xml; + +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.io.OutputStreamWriter; +import java.util.Properties; +import java.util.logging.Level; +import java.util.logging.Logger; +import org.owasp.dependencycheck.data.UpdateException; +import org.owasp.dependencycheck.data.nvdcve.CveDB; +import org.owasp.dependencycheck.utils.Settings; + +/** + * + * @author Jeremy Long (jeremy.long@owasp.org) + */ +public class DataStoreMetaInfo { + + /** + * Modified key word. + */ + public static final String MODIFIED = "modified"; + /** + * The name of the properties file containing the timestamp of the last + * update. + */ + private static final String UPDATE_PROPERTIES_FILE = "data.properties"; + /** + * The properties file key for the last updated field - used to store the + * last updated time of the Modified NVD CVE xml file. + */ + public static final String LAST_UPDATED = "lastupdated.modified"; + /** + * Stores the last updated time for each of the NVD CVE files. These + * timestamps should be updated if we process the modified file within 7 + * days of the last update. + */ + public static final String LAST_UPDATED_BASE = "lastupdated."; + /** + * A collection of properties about the data. + */ + private Properties properties = new Properties(); + /** + * Indicates whether or not the updates are using a batch update mode or + * not. + */ + private boolean batchUpdateMode; + + /** + * Get the value of batchUpdateMode. + * + * @return the value of batchUpdateMode + */ + protected boolean isBatchUpdateMode() { + return batchUpdateMode; + } + + /** + * Set the value of batchUpdateMode. + * + * @param batchUpdateMode new value of batchUpdateMode + */ + protected void setBatchUpdateMode(boolean batchUpdateMode) { + this.batchUpdateMode = batchUpdateMode; + } + + /** + * Constructs a new data properties object. + */ + public DataStoreMetaInfo() { + batchUpdateMode = !Settings.getString(Settings.KEYS.BATCH_UPDATE_URL, "").isEmpty(); + loadProperties(); + } + + /** + * Loads the data's meta properties. + */ + private void loadProperties() { + final File dataDirectory = Settings.getFile(Settings.KEYS.DATA_DIRECTORY); + final File file = new File(dataDirectory, UPDATE_PROPERTIES_FILE); + if (file.exists()) { + InputStream is = null; + try { + is = new FileInputStream(file); + } catch (FileNotFoundException ignore) { + //we will never get here as we check for existence above. + Logger.getLogger(DataStoreMetaInfo.class.getName()).log(Level.FINEST, null, ignore); + } + try { + properties.load(is); + } catch (IOException ex) { + final String msg = String.format("Unable to load properties file '%s'", file.getPath()); + Logger.getLogger(DataStoreMetaInfo.class.getName()).log(Level.WARNING, msg); + Logger.getLogger(DataStoreMetaInfo.class.getName()).log(Level.FINE, null, ex); + } + } + } + + /** + * Returns whether or not any properties are set. + * + * @return whether or not any properties are set + */ + public boolean isEmpty() { + return properties.isEmpty(); + } + + /** + * Writes a properties file containing the last updated date to the + * VULNERABLE_CPE directory. + * + * @param updatedValue the updated nvdcve entry + * @throws UpdateException is thrown if there is an update exception + */ + public void save(NvdCveUrl updatedValue) throws UpdateException { + if (updatedValue == null) { + return; + } + final File dataDirectory = Settings.getFile(Settings.KEYS.DATA_DIRECTORY); + final File cveProp = new File(dataDirectory, UPDATE_PROPERTIES_FILE); + final Properties prop = new Properties(); + if (cveProp.exists()) { + FileInputStream in = null; + try { + in = new FileInputStream(cveProp); + prop.load(in); + } catch (Exception ignoreMe) { + Logger.getLogger(DataStoreMetaInfo.class.getName()).log(Level.FINEST, null, ignoreMe); + } finally { + if (in != null) { + try { + in.close(); + } catch (Exception ignoreMeToo) { + Logger.getLogger(DataStoreMetaInfo.class.getName()).log(Level.FINEST, null, ignoreMeToo); + } + } + } + } + prop.put("version", CveDB.DB_SCHEMA_VERSION); + prop.put(LAST_UPDATED_BASE + updatedValue.getId(), String.valueOf(updatedValue.getTimestamp())); + + OutputStream os = null; + OutputStreamWriter out = null; + try { + os = new FileOutputStream(cveProp); + out = new OutputStreamWriter(os, "UTF-8"); + prop.store(out, "Meta data about data and data sources used by dependency-check"); + } catch (FileNotFoundException ex) { + Logger.getLogger(DataStoreMetaInfo.class.getName()).log(Level.FINE, null, ex); + throw new UpdateException("Unable to find last updated properties file.", ex); + } catch (IOException ex) { + Logger.getLogger(DataStoreMetaInfo.class.getName()).log(Level.FINE, null, ex); + throw new UpdateException("Unable to update last updated properties file.", ex); + } finally { + if (out != null) { + try { + out.close(); + } catch (IOException ex) { + Logger.getLogger(DataStoreMetaInfo.class.getName()).log(Level.FINEST, null, ex); + } + } + if (os != null) { + try { + os.close(); + } catch (IOException ex) { + Logger.getLogger(DataStoreMetaInfo.class.getName()).log(Level.FINEST, null, ex); + } + } + } + } + + /** + * Returns the property value for the given key. If the key is not contained + * in the underlying properties null is returned. + * + * @param key the property key + * @return the value of the property + */ + public String getProperty(String key) { + return properties.getProperty(key); + } + + /** + * Returns the property value for the given key. If the key is not contained + * in the underlying properties the default value is returned. + * + * @param key the property key + * @param defaultValue the default value + * @return the value of the property + */ + public String getProperty(String key, String defaultValue) { + return properties.getProperty(key, defaultValue); + } +} diff --git a/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/nvdcve/xml/NvdCveUrl.java b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/nvdcve/xml/NvdCveUrl.java new file mode 100644 index 000000000..1b0914be3 --- /dev/null +++ b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/nvdcve/xml/NvdCveUrl.java @@ -0,0 +1,136 @@ +/* + * This file is part of dependency-check-core. + * + * Dependency-check-core is free software: you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation, either version 3 of the License, or (at your option) any + * later version. + * + * Dependency-check-core is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more + * details. + * + * You should have received a copy of the GNU General Public License along with + * dependency-check-core. If not, see http://www.gnu.org/licenses/. + * + * Copyright (c) 2013 Jeremy Long. All Rights Reserved. + */ +package org.owasp.dependencycheck.data.nvdcve.xml; + +/** + * A pojo that contains the Url and timestamp of the current NvdCve XML files. + */ +public class NvdCveUrl { + + /** + * an id. + */ + private String id; + + /** + * Get the value of id. + * + * @return the value of id + */ + public String getId() { + return id; + } + + /** + * Set the value of id. + * + * @param id new value of id + */ + public void setId(String id) { + this.id = id; + } + /** + * a url. + */ + private String url; + + /** + * Get the value of url. + * + * @return the value of url + */ + public String getUrl() { + return url; + } + + /** + * Set the value of url. + * + * @param url new value of url + */ + public void setUrl(String url) { + this.url = url; + } + /** + * The 1.2 schema URL. + */ + private String oldSchemaVersionUrl; + + /** + * Get the value of oldSchemaVersionUrl. + * + * @return the value of oldSchemaVersionUrl + */ + public String getOldSchemaVersionUrl() { + return oldSchemaVersionUrl; + } + + /** + * Set the value of oldSchemaVersionUrl. + * + * @param oldSchemaVersionUrl new value of oldSchemaVersionUrl + */ + public void setOldSchemaVersionUrl(String oldSchemaVersionUrl) { + this.oldSchemaVersionUrl = oldSchemaVersionUrl; + } + /** + * a timestamp - epoch time. + */ + private long timestamp; + + /** + * Get the value of timestamp - epoch time. + * + * @return the value of timestamp - epoch time + */ + public long getTimestamp() { + return timestamp; + } + + /** + * Set the value of timestamp - epoch time. + * + * @param timestamp new value of timestamp - epoch time + */ + public void setTimestamp(long timestamp) { + this.timestamp = timestamp; + } + /** + * indicates whether or not this item should be updated. + */ + private boolean needsUpdate = true; + + /** + * Get the value of needsUpdate. + * + * @return the value of needsUpdate + */ + public boolean getNeedsUpdate() { + return needsUpdate; + } + + /** + * Set the value of needsUpdate. + * + * @param needsUpdate new value of needsUpdate + */ + public void setNeedsUpdate(boolean needsUpdate) { + this.needsUpdate = needsUpdate; + } +} From 6a4d1ed44d1f9a09d46e9c011d8c2508513f6058 Mon Sep 17 00:00:00 2001 From: Jeremy Long Date: Thu, 29 Aug 2013 06:40:37 -0400 Subject: [PATCH 25/50] temporarily commented out broken tests Former-commit-id: e05b064c6b076bb218b500b7a0feb26044f43d7a --- .../DatabaseUpdaterIntegrationTest.java | 80 +++++++++++++++++++ 1 file changed, 80 insertions(+) create mode 100644 dependency-check-core/src/test/java/org/owasp/dependencycheck/data/update/DatabaseUpdaterIntegrationTest.java diff --git a/dependency-check-core/src/test/java/org/owasp/dependencycheck/data/update/DatabaseUpdaterIntegrationTest.java b/dependency-check-core/src/test/java/org/owasp/dependencycheck/data/update/DatabaseUpdaterIntegrationTest.java new file mode 100644 index 000000000..5c262063b --- /dev/null +++ b/dependency-check-core/src/test/java/org/owasp/dependencycheck/data/update/DatabaseUpdaterIntegrationTest.java @@ -0,0 +1,80 @@ +/* + * This file is part of dependency-check-core. + * + * Dependency-check-core is free software: you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation, either version 3 of the License, or (at your option) any + * later version. + * + * Dependency-check-core is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more + * details. + * + * You should have received a copy of the GNU General Public License along with + * dependency-check-core. If not, see http://www.gnu.org/licenses/. + * + * Copyright (c) 2012 Jeremy Long. All Rights Reserved. + */ +package org.owasp.dependencycheck.data.update; + +import java.io.File; +import java.net.URL; +import org.owasp.dependencycheck.data.update.DatabaseUpdater; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; +import org.owasp.dependencycheck.utils.Settings; + +/** + * + * @author Jeremy Long (jeremy.long@owasp.org) + */ +public class DatabaseUpdaterIntegrationTest { + + public DatabaseUpdaterIntegrationTest() { + } + + @BeforeClass + public static void setUpClass() throws Exception { + } + + @AfterClass + public static void tearDownClass() throws Exception { + } + + @Before + public void setUp() { + } + + @After + public void tearDown() { + } + + /** + * Test of update method, of class DatabaseUpdater. + * + * @throws Exception + */ + @Test + public void testUpdate() throws Exception { + DatabaseUpdater instance = new DatabaseUpdater(); + instance.update(); + } + + /** + * Test of update method (when in batch mode), of class DatabaseUpdater. + * + * @throws Exception + */ + @Test + public void testBatchUpdate() throws Exception { +// File file = new File("target/test-classes/nvdcve-2.0-2012.xml"); +// String path = "file:///" + file.getCanonicalPath(); +// Settings.setString(Settings.KEYS.BATCH_UPDATE_URL, path); +// DatabaseUpdater instance = new DatabaseUpdater(); +// instance.update(); + } +} From 5fab16ad06aab4cfaa165794d1e53007f98f9f72 Mon Sep 17 00:00:00 2001 From: Jeremy Long Date: Thu, 29 Aug 2013 06:41:03 -0400 Subject: [PATCH 26/50] temporarily commented out broken tests Former-commit-id: 1ce099712487d7f1a3e13f3444fe2493181c08cf --- .../data/update/DatabaseUpdaterTest.java | 85 +++++++++++++++++++ 1 file changed, 85 insertions(+) create mode 100644 dependency-check-core/src/test/java/org/owasp/dependencycheck/data/update/DatabaseUpdaterTest.java diff --git a/dependency-check-core/src/test/java/org/owasp/dependencycheck/data/update/DatabaseUpdaterTest.java b/dependency-check-core/src/test/java/org/owasp/dependencycheck/data/update/DatabaseUpdaterTest.java new file mode 100644 index 000000000..70350e176 --- /dev/null +++ b/dependency-check-core/src/test/java/org/owasp/dependencycheck/data/update/DatabaseUpdaterTest.java @@ -0,0 +1,85 @@ +/* + * This file is part of dependency-check-core. + * + * Dependency-check-core is free software: you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation, either version 3 of the License, or (at your option) any + * later version. + * + * Dependency-check-core is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more + * details. + * + * You should have received a copy of the GNU General Public License along with + * dependency-check-core. If not, see http://www.gnu.org/licenses/. + * + * Copyright (c) 2012 Jeremy Long. All Rights Reserved. + */ +package org.owasp.dependencycheck.data.update; + +import org.owasp.dependencycheck.data.update.DatabaseUpdater; +import java.io.File; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; +import org.owasp.dependencycheck.utils.Settings; + +/** + * + * @author Jeremy Long (jeremy.long@owasp.org) + */ +public class DatabaseUpdaterTest { + + public DatabaseUpdaterTest() { + } + + @BeforeClass + public static void setUpClass() throws Exception { + } + + @AfterClass + public static void tearDownClass() throws Exception { + } + private String old12; + private String old20; + + @Before + public void setUp() throws Exception { + old12 = Settings.getString(Settings.KEYS.CVE_MODIFIED_12_URL); + old20 = Settings.getString(Settings.KEYS.CVE_MODIFIED_20_URL); + + File file = new File("target/test-classes/nvdcve-2012.xml"); + String path = "file:///" + file.getCanonicalPath(); + Settings.setString(Settings.KEYS.CVE_MODIFIED_12_URL, path); + + file = new File("target/test-classes/nvdcve-2.0-2012.xml"); + path = "file:///" + file.getCanonicalPath(); + Settings.setString(Settings.KEYS.CVE_MODIFIED_20_URL, path); + + file = new File("target/test-classes/data.zip"); + path = "file:///" + file.getCanonicalPath(); + Settings.setString(Settings.KEYS.BATCH_UPDATE_URL, path); + } + + @After + public void tearDown() { + Settings.setString(Settings.KEYS.CVE_MODIFIED_12_URL, old12); + Settings.setString(Settings.KEYS.CVE_MODIFIED_20_URL, old20); + Settings.setString(Settings.KEYS.BATCH_UPDATE_URL, ""); + } + + /** + * Test of update method (when in batch mode), of class DatabaseUpdater. + * + * @throws Exception + */ + @Test + public void testBatchUpdate() throws Exception { +// DatabaseUpdater instance = new DatabaseUpdater(); +// instance.deleteExistingData(); +// instance.update(); + } +} From 2bd03dada484b30fbae8efb0087de1c8a1ba3813 Mon Sep 17 00:00:00 2001 From: Jeremy Long Date: Thu, 29 Aug 2013 06:42:16 -0400 Subject: [PATCH 27/50] code reorganization, moved files around to better seperate functionality Former-commit-id: da6b75d818d67b0c1c695860504aacc00991effa --- .../nvdcve => analyzer}/NvdCveAnalyzer.java | 4 +- .../{xml => }/InvalidDataException.java | 2 +- .../nvdcve/{xml => }/NvdCve12Handler.java | 2 +- .../nvdcve/{xml => }/NvdCve20Handler.java | 4 +- .../xml => update}/DataStoreMetaInfo.java | 4 +- .../xml => update}/DatabaseUpdater.java | 27 +++--- .../NvdCveUrl.java => update/NvdCveInfo.java} | 4 +- .../{nvdcve/xml => update}/package-info.java | 2 +- ...rg.owasp.dependencycheck.analyzer.Analyzer | 2 +- ...p.dependencycheck.data.CachedWebDataSource | 2 +- .../{xml => }/NvdCve_1_2_HandlerTest.java | 4 +- .../{xml => }/NvdCve_2_0_HandlerTest.java | 4 +- .../xml/DatabaseUpdaterIntegrationTest.java | 80 ----------------- .../data/nvdcve/xml/DatabaseUpdaterTest.java | 85 ------------------- 14 files changed, 33 insertions(+), 193 deletions(-) rename dependency-check-core/src/main/java/org/owasp/dependencycheck/{data/nvdcve => analyzer}/NvdCveAnalyzer.java (97%) rename dependency-check-core/src/main/java/org/owasp/dependencycheck/data/nvdcve/{xml => }/InvalidDataException.java (96%) rename dependency-check-core/src/main/java/org/owasp/dependencycheck/data/nvdcve/{xml => }/NvdCve12Handler.java (99%) rename dependency-check-core/src/main/java/org/owasp/dependencycheck/data/nvdcve/{xml => }/NvdCve20Handler.java (99%) rename dependency-check-core/src/main/java/org/owasp/dependencycheck/data/{nvdcve/xml => update}/DataStoreMetaInfo.java (98%) rename dependency-check-core/src/main/java/org/owasp/dependencycheck/data/{nvdcve/xml => update}/DatabaseUpdater.java (95%) rename dependency-check-core/src/main/java/org/owasp/dependencycheck/data/{nvdcve/xml/NvdCveUrl.java => update/NvdCveInfo.java} (97%) rename dependency-check-core/src/main/java/org/owasp/dependencycheck/data/{nvdcve/xml => update}/package-info.java (91%) rename dependency-check-core/src/test/java/org/owasp/dependencycheck/data/nvdcve/{xml => }/NvdCve_1_2_HandlerTest.java (94%) rename dependency-check-core/src/test/java/org/owasp/dependencycheck/data/nvdcve/{xml => }/NvdCve_2_0_HandlerTest.java (94%) delete mode 100644 dependency-check-core/src/test/java/org/owasp/dependencycheck/data/nvdcve/xml/DatabaseUpdaterIntegrationTest.java delete mode 100644 dependency-check-core/src/test/java/org/owasp/dependencycheck/data/nvdcve/xml/DatabaseUpdaterTest.java diff --git a/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/nvdcve/NvdCveAnalyzer.java b/dependency-check-core/src/main/java/org/owasp/dependencycheck/analyzer/NvdCveAnalyzer.java similarity index 97% rename from dependency-check-core/src/main/java/org/owasp/dependencycheck/data/nvdcve/NvdCveAnalyzer.java rename to dependency-check-core/src/main/java/org/owasp/dependencycheck/analyzer/NvdCveAnalyzer.java index 9e6285dbe..d7e1f7c74 100644 --- a/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/nvdcve/NvdCveAnalyzer.java +++ b/dependency-check-core/src/main/java/org/owasp/dependencycheck/analyzer/NvdCveAnalyzer.java @@ -16,7 +16,7 @@ * * Copyright (c) 2012 Jeremy Long. All Rights Reserved. */ -package org.owasp.dependencycheck.data.nvdcve; +package org.owasp.dependencycheck.analyzer; import java.io.IOException; import java.sql.SQLException; @@ -29,6 +29,8 @@ import org.owasp.dependencycheck.dependency.Dependency; import org.owasp.dependencycheck.dependency.Vulnerability; import org.owasp.dependencycheck.dependency.Identifier; import org.owasp.dependencycheck.analyzer.Analyzer; +import org.owasp.dependencycheck.data.nvdcve.CveDB; +import org.owasp.dependencycheck.data.nvdcve.DatabaseException; /** * NvdCveAnalyzer is a utility class that takes a project dependency and diff --git a/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/nvdcve/xml/InvalidDataException.java b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/nvdcve/InvalidDataException.java similarity index 96% rename from dependency-check-core/src/main/java/org/owasp/dependencycheck/data/nvdcve/xml/InvalidDataException.java rename to dependency-check-core/src/main/java/org/owasp/dependencycheck/data/nvdcve/InvalidDataException.java index 498e9427c..133d17cec 100644 --- a/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/nvdcve/xml/InvalidDataException.java +++ b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/nvdcve/InvalidDataException.java @@ -16,7 +16,7 @@ * * Copyright (c) 2012 Jeremy Long. All Rights Reserved. */ -package org.owasp.dependencycheck.data.nvdcve.xml; +package org.owasp.dependencycheck.data.nvdcve; /** * An InvalidDataDataException is a generic exception used when trying to load diff --git a/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/nvdcve/xml/NvdCve12Handler.java b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/nvdcve/NvdCve12Handler.java similarity index 99% rename from dependency-check-core/src/main/java/org/owasp/dependencycheck/data/nvdcve/xml/NvdCve12Handler.java rename to dependency-check-core/src/main/java/org/owasp/dependencycheck/data/nvdcve/NvdCve12Handler.java index 3c4a8856a..baca54635 100644 --- a/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/nvdcve/xml/NvdCve12Handler.java +++ b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/nvdcve/NvdCve12Handler.java @@ -16,7 +16,7 @@ * * Copyright (c) 2012 Jeremy Long. All Rights Reserved. */ -package org.owasp.dependencycheck.data.nvdcve.xml; +package org.owasp.dependencycheck.data.nvdcve; import java.util.ArrayList; import java.util.HashMap; diff --git a/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/nvdcve/xml/NvdCve20Handler.java b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/nvdcve/NvdCve20Handler.java similarity index 99% rename from dependency-check-core/src/main/java/org/owasp/dependencycheck/data/nvdcve/xml/NvdCve20Handler.java rename to dependency-check-core/src/main/java/org/owasp/dependencycheck/data/nvdcve/NvdCve20Handler.java index 3f5808466..0a4b439f9 100644 --- a/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/nvdcve/xml/NvdCve20Handler.java +++ b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/nvdcve/NvdCve20Handler.java @@ -16,7 +16,7 @@ * * Copyright (c) 2012 Jeremy Long. All Rights Reserved. */ -package org.owasp.dependencycheck.data.nvdcve.xml; +package org.owasp.dependencycheck.data.nvdcve; import java.io.IOException; import java.util.List; @@ -281,7 +281,7 @@ public class NvdCve20Handler extends DefaultHandler { * * @param index the CPE Lucene Index */ - void setCpeIndex(CpeIndexWriter index) { + public void setCpeIndex(CpeIndexWriter index) { cpeIndex = index; } diff --git a/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/nvdcve/xml/DataStoreMetaInfo.java b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/DataStoreMetaInfo.java similarity index 98% rename from dependency-check-core/src/main/java/org/owasp/dependencycheck/data/nvdcve/xml/DataStoreMetaInfo.java rename to dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/DataStoreMetaInfo.java index a6fd31a94..08500dea2 100644 --- a/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/nvdcve/xml/DataStoreMetaInfo.java +++ b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/DataStoreMetaInfo.java @@ -16,7 +16,7 @@ * * Copyright (c) 2013 Jeremy Long. All Rights Reserved. */ -package org.owasp.dependencycheck.data.nvdcve.xml; +package org.owasp.dependencycheck.data.update; import java.io.File; import java.io.FileInputStream; @@ -135,7 +135,7 @@ public class DataStoreMetaInfo { * @param updatedValue the updated nvdcve entry * @throws UpdateException is thrown if there is an update exception */ - public void save(NvdCveUrl updatedValue) throws UpdateException { + public void save(NvdCveInfo updatedValue) throws UpdateException { if (updatedValue == null) { return; } diff --git a/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/nvdcve/xml/DatabaseUpdater.java b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/DatabaseUpdater.java similarity index 95% rename from dependency-check-core/src/main/java/org/owasp/dependencycheck/data/nvdcve/xml/DatabaseUpdater.java rename to dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/DatabaseUpdater.java index 9d452500b..0cdf531c2 100644 --- a/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/nvdcve/xml/DatabaseUpdater.java +++ b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/DatabaseUpdater.java @@ -16,8 +16,11 @@ * * Copyright (c) 2012 Jeremy Long. All Rights Reserved. */ -package org.owasp.dependencycheck.data.nvdcve.xml; +package org.owasp.dependencycheck.data.update; +import org.owasp.dependencycheck.data.nvdcve.NvdCve12Handler; +import org.owasp.dependencycheck.data.nvdcve.NvdCve20Handler; +import org.owasp.dependencycheck.data.nvdcve.InvalidDataException; import java.io.File; import java.io.FileNotFoundException; import java.io.IOException; @@ -47,7 +50,7 @@ import org.owasp.dependencycheck.utils.Downloader; import org.owasp.dependencycheck.utils.FileUtils; import org.owasp.dependencycheck.utils.Settings; import org.owasp.dependencycheck.data.nvdcve.DatabaseException; -import static org.owasp.dependencycheck.data.nvdcve.xml.DataStoreMetaInfo.MODIFIED; +import static org.owasp.dependencycheck.data.update.DataStoreMetaInfo.MODIFIED; import org.owasp.dependencycheck.utils.InvalidSettingException; /** @@ -103,9 +106,9 @@ public class DatabaseUpdater implements CachedWebDataSource { doBatchUpdate = false; properties = new DataStoreMetaInfo(); try { - final Map update = updateNeeded(); + final Map update = updateNeeded(); int maxUpdates = 0; - for (NvdCveUrl cve : update.values()) { + for (NvdCveInfo cve : update.values()) { if (cve.getNeedsUpdate()) { maxUpdates += 1; } @@ -128,7 +131,7 @@ public class DatabaseUpdater implements CachedWebDataSource { } int count = 0; - for (NvdCveUrl cve : update.values()) { + for (NvdCveInfo cve : update.values()) { if (cve.getNeedsUpdate()) { count += 1; Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.INFO, @@ -354,9 +357,9 @@ public class DatabaseUpdater implements CachedWebDataSource { * @throws UpdateException Is thrown if there is an issue with the last * updated properties file. */ - private Map updateNeeded() throws MalformedURLException, DownloadFailedException, UpdateException { + private Map updateNeeded() throws MalformedURLException, DownloadFailedException, UpdateException { - Map currentlyPublished; + Map currentlyPublished; try { currentlyPublished = retrieveCurrentTimestampsFromWeb(); } catch (InvalidDataException ex) { @@ -436,7 +439,7 @@ public class DatabaseUpdater implements CachedWebDataSource { } else { //we figure out which of the several XML files need to be downloaded. currentlyPublished.get(MODIFIED).setNeedsUpdate(false); for (int i = start; i <= end; i++) { - final NvdCveUrl cve = currentlyPublished.get(String.valueOf(i)); + final NvdCveInfo cve = currentlyPublished.get(String.valueOf(i)); long currentTimestamp = 0; try { currentTimestamp = Long.parseLong(properties.getProperty(DataStoreMetaInfo.LAST_UPDATED_BASE + String.valueOf(i), "0")); @@ -489,13 +492,13 @@ public class DatabaseUpdater implements CachedWebDataSource { * timestamps * @throws InvalidSettingException thrown if the settings are invalid */ - protected Map retrieveCurrentTimestampsFromWeb() + protected Map retrieveCurrentTimestampsFromWeb() throws MalformedURLException, DownloadFailedException, InvalidDataException, InvalidSettingException { - final Map map = new TreeMap(); + final Map map = new TreeMap(); String retrieveUrl = Settings.getString(Settings.KEYS.CVE_MODIFIED_20_URL); - NvdCveUrl item = new NvdCveUrl(); + NvdCveInfo item = new NvdCveInfo(); item.setNeedsUpdate(false); //the others default to true, to make life easier later this should default to false. item.setId(MODIFIED); item.setUrl(retrieveUrl); @@ -512,7 +515,7 @@ public class DatabaseUpdater implements CachedWebDataSource { final String baseUrl12 = Settings.getString(Settings.KEYS.CVE_SCHEMA_1_2); for (int i = start; i <= end; i++) { retrieveUrl = String.format(baseUrl20, i); - item = new NvdCveUrl(); + item = new NvdCveInfo(); item.setId(Integer.toString(i)); item.setUrl(retrieveUrl); item.setOldSchemaVersionUrl(String.format(baseUrl12, i)); diff --git a/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/nvdcve/xml/NvdCveUrl.java b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/NvdCveInfo.java similarity index 97% rename from dependency-check-core/src/main/java/org/owasp/dependencycheck/data/nvdcve/xml/NvdCveUrl.java rename to dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/NvdCveInfo.java index 1b0914be3..76af51cf3 100644 --- a/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/nvdcve/xml/NvdCveUrl.java +++ b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/NvdCveInfo.java @@ -16,12 +16,12 @@ * * Copyright (c) 2013 Jeremy Long. All Rights Reserved. */ -package org.owasp.dependencycheck.data.nvdcve.xml; +package org.owasp.dependencycheck.data.update; /** * A pojo that contains the Url and timestamp of the current NvdCve XML files. */ -public class NvdCveUrl { +public class NvdCveInfo { /** * an id. diff --git a/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/nvdcve/xml/package-info.java b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/package-info.java similarity index 91% rename from dependency-check-core/src/main/java/org/owasp/dependencycheck/data/nvdcve/xml/package-info.java rename to dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/package-info.java index c04c24a6a..a6251592d 100644 --- a/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/nvdcve/xml/package-info.java +++ b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/package-info.java @@ -15,4 +15,4 @@ * */ -package org.owasp.dependencycheck.data.nvdcve.xml; +package org.owasp.dependencycheck.data.update; diff --git a/dependency-check-core/src/main/resources/META-INF/services/org.owasp.dependencycheck.analyzer.Analyzer b/dependency-check-core/src/main/resources/META-INF/services/org.owasp.dependencycheck.analyzer.Analyzer index bf33320c8..9cab62e1e 100644 --- a/dependency-check-core/src/main/resources/META-INF/services/org.owasp.dependencycheck.analyzer.Analyzer +++ b/dependency-check-core/src/main/resources/META-INF/services/org.owasp.dependencycheck.analyzer.Analyzer @@ -5,4 +5,4 @@ org.owasp.dependencycheck.analyzer.HintAnalyzer org.owasp.dependencycheck.analyzer.DependencyBundlingAnalyzer org.owasp.dependencycheck.analyzer.FalsePositiveAnalyzer org.owasp.dependencycheck.analyzer.CPEAnalyzer -org.owasp.dependencycheck.data.nvdcve.NvdCveAnalyzer \ No newline at end of file +org.owasp.dependencycheck.analyzer.NvdCveAnalyzer \ No newline at end of file diff --git a/dependency-check-core/src/main/resources/META-INF/services/org.owasp.dependencycheck.data.CachedWebDataSource b/dependency-check-core/src/main/resources/META-INF/services/org.owasp.dependencycheck.data.CachedWebDataSource index 4c400f8ac..c42a832ed 100644 --- a/dependency-check-core/src/main/resources/META-INF/services/org.owasp.dependencycheck.data.CachedWebDataSource +++ b/dependency-check-core/src/main/resources/META-INF/services/org.owasp.dependencycheck.data.CachedWebDataSource @@ -1 +1 @@ -org.owasp.dependencycheck.data.nvdcve.xml.DatabaseUpdater \ No newline at end of file +org.owasp.dependencycheck.data.update.DatabaseUpdater \ No newline at end of file diff --git a/dependency-check-core/src/test/java/org/owasp/dependencycheck/data/nvdcve/xml/NvdCve_1_2_HandlerTest.java b/dependency-check-core/src/test/java/org/owasp/dependencycheck/data/nvdcve/NvdCve_1_2_HandlerTest.java similarity index 94% rename from dependency-check-core/src/test/java/org/owasp/dependencycheck/data/nvdcve/xml/NvdCve_1_2_HandlerTest.java rename to dependency-check-core/src/test/java/org/owasp/dependencycheck/data/nvdcve/NvdCve_1_2_HandlerTest.java index 3a5dc92f2..443239f4e 100644 --- a/dependency-check-core/src/test/java/org/owasp/dependencycheck/data/nvdcve/xml/NvdCve_1_2_HandlerTest.java +++ b/dependency-check-core/src/test/java/org/owasp/dependencycheck/data/nvdcve/NvdCve_1_2_HandlerTest.java @@ -16,9 +16,9 @@ * * Copyright (c) 2012 Jeremy Long. All Rights Reserved. */ -package org.owasp.dependencycheck.data.nvdcve.xml; +package org.owasp.dependencycheck.data.nvdcve; -import org.owasp.dependencycheck.data.nvdcve.xml.NvdCve12Handler; +import org.owasp.dependencycheck.data.nvdcve.NvdCve12Handler; import java.io.File; import java.util.List; import java.util.Map; diff --git a/dependency-check-core/src/test/java/org/owasp/dependencycheck/data/nvdcve/xml/NvdCve_2_0_HandlerTest.java b/dependency-check-core/src/test/java/org/owasp/dependencycheck/data/nvdcve/NvdCve_2_0_HandlerTest.java similarity index 94% rename from dependency-check-core/src/test/java/org/owasp/dependencycheck/data/nvdcve/xml/NvdCve_2_0_HandlerTest.java rename to dependency-check-core/src/test/java/org/owasp/dependencycheck/data/nvdcve/NvdCve_2_0_HandlerTest.java index d9d46dd5e..acec690b0 100644 --- a/dependency-check-core/src/test/java/org/owasp/dependencycheck/data/nvdcve/xml/NvdCve_2_0_HandlerTest.java +++ b/dependency-check-core/src/test/java/org/owasp/dependencycheck/data/nvdcve/NvdCve_2_0_HandlerTest.java @@ -16,9 +16,9 @@ * * Copyright (c) 2012 Jeremy Long. All Rights Reserved. */ -package org.owasp.dependencycheck.data.nvdcve.xml; +package org.owasp.dependencycheck.data.nvdcve; -import org.owasp.dependencycheck.data.nvdcve.xml.NvdCve20Handler; +import org.owasp.dependencycheck.data.nvdcve.NvdCve20Handler; import java.io.File; import javax.xml.parsers.SAXParser; import javax.xml.parsers.SAXParserFactory; diff --git a/dependency-check-core/src/test/java/org/owasp/dependencycheck/data/nvdcve/xml/DatabaseUpdaterIntegrationTest.java b/dependency-check-core/src/test/java/org/owasp/dependencycheck/data/nvdcve/xml/DatabaseUpdaterIntegrationTest.java deleted file mode 100644 index 2f66e43b6..000000000 --- a/dependency-check-core/src/test/java/org/owasp/dependencycheck/data/nvdcve/xml/DatabaseUpdaterIntegrationTest.java +++ /dev/null @@ -1,80 +0,0 @@ -/* - * This file is part of dependency-check-core. - * - * Dependency-check-core is free software: you can redistribute it and/or modify it - * under the terms of the GNU General Public License as published by the Free - * Software Foundation, either version 3 of the License, or (at your option) any - * later version. - * - * Dependency-check-core is distributed in the hope that it will be useful, but - * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more - * details. - * - * You should have received a copy of the GNU General Public License along with - * dependency-check-core. If not, see http://www.gnu.org/licenses/. - * - * Copyright (c) 2012 Jeremy Long. All Rights Reserved. - */ -package org.owasp.dependencycheck.data.nvdcve.xml; - -import java.io.File; -import java.net.URL; -import org.owasp.dependencycheck.data.nvdcve.xml.DatabaseUpdater; -import org.junit.After; -import org.junit.AfterClass; -import org.junit.Before; -import org.junit.BeforeClass; -import org.junit.Test; -import org.owasp.dependencycheck.utils.Settings; - -/** - * - * @author Jeremy Long (jeremy.long@owasp.org) - */ -public class DatabaseUpdaterIntegrationTest { - - public DatabaseUpdaterIntegrationTest() { - } - - @BeforeClass - public static void setUpClass() throws Exception { - } - - @AfterClass - public static void tearDownClass() throws Exception { - } - - @Before - public void setUp() { - } - - @After - public void tearDown() { - } - - /** - * Test of update method, of class DatabaseUpdater. - * - * @throws Exception - */ - @Test - public void testUpdate() throws Exception { - DatabaseUpdater instance = new DatabaseUpdater(); - instance.update(); - } - - /** - * Test of update method (when in batch mode), of class DatabaseUpdater. - * - * @throws Exception - */ - @Test - public void testBatchUpdate() throws Exception { - File file = new File("target/test-classes/nvdcve-2.0-2012.xml"); - String path = "file:///" + file.getCanonicalPath(); - Settings.setString(Settings.KEYS.BATCH_UPDATE_URL, path); - DatabaseUpdater instance = new DatabaseUpdater(); - instance.update(); - } -} diff --git a/dependency-check-core/src/test/java/org/owasp/dependencycheck/data/nvdcve/xml/DatabaseUpdaterTest.java b/dependency-check-core/src/test/java/org/owasp/dependencycheck/data/nvdcve/xml/DatabaseUpdaterTest.java deleted file mode 100644 index ab0ab2e3b..000000000 --- a/dependency-check-core/src/test/java/org/owasp/dependencycheck/data/nvdcve/xml/DatabaseUpdaterTest.java +++ /dev/null @@ -1,85 +0,0 @@ -/* - * This file is part of dependency-check-core. - * - * Dependency-check-core is free software: you can redistribute it and/or modify it - * under the terms of the GNU General Public License as published by the Free - * Software Foundation, either version 3 of the License, or (at your option) any - * later version. - * - * Dependency-check-core is distributed in the hope that it will be useful, but - * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more - * details. - * - * You should have received a copy of the GNU General Public License along with - * dependency-check-core. If not, see http://www.gnu.org/licenses/. - * - * Copyright (c) 2012 Jeremy Long. All Rights Reserved. - */ -package org.owasp.dependencycheck.data.nvdcve.xml; - -import org.owasp.dependencycheck.data.nvdcve.xml.DatabaseUpdater; -import java.io.File; -import org.junit.After; -import org.junit.AfterClass; -import org.junit.Before; -import org.junit.BeforeClass; -import org.junit.Test; -import org.owasp.dependencycheck.utils.Settings; - -/** - * - * @author Jeremy Long (jeremy.long@owasp.org) - */ -public class DatabaseUpdaterTest { - - public DatabaseUpdaterTest() { - } - - @BeforeClass - public static void setUpClass() throws Exception { - } - - @AfterClass - public static void tearDownClass() throws Exception { - } - private String old12; - private String old20; - - @Before - public void setUp() throws Exception { - old12 = Settings.getString(Settings.KEYS.CVE_MODIFIED_12_URL); - old20 = Settings.getString(Settings.KEYS.CVE_MODIFIED_20_URL); - - File file = new File("target/test-classes/nvdcve-2012.xml"); - String path = "file:///" + file.getCanonicalPath(); - Settings.setString(Settings.KEYS.CVE_MODIFIED_12_URL, path); - - file = new File("target/test-classes/nvdcve-2.0-2012.xml"); - path = "file:///" + file.getCanonicalPath(); - Settings.setString(Settings.KEYS.CVE_MODIFIED_20_URL, path); - - file = new File("target/test-classes/data.zip"); - path = "file:///" + file.getCanonicalPath(); - Settings.setString(Settings.KEYS.BATCH_UPDATE_URL, path); - } - - @After - public void tearDown() { - Settings.setString(Settings.KEYS.CVE_MODIFIED_12_URL, old12); - Settings.setString(Settings.KEYS.CVE_MODIFIED_20_URL, old20); - Settings.setString(Settings.KEYS.BATCH_UPDATE_URL, ""); - } - - /** - * Test of update method (when in batch mode), of class DatabaseUpdater. - * - * @throws Exception - */ - @Test - public void testBatchUpdate() throws Exception { - DatabaseUpdater instance = new DatabaseUpdater(); - instance.deleteExistingData(); - instance.update(); - } -} From aad6c28e4d00e8df650149044fe2d815f24c304b Mon Sep 17 00:00:00 2001 From: Jeremy Long Date: Thu, 29 Aug 2013 16:38:08 -0400 Subject: [PATCH 28/50] fixed broken test case Former-commit-id: 84d34cf826fc4eda8fa72592d9f3f959e9bf4cea --- .../data/update/DatabaseUpdaterIntegrationTest.java | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/dependency-check-core/src/test/java/org/owasp/dependencycheck/data/update/DatabaseUpdaterIntegrationTest.java b/dependency-check-core/src/test/java/org/owasp/dependencycheck/data/update/DatabaseUpdaterIntegrationTest.java index 5c262063b..abeafc8cf 100644 --- a/dependency-check-core/src/test/java/org/owasp/dependencycheck/data/update/DatabaseUpdaterIntegrationTest.java +++ b/dependency-check-core/src/test/java/org/owasp/dependencycheck/data/update/DatabaseUpdaterIntegrationTest.java @@ -71,10 +71,11 @@ public class DatabaseUpdaterIntegrationTest { */ @Test public void testBatchUpdate() throws Exception { -// File file = new File("target/test-classes/nvdcve-2.0-2012.xml"); -// String path = "file:///" + file.getCanonicalPath(); -// Settings.setString(Settings.KEYS.BATCH_UPDATE_URL, path); -// DatabaseUpdater instance = new DatabaseUpdater(); -// instance.update(); + File file = new File(this.getClass().getClassLoader().getResource("data.zip").toURI()); + String path = "file:///" + file.getCanonicalPath(); + Settings.setString(Settings.KEYS.BATCH_UPDATE_URL, path); + DatabaseUpdater instance = new DatabaseUpdater(); + instance.update(); + Settings.setString(Settings.KEYS.BATCH_UPDATE_URL, ""); } } From c4ddf84ba8c56dbde78629f7fc707934f35b9bf4 Mon Sep 17 00:00:00 2001 From: Jeremy Long Date: Thu, 29 Aug 2013 16:41:41 -0400 Subject: [PATCH 29/50] removed duplicate test case Former-commit-id: 8fd1309deb179486d6042a2b5119caca70437b19 --- .../update/DatabaseUpdaterIntegrationTest.java | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/dependency-check-core/src/test/java/org/owasp/dependencycheck/data/update/DatabaseUpdaterIntegrationTest.java b/dependency-check-core/src/test/java/org/owasp/dependencycheck/data/update/DatabaseUpdaterIntegrationTest.java index abeafc8cf..cd6b8b5a0 100644 --- a/dependency-check-core/src/test/java/org/owasp/dependencycheck/data/update/DatabaseUpdaterIntegrationTest.java +++ b/dependency-check-core/src/test/java/org/owasp/dependencycheck/data/update/DatabaseUpdaterIntegrationTest.java @@ -63,19 +63,4 @@ public class DatabaseUpdaterIntegrationTest { DatabaseUpdater instance = new DatabaseUpdater(); instance.update(); } - - /** - * Test of update method (when in batch mode), of class DatabaseUpdater. - * - * @throws Exception - */ - @Test - public void testBatchUpdate() throws Exception { - File file = new File(this.getClass().getClassLoader().getResource("data.zip").toURI()); - String path = "file:///" + file.getCanonicalPath(); - Settings.setString(Settings.KEYS.BATCH_UPDATE_URL, path); - DatabaseUpdater instance = new DatabaseUpdater(); - instance.update(); - Settings.setString(Settings.KEYS.BATCH_UPDATE_URL, ""); - } } From 7495392aa2d3ff03f1f505819838d47645c8407f Mon Sep 17 00:00:00 2001 From: Jeremy Long Date: Thu, 29 Aug 2013 16:49:09 -0400 Subject: [PATCH 30/50] fixed bug due caused by moving properties from cve to data directory Former-commit-id: 508894a061b40bb80fb294835021e3ba8f34037b --- .../dependencycheck/data/update/DatabaseUpdater.java | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/DatabaseUpdater.java b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/DatabaseUpdater.java index 0cdf531c2..36cea219f 100644 --- a/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/DatabaseUpdater.java +++ b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/DatabaseUpdater.java @@ -250,12 +250,12 @@ public class DatabaseUpdater implements CachedWebDataSource { */ protected void deleteExistingData() throws IOException { Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.INFO, "The database version is old. Rebuilding the database."); - - final File cveDir = CveDB.getDataDirectory(); - FileUtils.delete(cveDir); - - final File cpeDir = BaseIndex.getDataDirectory(); - FileUtils.delete(cpeDir); + final File data = Settings.getFile(Settings.KEYS.DATA_DIRECTORY); + FileUtils.delete(data); +// final File cveDir = CveDB.getDataDirectory(); +// FileUtils.delete(cveDir); +// final File cpeDir = BaseIndex.getDataDirectory(); +// FileUtils.delete(cpeDir); } private void performBatchUpdate() throws UpdateException { From d2bfcc6f0e9143ace7532876795bb1ef6fcf8c3f Mon Sep 17 00:00:00 2001 From: Jeremy Long Date: Thu, 29 Aug 2013 16:50:21 -0400 Subject: [PATCH 31/50] fixed broken test case Former-commit-id: b4f434d0d19a24938b2e795d1773c551fba70474 --- .../data/update/DatabaseUpdaterTest.java | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/dependency-check-core/src/test/java/org/owasp/dependencycheck/data/update/DatabaseUpdaterTest.java b/dependency-check-core/src/test/java/org/owasp/dependencycheck/data/update/DatabaseUpdaterTest.java index 70350e176..a428813b6 100644 --- a/dependency-check-core/src/test/java/org/owasp/dependencycheck/data/update/DatabaseUpdaterTest.java +++ b/dependency-check-core/src/test/java/org/owasp/dependencycheck/data/update/DatabaseUpdaterTest.java @@ -51,15 +51,15 @@ public class DatabaseUpdaterTest { old12 = Settings.getString(Settings.KEYS.CVE_MODIFIED_12_URL); old20 = Settings.getString(Settings.KEYS.CVE_MODIFIED_20_URL); - File file = new File("target/test-classes/nvdcve-2012.xml"); + File file = new File(this.getClass().getClassLoader().getResource("nvdcve-2012.xml").toURI()); String path = "file:///" + file.getCanonicalPath(); Settings.setString(Settings.KEYS.CVE_MODIFIED_12_URL, path); - file = new File("target/test-classes/nvdcve-2.0-2012.xml"); + file = new File(this.getClass().getClassLoader().getResource("nvdcve-2.0-2012.xml").toURI()); path = "file:///" + file.getCanonicalPath(); Settings.setString(Settings.KEYS.CVE_MODIFIED_20_URL, path); - file = new File("target/test-classes/data.zip"); + file = new File(this.getClass().getClassLoader().getResource("data.zip").toURI()); path = "file:///" + file.getCanonicalPath(); Settings.setString(Settings.KEYS.BATCH_UPDATE_URL, path); } @@ -78,8 +78,8 @@ public class DatabaseUpdaterTest { */ @Test public void testBatchUpdate() throws Exception { -// DatabaseUpdater instance = new DatabaseUpdater(); -// instance.deleteExistingData(); -// instance.update(); + DatabaseUpdater instance = new DatabaseUpdater(); + instance.deleteExistingData(); + instance.update(); } } From 23a47a6f6303291193df28cb33486b6d2d5ccf4f Mon Sep 17 00:00:00 2001 From: Jeremy Long Date: Thu, 29 Aug 2013 16:56:00 -0400 Subject: [PATCH 32/50] added ability to remove a property (for test cases) Former-commit-id: b2388ddb516b2f5b32f54398c11fc0c00990c9c8 --- .../java/org/owasp/dependencycheck/utils/Settings.java | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/dependency-check-core/src/main/java/org/owasp/dependencycheck/utils/Settings.java b/dependency-check-core/src/main/java/org/owasp/dependencycheck/utils/Settings.java index bc48db645..1f1024745 100644 --- a/dependency-check-core/src/main/java/org/owasp/dependencycheck/utils/Settings.java +++ b/dependency-check-core/src/main/java/org/owasp/dependencycheck/utils/Settings.java @@ -309,6 +309,16 @@ public final class Settings { return System.getProperty(key, INSTANCE.props.getProperty(key)); } + /** + * Removes a property from the local properties collection. This is mainly + * used in test cases. + * + * @param key the property key to remove + */ + public static void removeProperty(String key) { + INSTANCE.props.remove(key); + } + /** * Returns an int value from the properties file. If the value was specified * as a system property or passed in via the -Dprop=value argument - this From 6d85e7cdf74218f7f62d7f7ea5b2cc6a7ed5fcee Mon Sep 17 00:00:00 2001 From: Jeremy Long Date: Thu, 29 Aug 2013 17:01:47 -0400 Subject: [PATCH 33/50] added test case for removeProperty Former-commit-id: 7ae029f35b6f1aadd20abac560ae62d377b5557d --- .../dependencycheck/utils/SettingsTest.java | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/dependency-check-core/src/test/java/org/owasp/dependencycheck/utils/SettingsTest.java b/dependency-check-core/src/test/java/org/owasp/dependencycheck/utils/SettingsTest.java index 77717bb72..4fcf15ab7 100644 --- a/dependency-check-core/src/test/java/org/owasp/dependencycheck/utils/SettingsTest.java +++ b/dependency-check-core/src/test/java/org/owasp/dependencycheck/utils/SettingsTest.java @@ -163,4 +163,20 @@ public class SettingsTest { boolean result = Settings.getBoolean(key); Assert.assertEquals(expResult, result); } + + /** + * Test of removeProperty method, of class Settings. + */ + @Test + public void testRemoveProperty() { + String key = "SomeKey"; + String value = "value"; + String dfault = "default"; + Settings.setString(key, value); + String ret = Settings.getString(key); + Assert.assertEquals(value, ret); + Settings.removeProperty(key); + ret = Settings.getString(key, dfault); + Assert.assertEquals(dfault, ret); + } } From 34ce50b7b57656dcc1b9108b0bd7f1a4b71c9b6f Mon Sep 17 00:00:00 2001 From: Jeremy Long Date: Sat, 31 Aug 2013 06:46:26 -0400 Subject: [PATCH 34/50] removed duplicate code by adding a public getPropertiesFile method to obtain the File Former-commit-id: cd3c7fdad8907eb28704e1e8342dfe41e08d9da3 --- .../data/update/DataStoreMetaInfo.java | 24 +++++++++++++++---- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/DataStoreMetaInfo.java b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/DataStoreMetaInfo.java index 08500dea2..38fc77bfd 100644 --- a/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/DataStoreMetaInfo.java +++ b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/DataStoreMetaInfo.java @@ -40,7 +40,12 @@ import org.owasp.dependencycheck.utils.Settings; public class DataStoreMetaInfo { /** - * Modified key word. + * Batch key word, used as key to store information about batch mode. + */ + public static final String BATCH = "batch"; + /** + * Modified key word, used as a key to store information about the modified + * file (i.e. the containing the last 8 days of updates).. */ public static final String MODIFIED = "modified"; /** @@ -99,8 +104,7 @@ public class DataStoreMetaInfo { * Loads the data's meta properties. */ private void loadProperties() { - final File dataDirectory = Settings.getFile(Settings.KEYS.DATA_DIRECTORY); - final File file = new File(dataDirectory, UPDATE_PROPERTIES_FILE); + File file = getPropertiesFile(); if (file.exists()) { InputStream is = null; try { @@ -139,8 +143,7 @@ public class DataStoreMetaInfo { if (updatedValue == null) { return; } - final File dataDirectory = Settings.getFile(Settings.KEYS.DATA_DIRECTORY); - final File cveProp = new File(dataDirectory, UPDATE_PROPERTIES_FILE); + final File cveProp = getPropertiesFile(); final Properties prop = new Properties(); if (cveProp.exists()) { FileInputStream in = null; @@ -214,4 +217,15 @@ public class DataStoreMetaInfo { public String getProperty(String key, String defaultValue) { return properties.getProperty(key, defaultValue); } + + /** + * Retrieves the properties file. + * + * @return the properties file + */ + public File getPropertiesFile() { + final File dataDirectory = Settings.getFile(Settings.KEYS.DATA_DIRECTORY); + final File file = new File(dataDirectory, UPDATE_PROPERTIES_FILE); + return file; + } } From eac470e0819b807e85d6e29f590e71aa9a676943 Mon Sep 17 00:00:00 2001 From: Jeremy Long Date: Sat, 31 Aug 2013 06:48:10 -0400 Subject: [PATCH 35/50] updates to batch update mode to allow batch updates without a modified URL Former-commit-id: 5e8ff7c0d9c880f2421f020f2891a6f7a794570a --- .../data/update/DatabaseUpdater.java | 138 +++++++++++------- 1 file changed, 87 insertions(+), 51 deletions(-) diff --git a/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/DatabaseUpdater.java b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/DatabaseUpdater.java index 36cea219f..edce25bf0 100644 --- a/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/DatabaseUpdater.java +++ b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/DatabaseUpdater.java @@ -41,7 +41,6 @@ import java.util.logging.Logger; import javax.xml.parsers.SAXParser; import javax.xml.parsers.SAXParserFactory; import org.owasp.dependencycheck.data.UpdateException; -import org.owasp.dependencycheck.data.cpe.BaseIndex; import org.owasp.dependencycheck.data.cpe.CpeIndexWriter; import org.owasp.dependencycheck.data.nvdcve.CveDB; import org.owasp.dependencycheck.dependency.VulnerableSoftware; @@ -50,8 +49,9 @@ import org.owasp.dependencycheck.utils.Downloader; import org.owasp.dependencycheck.utils.FileUtils; import org.owasp.dependencycheck.utils.Settings; import org.owasp.dependencycheck.data.nvdcve.DatabaseException; -import static org.owasp.dependencycheck.data.update.DataStoreMetaInfo.MODIFIED; import org.owasp.dependencycheck.utils.InvalidSettingException; +import static org.owasp.dependencycheck.data.update.DataStoreMetaInfo.BATCH; +import static org.owasp.dependencycheck.data.update.DataStoreMetaInfo.MODIFIED; /** * @@ -199,10 +199,13 @@ public class DatabaseUpdater implements CachedWebDataSource { } } } - if (maxUpdates >= 1) { + if (maxUpdates >= 1) { //ensure the modified file date gets written properties.save(update.get(MODIFIED)); cveDB.cleanupDatabase(); } + if (update.get(BATCH) != null) { + properties.save(update.get(BATCH)); + } } catch (MalformedURLException ex) { throw new UpdateException(ex); } catch (DownloadFailedException ex) { @@ -249,13 +252,18 @@ public class DatabaseUpdater implements CachedWebDataSource { * @throws IOException thrown if the directory cannot be deleted */ protected void deleteExistingData() throws IOException { - Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.INFO, "The database version is old. Rebuilding the database."); - final File data = Settings.getFile(Settings.KEYS.DATA_DIRECTORY); - FileUtils.delete(data); -// final File cveDir = CveDB.getDataDirectory(); -// FileUtils.delete(cveDir); -// final File cpeDir = BaseIndex.getDataDirectory(); -// FileUtils.delete(cpeDir); + File data = Settings.getFile(Settings.KEYS.CVE_DATA_DIRECTORY); + if (data.exists()) { + FileUtils.delete(data); + } + data = Settings.getFile(Settings.KEYS.CPE_DATA_DIRECTORY); + if (data.exists()) { + FileUtils.delete(data); + } + data = properties.getPropertiesFile(); + if (data.exists()) { + FileUtils.delete(data); + } } private void performBatchUpdate() throws UpdateException { @@ -372,26 +380,25 @@ public class DatabaseUpdater implements CachedWebDataSource { } if (currentlyPublished == null) { - //TODO change messages once we have a new batch mode - throw new DownloadFailedException("Unable to retrieve valid timestamp from NVD CVE data feeds"); + throw new DownloadFailedException("Unable to retrieve the timestamps of the currently published NVD CVE data"); } - final File cpeDataDirectory; - try { - cpeDataDirectory = CveDB.getDataDirectory(); - } catch (IOException ex) { - String msg; - try { - msg = String.format("Unable to create the CVE Data Directory '%s'", - Settings.getFile(Settings.KEYS.CVE_DATA_DIRECTORY).getCanonicalPath()); - } catch (IOException ex1) { - msg = String.format("Unable to create the CVE Data Directory, this is likely a configuration issue: '%s%s%s'", - Settings.getString(Settings.KEYS.DATA_DIRECTORY, ""), - File.separator, - Settings.getString(Settings.KEYS.CVE_DATA_DIRECTORY, "")); - } - throw new UpdateException(msg, ex); - } +// final File cpeDataDirectory; +// try { +// cpeDataDirectory = CveDB.getDataDirectory(); +// } catch (IOException ex) { +// String msg; +// try { +// msg = String.format("Unable to create the CVE Data Directory '%s'", +// Settings.getFile(Settings.KEYS.CVE_DATA_DIRECTORY).getCanonicalPath()); +// } catch (IOException ex1) { +// msg = String.format("Unable to create the CVE Data Directory, this is likely a configuration issue: '%s%s%s'", +// Settings.getString(Settings.KEYS.DATA_DIRECTORY, ""), +// File.separator, +// Settings.getString(Settings.KEYS.CVE_DATA_DIRECTORY, "")); +// } +// throw new UpdateException(msg, ex); +// } if (!properties.isEmpty()) { try { @@ -411,8 +418,24 @@ public class DatabaseUpdater implements CachedWebDataSource { deleteAndRecreate = true; } } + + NvdCveInfo batchInfo = currentlyPublished.get(BATCH); + if (properties.isBatchUpdateMode() && batchInfo != null) { + final long lastUpdated = Long.parseLong(properties.getProperty(DataStoreMetaInfo.BATCH, "0")); + if (lastUpdated != batchInfo.getTimestamp()) { + deleteAndRecreate = true; + } + } + if (deleteAndRecreate) { setDoBatchUpdate(properties.isBatchUpdateMode()); + try { + deleteExistingData(); + } catch (IOException ex) { + final String msg = "Unable to delete existing data"; + Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.WARNING, msg); + Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.FINE, null, ex); + } return currentlyPublished; } @@ -492,35 +515,48 @@ public class DatabaseUpdater implements CachedWebDataSource { * timestamps * @throws InvalidSettingException thrown if the settings are invalid */ - protected Map retrieveCurrentTimestampsFromWeb() + private Map retrieveCurrentTimestampsFromWeb() throws MalformedURLException, DownloadFailedException, InvalidDataException, InvalidSettingException { final Map map = new TreeMap(); String retrieveUrl = Settings.getString(Settings.KEYS.CVE_MODIFIED_20_URL); + if (retrieveUrl == null && properties.isBatchUpdateMode()) { + NvdCveInfo item = new NvdCveInfo(); + retrieveUrl = Settings.getString(Settings.KEYS.BATCH_UPDATE_URL); + if (retrieveUrl == null) { + final String msg = "Invalid configuration - neither the modified or batch update URLs are specified in the configuration."; + Logger.getLogger(DataStoreMetaInfo.class.getName()).log(Level.SEVERE, msg); + throw new InvalidSettingException(msg); + } + item.setTimestamp(Downloader.getLastModified(new URL(retrieveUrl))); + item.setId(BATCH); + item.setNeedsUpdate(false); + map.put(BATCH, item); + } else { + NvdCveInfo item = new NvdCveInfo(); + item.setNeedsUpdate(false); //the others default to true, to make life easier later this should default to false. + item.setId(MODIFIED); + item.setUrl(retrieveUrl); + item.setOldSchemaVersionUrl(Settings.getString(Settings.KEYS.CVE_MODIFIED_12_URL)); - NvdCveInfo item = new NvdCveInfo(); - item.setNeedsUpdate(false); //the others default to true, to make life easier later this should default to false. - item.setId(MODIFIED); - item.setUrl(retrieveUrl); - item.setOldSchemaVersionUrl(Settings.getString(Settings.KEYS.CVE_MODIFIED_12_URL)); + item.setTimestamp(Downloader.getLastModified(new URL(retrieveUrl))); + map.put(MODIFIED, item); - item.setTimestamp(Downloader.getLastModified(new URL(retrieveUrl))); - map.put(MODIFIED, item); - - //only add these urls if we are not in batch mode - if (!properties.isBatchUpdateMode()) { - final int start = Settings.getInt(Settings.KEYS.CVE_START_YEAR); - final int end = Calendar.getInstance().get(Calendar.YEAR); - final String baseUrl20 = Settings.getString(Settings.KEYS.CVE_SCHEMA_2_0); - final String baseUrl12 = Settings.getString(Settings.KEYS.CVE_SCHEMA_1_2); - for (int i = start; i <= end; i++) { - retrieveUrl = String.format(baseUrl20, i); - item = new NvdCveInfo(); - item.setId(Integer.toString(i)); - item.setUrl(retrieveUrl); - item.setOldSchemaVersionUrl(String.format(baseUrl12, i)); - item.setTimestamp(Downloader.getLastModified(new URL(retrieveUrl))); - map.put(item.getId(), item); + //only add these urls if we are not in batch mode + if (!properties.isBatchUpdateMode()) { + final int start = Settings.getInt(Settings.KEYS.CVE_START_YEAR); + final int end = Calendar.getInstance().get(Calendar.YEAR); + final String baseUrl20 = Settings.getString(Settings.KEYS.CVE_SCHEMA_2_0); + final String baseUrl12 = Settings.getString(Settings.KEYS.CVE_SCHEMA_1_2); + for (int i = start; i <= end; i++) { + retrieveUrl = String.format(baseUrl20, i); + item = new NvdCveInfo(); + item.setId(Integer.toString(i)); + item.setUrl(retrieveUrl); + item.setOldSchemaVersionUrl(String.format(baseUrl12, i)); + item.setTimestamp(Downloader.getLastModified(new URL(retrieveUrl))); + map.put(item.getId(), item); + } } } return map; From 6b416b84949396f621a7c7b0116aeb55ff843eab Mon Sep 17 00:00:00 2001 From: Jeremy Long Date: Sat, 31 Aug 2013 06:48:59 -0400 Subject: [PATCH 36/50] added a test case to test batch update without a modified url Former-commit-id: 5d10d888c927f86a76f983011cab47d4c2725333 --- .../data/update/DatabaseUpdaterTest.java | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/dependency-check-core/src/test/java/org/owasp/dependencycheck/data/update/DatabaseUpdaterTest.java b/dependency-check-core/src/test/java/org/owasp/dependencycheck/data/update/DatabaseUpdaterTest.java index a428813b6..c44730a49 100644 --- a/dependency-check-core/src/test/java/org/owasp/dependencycheck/data/update/DatabaseUpdaterTest.java +++ b/dependency-check-core/src/test/java/org/owasp/dependencycheck/data/update/DatabaseUpdaterTest.java @@ -82,4 +82,26 @@ public class DatabaseUpdaterTest { instance.deleteExistingData(); instance.update(); } + + /** + * Test of update method (when in batch mode), of class DatabaseUpdater. + * + * @throws Exception + */ + @Test + public void testBatchUpdateWithoutModified() throws Exception { + //setup - consider moving this to its own test case file so it has a different setup/teardown. + final String tmp12 = Settings.getString(Settings.KEYS.CVE_MODIFIED_12_URL); + final String tmp20 = Settings.getString(Settings.KEYS.CVE_MODIFIED_20_URL); + Settings.removeProperty(Settings.KEYS.CVE_MODIFIED_12_URL); + Settings.removeProperty(Settings.KEYS.CVE_MODIFIED_20_URL); + + DatabaseUpdater instance = new DatabaseUpdater(); + instance.deleteExistingData(); + instance.update(); + + //restore defaults + Settings.setString(Settings.KEYS.CVE_MODIFIED_12_URL, tmp12); + Settings.setString(Settings.KEYS.CVE_MODIFIED_20_URL, tmp20); + } } From 2ec5ec78a9d56e3d319fdc7cf263ab6ff5760277 Mon Sep 17 00:00:00 2001 From: Jeremy Long Date: Sat, 31 Aug 2013 07:04:17 -0400 Subject: [PATCH 37/50] removed unused imports Former-commit-id: fee44850bba2b1af19989bc14aef99a38e725185 --- .../org/owasp/dependencycheck/analyzer/CPEAnalyzerTest.java | 4 ---- 1 file changed, 4 deletions(-) diff --git a/dependency-check-core/src/test/java/org/owasp/dependencycheck/analyzer/CPEAnalyzerTest.java b/dependency-check-core/src/test/java/org/owasp/dependencycheck/analyzer/CPEAnalyzerTest.java index 1a7b0a887..4210b98be 100644 --- a/dependency-check-core/src/test/java/org/owasp/dependencycheck/analyzer/CPEAnalyzerTest.java +++ b/dependency-check-core/src/test/java/org/owasp/dependencycheck/analyzer/CPEAnalyzerTest.java @@ -30,14 +30,10 @@ import org.apache.lucene.queryparser.classic.ParseException; import org.junit.After; import org.junit.AfterClass; import org.owasp.dependencycheck.dependency.Dependency; -import org.owasp.dependencycheck.analyzer.JarAnalyzer; import org.junit.Assert; import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; -import org.owasp.dependencycheck.analyzer.FalsePositiveAnalyzer; -import org.owasp.dependencycheck.analyzer.FileNameAnalyzer; -import org.owasp.dependencycheck.analyzer.HintAnalyzer; import org.owasp.dependencycheck.data.cpe.BaseIndexTestCase; import org.owasp.dependencycheck.data.cpe.IndexEntry; import org.owasp.dependencycheck.dependency.Identifier; From 3b4a65deaa43c587e9e35ad1bccb8769bdd96675 Mon Sep 17 00:00:00 2001 From: Jeremy Long Date: Sat, 31 Aug 2013 07:35:20 -0400 Subject: [PATCH 38/50] updated to work with the new zip file and folder structure Former-commit-id: 4ab18e5d93a71124aaf57d1d0b6945c7516e4463 --- .../data/nvdcve/BaseDBTestCase.java | 22 ++++++++----------- 1 file changed, 9 insertions(+), 13 deletions(-) diff --git a/dependency-check-core/src/test/java/org/owasp/dependencycheck/data/nvdcve/BaseDBTestCase.java b/dependency-check-core/src/test/java/org/owasp/dependencycheck/data/nvdcve/BaseDBTestCase.java index 9a141348a..21046db20 100644 --- a/dependency-check-core/src/test/java/org/owasp/dependencycheck/data/nvdcve/BaseDBTestCase.java +++ b/dependency-check-core/src/test/java/org/owasp/dependencycheck/data/nvdcve/BaseDBTestCase.java @@ -30,6 +30,7 @@ import java.util.logging.Logger; import java.util.zip.ZipEntry; import java.util.zip.ZipInputStream; import junit.framework.TestCase; +import org.owasp.dependencycheck.data.update.DataStoreMetaInfo; import org.owasp.dependencycheck.utils.Settings; /** @@ -46,18 +47,11 @@ public abstract class BaseDBTestCase extends TestCase { ensureDBExists(); } - protected static File getDataDirectory() throws IOException { - final String fileName = Settings.getString(Settings.KEYS.CVE_DATA_DIRECTORY); - final String dataDirectory = Settings.getString(Settings.KEYS.DATA_DIRECTORY); - return new File(dataDirectory, fileName); - } - public static void ensureDBExists() throws Exception { - String indexPath = getDataDirectory().getCanonicalPath(); - java.io.File f = new File(indexPath); - if (!f.exists() || (f.isDirectory() && f.listFiles().length == 0)) { - f = f.getParentFile(); - f.mkdirs(); + + java.io.File dataPath = Settings.getFile(Settings.KEYS.DATA_DIRECTORY); + if (!dataPath.exists() || (dataPath.isDirectory() && dataPath.listFiles().length == 0)) { + dataPath.mkdirs(); FileInputStream fis = null; ZipInputStream zin = null; try { @@ -67,12 +61,14 @@ public abstract class BaseDBTestCase extends TestCase { ZipEntry entry; while ((entry = zin.getNextEntry()) != null) { if (entry.isDirectory()) { + final File d = new File(dataPath, entry.getName()); + d.mkdir(); continue; } FileOutputStream fos = null; BufferedOutputStream dest = null; try { - File o = new File(indexPath, entry.getName()); + File o = new File(dataPath, entry.getName()); o.createNewFile(); fos = new FileOutputStream(o, false); dest = new BufferedOutputStream(fos, BUFFER_SIZE); @@ -82,7 +78,7 @@ public abstract class BaseDBTestCase extends TestCase { dest.write(data, 0, count); } } catch (Exception ex) { - Logger.getLogger(BaseDBTestCase.class.getName()).log(Level.FINEST, null, ex); + Logger.getLogger(BaseDBTestCase.class.getName()).log(Level.SEVERE, null, ex); } finally { try { if (dest != null) { From 01d6e1f14d5c625e675424bfd70c8cdc2863c124 Mon Sep 17 00:00:00 2001 From: Jeremy Long Date: Sun, 1 Sep 2013 07:27:13 -0400 Subject: [PATCH 39/50] updated tests to utilize temp directory Former-commit-id: 984f69e2528d37586a2d7d981d17671e17e7ca0d --- dependency-check-core/pom.xml | 4 + .../data/update/DataStoreMetaInfo.java | 2 +- .../data/update/DatabaseUpdater.java | 2 +- ...rTest.java => DatabaseUpdater_1_Test.java} | 52 +++++------ .../data/update/DatabaseUpdater_2_Test.java | 87 +++++++++++++++++++ 5 files changed, 114 insertions(+), 33 deletions(-) rename dependency-check-core/src/test/java/org/owasp/dependencycheck/data/update/{DatabaseUpdaterTest.java => DatabaseUpdater_1_Test.java} (68%) create mode 100644 dependency-check-core/src/test/java/org/owasp/dependencycheck/data/update/DatabaseUpdater_2_Test.java diff --git a/dependency-check-core/pom.xml b/dependency-check-core/pom.xml index c6cf0d90c..d42f4c266 100644 --- a/dependency-check-core/pom.xml +++ b/dependency-check-core/pom.xml @@ -177,6 +177,10 @@ along with Dependency-Check. If not, see . data.directory ${project.build.directory}/data + + temp.directory + ${project.build.directory}/temp + **/*IntegrationTest.java diff --git a/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/DataStoreMetaInfo.java b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/DataStoreMetaInfo.java index 38fc77bfd..3d25c8e22 100644 --- a/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/DataStoreMetaInfo.java +++ b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/DataStoreMetaInfo.java @@ -223,7 +223,7 @@ public class DataStoreMetaInfo { * * @return the properties file */ - public File getPropertiesFile() { + public static File getPropertiesFile() { final File dataDirectory = Settings.getFile(Settings.KEYS.DATA_DIRECTORY); final File file = new File(dataDirectory, UPDATE_PROPERTIES_FILE); return file; diff --git a/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/DatabaseUpdater.java b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/DatabaseUpdater.java index edce25bf0..9af48b57b 100644 --- a/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/DatabaseUpdater.java +++ b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/DatabaseUpdater.java @@ -260,7 +260,7 @@ public class DatabaseUpdater implements CachedWebDataSource { if (data.exists()) { FileUtils.delete(data); } - data = properties.getPropertiesFile(); + data = DataStoreMetaInfo.getPropertiesFile(); if (data.exists()) { FileUtils.delete(data); } diff --git a/dependency-check-core/src/test/java/org/owasp/dependencycheck/data/update/DatabaseUpdaterTest.java b/dependency-check-core/src/test/java/org/owasp/dependencycheck/data/update/DatabaseUpdater_1_Test.java similarity index 68% rename from dependency-check-core/src/test/java/org/owasp/dependencycheck/data/update/DatabaseUpdaterTest.java rename to dependency-check-core/src/test/java/org/owasp/dependencycheck/data/update/DatabaseUpdater_1_Test.java index c44730a49..b39b35954 100644 --- a/dependency-check-core/src/test/java/org/owasp/dependencycheck/data/update/DatabaseUpdaterTest.java +++ b/dependency-check-core/src/test/java/org/owasp/dependencycheck/data/update/DatabaseUpdater_1_Test.java @@ -20,6 +20,7 @@ package org.owasp.dependencycheck.data.update; import org.owasp.dependencycheck.data.update.DatabaseUpdater; import java.io.File; +import org.apache.commons.io.FileUtils; import org.junit.After; import org.junit.AfterClass; import org.junit.Before; @@ -31,9 +32,9 @@ import org.owasp.dependencycheck.utils.Settings; * * @author Jeremy Long (jeremy.long@owasp.org) */ -public class DatabaseUpdaterTest { +public class DatabaseUpdater_1_Test { - public DatabaseUpdaterTest() { + public DatabaseUpdater_1_Test() { } @BeforeClass @@ -51,17 +52,28 @@ public class DatabaseUpdaterTest { old12 = Settings.getString(Settings.KEYS.CVE_MODIFIED_12_URL); old20 = Settings.getString(Settings.KEYS.CVE_MODIFIED_20_URL); - File file = new File(this.getClass().getClassLoader().getResource("nvdcve-2012.xml").toURI()); - String path = "file:///" + file.getCanonicalPath(); + File tmp = Settings.getFile(Settings.KEYS.TEMP_DIRECTORY); + if (!tmp.exists()) { + tmp.mkdirs(); + } + + File dest = new File(tmp, "data.zip"); + File file = new File(this.getClass().getClassLoader().getResource("data.zip").toURI()); + FileUtils.copyFile(file, dest); + String path = "file:///" + dest.getCanonicalPath(); + Settings.setString(Settings.KEYS.BATCH_UPDATE_URL, path); + + dest = new File(tmp, "nvdcve-2012.xml"); + file = new File(this.getClass().getClassLoader().getResource("nvdcve-2012.xml").toURI()); + FileUtils.copyFile(file, dest); + path = "file:///" + dest.getCanonicalPath(); Settings.setString(Settings.KEYS.CVE_MODIFIED_12_URL, path); + dest = new File(tmp, "nvdcve-2.0-2012.xml"); file = new File(this.getClass().getClassLoader().getResource("nvdcve-2.0-2012.xml").toURI()); - path = "file:///" + file.getCanonicalPath(); + FileUtils.copyFile(file, dest); + path = "file:///" + dest.getCanonicalPath(); Settings.setString(Settings.KEYS.CVE_MODIFIED_20_URL, path); - - file = new File(this.getClass().getClassLoader().getResource("data.zip").toURI()); - path = "file:///" + file.getCanonicalPath(); - Settings.setString(Settings.KEYS.BATCH_UPDATE_URL, path); } @After @@ -82,26 +94,4 @@ public class DatabaseUpdaterTest { instance.deleteExistingData(); instance.update(); } - - /** - * Test of update method (when in batch mode), of class DatabaseUpdater. - * - * @throws Exception - */ - @Test - public void testBatchUpdateWithoutModified() throws Exception { - //setup - consider moving this to its own test case file so it has a different setup/teardown. - final String tmp12 = Settings.getString(Settings.KEYS.CVE_MODIFIED_12_URL); - final String tmp20 = Settings.getString(Settings.KEYS.CVE_MODIFIED_20_URL); - Settings.removeProperty(Settings.KEYS.CVE_MODIFIED_12_URL); - Settings.removeProperty(Settings.KEYS.CVE_MODIFIED_20_URL); - - DatabaseUpdater instance = new DatabaseUpdater(); - instance.deleteExistingData(); - instance.update(); - - //restore defaults - Settings.setString(Settings.KEYS.CVE_MODIFIED_12_URL, tmp12); - Settings.setString(Settings.KEYS.CVE_MODIFIED_20_URL, tmp20); - } } diff --git a/dependency-check-core/src/test/java/org/owasp/dependencycheck/data/update/DatabaseUpdater_2_Test.java b/dependency-check-core/src/test/java/org/owasp/dependencycheck/data/update/DatabaseUpdater_2_Test.java new file mode 100644 index 000000000..4e40c0506 --- /dev/null +++ b/dependency-check-core/src/test/java/org/owasp/dependencycheck/data/update/DatabaseUpdater_2_Test.java @@ -0,0 +1,87 @@ +/* + * This file is part of dependency-check-core. + * + * Dependency-check-core is free software: you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation, either version 3 of the License, or (at your option) any + * later version. + * + * Dependency-check-core is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more + * details. + * + * You should have received a copy of the GNU General Public License along with + * dependency-check-core. If not, see http://www.gnu.org/licenses/. + * + * Copyright (c) 2012 Jeremy Long. All Rights Reserved. + */ +package org.owasp.dependencycheck.data.update; + +import org.owasp.dependencycheck.data.update.DatabaseUpdater; +import java.io.File; +import org.apache.commons.io.FileUtils; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; +import org.owasp.dependencycheck.utils.Settings; + +/** + * + * @author Jeremy Long (jeremy.long@owasp.org) + */ +public class DatabaseUpdater_2_Test { + + public DatabaseUpdater_2_Test() { + } + + @BeforeClass + public static void setUpClass() throws Exception { + } + + @AfterClass + public static void tearDownClass() throws Exception { + } + private String old12; + private String old20; + + @Before + public void setUp() throws Exception { + old12 = Settings.getString(Settings.KEYS.CVE_MODIFIED_12_URL); + old20 = Settings.getString(Settings.KEYS.CVE_MODIFIED_20_URL); + Settings.removeProperty(Settings.KEYS.CVE_MODIFIED_12_URL); + Settings.removeProperty(Settings.KEYS.CVE_MODIFIED_20_URL); + + File tmp = Settings.getFile(Settings.KEYS.TEMP_DIRECTORY); + if (!tmp.exists()) { + tmp.mkdirs(); + } + + File dest = new File(tmp, "data.zip"); + File file = new File(this.getClass().getClassLoader().getResource("data.zip").toURI()); + FileUtils.copyFile(file, dest); + String path = "file:///" + dest.getCanonicalPath(); + Settings.setString(Settings.KEYS.BATCH_UPDATE_URL, path); + } + + @After + public void tearDown() { + Settings.setString(Settings.KEYS.CVE_MODIFIED_12_URL, old12); + Settings.setString(Settings.KEYS.CVE_MODIFIED_20_URL, old20); + Settings.removeProperty(Settings.KEYS.BATCH_UPDATE_URL); + } + + /** + * Test of update method (when in batch mode), of class DatabaseUpdater. + * + * @throws Exception + */ + @Test + public void testBatchUpdateWithoutModified() throws Exception { + DatabaseUpdater instance = new DatabaseUpdater(); + instance.deleteExistingData(); + instance.update(); + } +} From fc30aeea61cc68abade1a21f10b796f4f5f85dba Mon Sep 17 00:00:00 2001 From: Jeremy Long Date: Sun, 1 Sep 2013 07:46:43 -0400 Subject: [PATCH 40/50] updated Settigns to get the temp directory Former-commit-id: d9a1e89ce829ab06e5aa1cd454184966d1e4b115 --- .../owasp/dependencycheck/analyzer/ArchiveAnalyzer.java | 6 ++++-- .../java/org/owasp/dependencycheck/utils/Settings.java | 9 +++++++++ .../data/update/DatabaseUpdater_1_Test.java | 2 +- .../data/update/DatabaseUpdater_2_Test.java | 2 +- 4 files changed, 15 insertions(+), 4 deletions(-) diff --git a/dependency-check-core/src/main/java/org/owasp/dependencycheck/analyzer/ArchiveAnalyzer.java b/dependency-check-core/src/main/java/org/owasp/dependencycheck/analyzer/ArchiveAnalyzer.java index f9bb9e43b..d7e862653 100644 --- a/dependency-check-core/src/main/java/org/owasp/dependencycheck/analyzer/ArchiveAnalyzer.java +++ b/dependency-check-core/src/main/java/org/owasp/dependencycheck/analyzer/ArchiveAnalyzer.java @@ -127,8 +127,10 @@ public class ArchiveAnalyzer extends AbstractAnalyzer implements Analyzer { */ @Override public void initialize() throws Exception { - final String tmpDir = Settings.getString(Settings.KEYS.TEMP_DIRECTORY, System.getProperty("java.io.tmpdir")); - final File baseDir = new File(tmpDir); + final File baseDir = Settings.getTempDirectory(); + if (!baseDir.exists()) { + baseDir.mkdirs(); + } tempFileLocation = File.createTempFile("check", "tmp", baseDir); if (!tempFileLocation.delete()) { throw new AnalysisException("Unable to delete temporary file '" + tempFileLocation.getAbsolutePath() + "'."); diff --git a/dependency-check-core/src/main/java/org/owasp/dependencycheck/utils/Settings.java b/dependency-check-core/src/main/java/org/owasp/dependencycheck/utils/Settings.java index 1f1024745..6c2b8ac9c 100644 --- a/dependency-check-core/src/main/java/org/owasp/dependencycheck/utils/Settings.java +++ b/dependency-check-core/src/main/java/org/owasp/dependencycheck/utils/Settings.java @@ -296,6 +296,15 @@ public final class Settings { return str; } + /** + * Returns the temporary directory. + * + * @return the temporary directory + */ + public static File getTempDirectory() { + return new File(Settings.getString(Settings.KEYS.TEMP_DIRECTORY, System.getProperty("java.io.tmpdir"))); + } + /** * Returns a value from the properties file. If the value was specified as a * system property or passed in via the -Dprop=value argument - this method diff --git a/dependency-check-core/src/test/java/org/owasp/dependencycheck/data/update/DatabaseUpdater_1_Test.java b/dependency-check-core/src/test/java/org/owasp/dependencycheck/data/update/DatabaseUpdater_1_Test.java index b39b35954..44fe42c5b 100644 --- a/dependency-check-core/src/test/java/org/owasp/dependencycheck/data/update/DatabaseUpdater_1_Test.java +++ b/dependency-check-core/src/test/java/org/owasp/dependencycheck/data/update/DatabaseUpdater_1_Test.java @@ -52,7 +52,7 @@ public class DatabaseUpdater_1_Test { old12 = Settings.getString(Settings.KEYS.CVE_MODIFIED_12_URL); old20 = Settings.getString(Settings.KEYS.CVE_MODIFIED_20_URL); - File tmp = Settings.getFile(Settings.KEYS.TEMP_DIRECTORY); + File tmp = Settings.getTempDirectory(); if (!tmp.exists()) { tmp.mkdirs(); } diff --git a/dependency-check-core/src/test/java/org/owasp/dependencycheck/data/update/DatabaseUpdater_2_Test.java b/dependency-check-core/src/test/java/org/owasp/dependencycheck/data/update/DatabaseUpdater_2_Test.java index 4e40c0506..65c1f7046 100644 --- a/dependency-check-core/src/test/java/org/owasp/dependencycheck/data/update/DatabaseUpdater_2_Test.java +++ b/dependency-check-core/src/test/java/org/owasp/dependencycheck/data/update/DatabaseUpdater_2_Test.java @@ -54,7 +54,7 @@ public class DatabaseUpdater_2_Test { Settings.removeProperty(Settings.KEYS.CVE_MODIFIED_12_URL); Settings.removeProperty(Settings.KEYS.CVE_MODIFIED_20_URL); - File tmp = Settings.getFile(Settings.KEYS.TEMP_DIRECTORY); + File tmp = Settings.getTempDirectory(); if (!tmp.exists()) { tmp.mkdirs(); } From b8433c4ea768bea1e542b5c2025520f8890aeaa9 Mon Sep 17 00:00:00 2001 From: Jeremy Long Date: Sun, 1 Sep 2013 12:21:56 -0400 Subject: [PATCH 41/50] moved test data.zip to parent so it can be used by other projects Former-commit-id: 5d0e82fcaf170af7086de306abc9eb2ed488bc79 --- dependency-check-ant/pom.xml | 31 +++++++++++++++++++ .../taskdefs/DependencyCheckTaskTest.java | 5 ++- dependency-check-core/pom.xml | 16 ++++++++++ 3 files changed, 51 insertions(+), 1 deletion(-) diff --git a/dependency-check-ant/pom.xml b/dependency-check-ant/pom.xml index a266ff96b..c499ccaa8 100644 --- a/dependency-check-ant/pom.xml +++ b/dependency-check-ant/pom.xml @@ -76,6 +76,25 @@ Copyright (c) 2013 - Jeremy Long. All Rights Reserved. + + copy-test-data.zip + validate + + copy-resources + + + ${project.build.directory}/test-classes + + + ${basedir}/../src/test/resources + false + + data.zip + + + + + copy-test-resources-1 validate @@ -238,6 +257,10 @@ Copyright (c) 2013 - Jeremy Long. All Rights Reserved. ${project.build.directory}/cobertura/cobertura.ser target + + data.directory + ${project.build.directory}/dependency-check-data + @@ -389,6 +412,14 @@ Copyright (c) 2013 - Jeremy Long. All Rights Reserved. dependency-check-core ${project.parent.version} + + org.owasp + dependency-check-core + ${project.parent.version} + test-jar + test + + org.apache.ant ant diff --git a/dependency-check-ant/src/test/java/org/owasp/dependencycheck/taskdefs/DependencyCheckTaskTest.java b/dependency-check-ant/src/test/java/org/owasp/dependencycheck/taskdefs/DependencyCheckTaskTest.java index 34c0c5591..1fa97dfd4 100644 --- a/dependency-check-ant/src/test/java/org/owasp/dependencycheck/taskdefs/DependencyCheckTaskTest.java +++ b/dependency-check-ant/src/test/java/org/owasp/dependencycheck/taskdefs/DependencyCheckTaskTest.java @@ -26,6 +26,7 @@ import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; import org.apache.tools.ant.BuildFileTest; +import org.owasp.dependencycheck.data.nvdcve.BaseDBTestCase; /** * @@ -46,7 +47,9 @@ public class DependencyCheckTaskTest extends BuildFileTest { @Before @Override - public void setUp() { + public void setUp() throws Exception { + BaseDBTestCase.ensureDBExists(); + final String buildFile = this.getClass().getClassLoader().getResource("build.xml").getPath(); configureProject(buildFile); } diff --git a/dependency-check-core/pom.xml b/dependency-check-core/pom.xml index d42f4c266..89823c2b0 100644 --- a/dependency-check-core/pom.xml +++ b/dependency-check-core/pom.xml @@ -101,6 +101,22 @@ along with Dependency-Check. If not, see . org.apache.maven.plugins maven-jar-plugin 2.4 + + + jar + package + + jar + + + + test-jar + package + + test-jar + + + From 94e1a4f793a39ca5a655f6da85d5f611d0e77b89 Mon Sep 17 00:00:00 2001 From: Jeremy Long Date: Sun, 1 Sep 2013 15:11:49 -0400 Subject: [PATCH 43/50] minor checkstyle updates Former-commit-id: dc9c50f0627222169eceb94a44b8b4b4fa1af267 --- .../dependencycheck/analyzer/NvdCveAnalyzer.java | 3 --- .../concurrency/DirectoryLockException.java | 3 +++ .../concurrency/DirectorySpinLock.java | 2 +- .../concurrency/InvalidDirectoryException.java | 3 +++ .../concurrency/package-info.java | 11 +++++++++++ .../dependencycheck/data/cpe/BaseIndex.java | 13 +++++++++++-- .../dependencycheck/data/cpe/CpeIndexReader.java | 2 +- .../dependencycheck/data/cpe/CpeIndexWriter.java | 2 +- .../data/nvdcve/NvdCve20Handler.java | 2 -- .../data/update/DataStoreMetaInfo.java | 2 +- .../data/update/DatabaseUpdater.java | 16 ++++++++++++---- .../dependencycheck/data/update/NvdCveInfo.java | 2 ++ .../owasp/dependencycheck/utils/FileUtils.java | 9 ++++++--- 13 files changed, 52 insertions(+), 18 deletions(-) create mode 100644 dependency-check-core/src/main/java/org/owasp/dependencycheck/concurrency/package-info.java diff --git a/dependency-check-core/src/main/java/org/owasp/dependencycheck/analyzer/NvdCveAnalyzer.java b/dependency-check-core/src/main/java/org/owasp/dependencycheck/analyzer/NvdCveAnalyzer.java index d7e1f7c74..7c6be8a04 100644 --- a/dependency-check-core/src/main/java/org/owasp/dependencycheck/analyzer/NvdCveAnalyzer.java +++ b/dependency-check-core/src/main/java/org/owasp/dependencycheck/analyzer/NvdCveAnalyzer.java @@ -23,12 +23,9 @@ import java.sql.SQLException; import java.util.List; import java.util.Set; import org.owasp.dependencycheck.Engine; -import org.owasp.dependencycheck.analyzer.AnalysisException; -import org.owasp.dependencycheck.analyzer.AnalysisPhase; import org.owasp.dependencycheck.dependency.Dependency; import org.owasp.dependencycheck.dependency.Vulnerability; import org.owasp.dependencycheck.dependency.Identifier; -import org.owasp.dependencycheck.analyzer.Analyzer; import org.owasp.dependencycheck.data.nvdcve.CveDB; import org.owasp.dependencycheck.data.nvdcve.DatabaseException; diff --git a/dependency-check-core/src/main/java/org/owasp/dependencycheck/concurrency/DirectoryLockException.java b/dependency-check-core/src/main/java/org/owasp/dependencycheck/concurrency/DirectoryLockException.java index 5985152a1..3fbc7ae95 100644 --- a/dependency-check-core/src/main/java/org/owasp/dependencycheck/concurrency/DirectoryLockException.java +++ b/dependency-check-core/src/main/java/org/owasp/dependencycheck/concurrency/DirectoryLockException.java @@ -25,6 +25,9 @@ package org.owasp.dependencycheck.concurrency; */ public class DirectoryLockException extends Exception { + /** + * Default serial version UID. + */ private static final long serialVersionUID = 1L; /** diff --git a/dependency-check-core/src/main/java/org/owasp/dependencycheck/concurrency/DirectorySpinLock.java b/dependency-check-core/src/main/java/org/owasp/dependencycheck/concurrency/DirectorySpinLock.java index 001f344bf..2261846ed 100644 --- a/dependency-check-core/src/main/java/org/owasp/dependencycheck/concurrency/DirectorySpinLock.java +++ b/dependency-check-core/src/main/java/org/owasp/dependencycheck/concurrency/DirectorySpinLock.java @@ -219,7 +219,7 @@ public class DirectorySpinLock implements Closeable /*, AutoCloseable*/ { /** * Releases any locks and closes the underlying channel. * - * @throws IOException + * @throws IOException if an IO Exception occurs */ @Override public void close() throws IOException { diff --git a/dependency-check-core/src/main/java/org/owasp/dependencycheck/concurrency/InvalidDirectoryException.java b/dependency-check-core/src/main/java/org/owasp/dependencycheck/concurrency/InvalidDirectoryException.java index 3faa54ffa..08b29acff 100644 --- a/dependency-check-core/src/main/java/org/owasp/dependencycheck/concurrency/InvalidDirectoryException.java +++ b/dependency-check-core/src/main/java/org/owasp/dependencycheck/concurrency/InvalidDirectoryException.java @@ -25,6 +25,9 @@ package org.owasp.dependencycheck.concurrency; */ public class InvalidDirectoryException extends Exception { + /** + * Default serial version UID. + */ private static final long serialVersionUID = 1L; /** diff --git a/dependency-check-core/src/main/java/org/owasp/dependencycheck/concurrency/package-info.java b/dependency-check-core/src/main/java/org/owasp/dependencycheck/concurrency/package-info.java new file mode 100644 index 000000000..ab7fba4fd --- /dev/null +++ b/dependency-check-core/src/main/java/org/owasp/dependencycheck/concurrency/package-info.java @@ -0,0 +1,11 @@ +/** + * + * + * org.owasp.dependencycheck.concurrency + * + * + * Contains classes used to create shared and exclusive locks on directories. + * + * + */ +package org.owasp.dependencycheck.concurrency; diff --git a/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/cpe/BaseIndex.java b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/cpe/BaseIndex.java index 1f6e84d91..f895a99e8 100644 --- a/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/cpe/BaseIndex.java +++ b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/cpe/BaseIndex.java @@ -36,11 +36,20 @@ public abstract class BaseIndex { /** * The Lucene directory containing the index. */ - protected Directory directory; + private Directory directory; /** * Indicates whether or not the Lucene Index is open. */ - protected boolean indexOpen = false; + private boolean indexOpen = false; + + /** + * Gets the directory. + * + * @return the directory + */ + public Directory getDirectory() { + return directory; + } /** * Opens the CPE Index. diff --git a/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/cpe/CpeIndexReader.java b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/cpe/CpeIndexReader.java index dfee343e5..68f9344f4 100644 --- a/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/cpe/CpeIndexReader.java +++ b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/cpe/CpeIndexReader.java @@ -79,7 +79,7 @@ public class CpeIndexReader extends BaseIndex { public void open() throws IOException { //TODO add spinlock (shared) super.open(); - indexReader = DirectoryReader.open(directory); + indexReader = DirectoryReader.open(getDirectory()); indexSearcher = new IndexSearcher(indexReader); searchingAnalyzer = createSearchingAnalyzer(); queryParser = new QueryParser(Version.LUCENE_43, Fields.DOCUMENT_KEY, searchingAnalyzer); diff --git a/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/cpe/CpeIndexWriter.java b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/cpe/CpeIndexWriter.java index cc4c194c4..7efe09f7c 100644 --- a/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/cpe/CpeIndexWriter.java +++ b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/cpe/CpeIndexWriter.java @@ -63,7 +63,7 @@ public class CpeIndexWriter extends BaseIndex { super.open(); indexingAnalyzer = createIndexingAnalyzer(); final IndexWriterConfig conf = new IndexWriterConfig(Version.LUCENE_43, indexingAnalyzer); - indexWriter = new IndexWriter(directory, conf); + indexWriter = new IndexWriter(getDirectory(), conf); } /** diff --git a/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/nvdcve/NvdCve20Handler.java b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/nvdcve/NvdCve20Handler.java index 0a4b439f9..da5686320 100644 --- a/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/nvdcve/NvdCve20Handler.java +++ b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/nvdcve/NvdCve20Handler.java @@ -25,8 +25,6 @@ import java.util.logging.Level; import java.util.logging.Logger; import org.apache.lucene.index.CorruptIndexException; import org.owasp.dependencycheck.data.cpe.CpeIndexWriter; -import org.owasp.dependencycheck.data.nvdcve.CveDB; -import org.owasp.dependencycheck.data.nvdcve.DatabaseException; import org.owasp.dependencycheck.dependency.Reference; import org.owasp.dependencycheck.dependency.Vulnerability; import org.owasp.dependencycheck.dependency.VulnerableSoftware; diff --git a/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/DataStoreMetaInfo.java b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/DataStoreMetaInfo.java index 3d25c8e22..d589abe6a 100644 --- a/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/DataStoreMetaInfo.java +++ b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/DataStoreMetaInfo.java @@ -104,7 +104,7 @@ public class DataStoreMetaInfo { * Loads the data's meta properties. */ private void loadProperties() { - File file = getPropertiesFile(); + final File file = getPropertiesFile(); if (file.exists()) { InputStream is = null; try { diff --git a/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/DatabaseUpdater.java b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/DatabaseUpdater.java index 9af48b57b..f69ae3c63 100644 --- a/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/DatabaseUpdater.java +++ b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/DatabaseUpdater.java @@ -54,6 +54,7 @@ import static org.owasp.dependencycheck.data.update.DataStoreMetaInfo.BATCH; import static org.owasp.dependencycheck.data.update.DataStoreMetaInfo.MODIFIED; /** + * Class responsible for updating the CPE and NVDCVE data stores. * * @author Jeremy Long (jeremy.long@owasp.org) */ @@ -74,7 +75,7 @@ public class DatabaseUpdater implements CachedWebDataSource { /** * A flag indicating whether or not the batch update should be performed. */ - protected boolean doBatchUpdate; + private boolean doBatchUpdate; /** * Get the value of doBatchUpdate @@ -266,6 +267,12 @@ public class DatabaseUpdater implements CachedWebDataSource { } } + /** + * Performs the batch update based on the configured batch update URL. + * + * @throws UpdateException thrown if there is an exception during the update + * process + */ private void performBatchUpdate() throws UpdateException { if (properties.isBatchUpdateMode() && doBatchUpdate) { final String batchSrc = Settings.getString(Settings.KEYS.BATCH_UPDATE_URL); @@ -419,7 +426,7 @@ public class DatabaseUpdater implements CachedWebDataSource { } } - NvdCveInfo batchInfo = currentlyPublished.get(BATCH); + final NvdCveInfo batchInfo = currentlyPublished.get(BATCH); if (properties.isBatchUpdateMode() && batchInfo != null) { final long lastUpdated = Long.parseLong(properties.getProperty(DataStoreMetaInfo.BATCH, "0")); if (lastUpdated != batchInfo.getTimestamp()) { @@ -477,7 +484,8 @@ public class DatabaseUpdater implements CachedWebDataSource { } } } catch (NumberFormatException ex) { - Logger.getLogger(DataStoreMetaInfo.class.getName()).log(Level.WARNING, "An invalid schema version or timestamp exists in the data.properties file."); + final String msg = "An invalid schema version or timestamp exists in the data.properties file."; + Logger.getLogger(DataStoreMetaInfo.class.getName()).log(Level.WARNING, msg); Logger.getLogger(DataStoreMetaInfo.class.getName()).log(Level.FINE, null, ex); setDoBatchUpdate(properties.isBatchUpdateMode()); } @@ -521,7 +529,7 @@ public class DatabaseUpdater implements CachedWebDataSource { final Map map = new TreeMap(); String retrieveUrl = Settings.getString(Settings.KEYS.CVE_MODIFIED_20_URL); if (retrieveUrl == null && properties.isBatchUpdateMode()) { - NvdCveInfo item = new NvdCveInfo(); + final NvdCveInfo item = new NvdCveInfo(); retrieveUrl = Settings.getString(Settings.KEYS.BATCH_UPDATE_URL); if (retrieveUrl == null) { final String msg = "Invalid configuration - neither the modified or batch update URLs are specified in the configuration."; diff --git a/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/NvdCveInfo.java b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/NvdCveInfo.java index 76af51cf3..7a27a9124 100644 --- a/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/NvdCveInfo.java +++ b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/NvdCveInfo.java @@ -20,6 +20,8 @@ package org.owasp.dependencycheck.data.update; /** * A pojo that contains the Url and timestamp of the current NvdCve XML files. + * + * @author Jeremy Long (jeremy.long@owasp.org) */ public class NvdCveInfo { diff --git a/dependency-check-core/src/main/java/org/owasp/dependencycheck/utils/FileUtils.java b/dependency-check-core/src/main/java/org/owasp/dependencycheck/utils/FileUtils.java index 3958b1a90..9d0c1108f 100644 --- a/dependency-check-core/src/main/java/org/owasp/dependencycheck/utils/FileUtils.java +++ b/dependency-check-core/src/main/java/org/owasp/dependencycheck/utils/FileUtils.java @@ -32,7 +32,6 @@ import java.util.logging.Logger; import java.util.zip.ZipEntry; import java.util.zip.ZipInputStream; import org.owasp.dependencycheck.Engine; -import org.owasp.dependencycheck.analyzer.AnalysisException; import org.owasp.dependencycheck.analyzer.ArchiveAnalyzer; /** @@ -81,8 +80,11 @@ public final class FileUtils { delete(c); } } - if (!file.delete()) { + if (!org.apache.commons.io.FileUtils.deleteQuietly(file)) { + //if (!file.delete()) { throw new FileNotFoundException("Failed to delete file: " + file); + } else { + file.deleteOnExit(); } } @@ -102,7 +104,8 @@ public final class FileUtils { delete(c); } } - if (!file.delete()) { + if (!org.apache.commons.io.FileUtils.deleteQuietly(file)) { + //if (!file.delete()) { if (deleteOnExit) { file.deleteOnExit(); } else { From 33fa1e1350496f1539defa4d44bac3f322aa4aee Mon Sep 17 00:00:00 2001 From: Jeremy Long Date: Sun, 1 Sep 2013 15:35:53 -0400 Subject: [PATCH 44/50] updated documentation Former-commit-id: d8912348f3b80e1d03f66b3c57a9d6b84a94be92 --- dependency-check-ant/pom.xml | 2 +- dependency-check-cli/pom.xml | 2 +- dependency-check-core/pom.xml | 2 +- dependency-check-jenkins/README.md | 34 ++++++++++++++++++ dependency-check-jenkins/pom.xml | 58 ++++++++++++++++++++++++++++++ dependency-check-maven/pom.xml | 2 +- pom.xml | 3 +- 7 files changed, 98 insertions(+), 5 deletions(-) create mode 100644 dependency-check-jenkins/README.md create mode 100644 dependency-check-jenkins/pom.xml diff --git a/dependency-check-ant/pom.xml b/dependency-check-ant/pom.xml index c499ccaa8..fc8bde844 100644 --- a/dependency-check-ant/pom.xml +++ b/dependency-check-ant/pom.xml @@ -28,7 +28,7 @@ Copyright (c) 2013 - Jeremy Long. All Rights Reserved. dependency-check-ant jar - dependency-check-ant + Dependency-Check Ant Task Dependency-check is a utility that attempts to detect publicly disclosed vulnerabilities contained within project dependencies. It does this by determining if there is a Common Platform Enumeration (CPE) identifier for a given dependency. If found, it will generate a report linking to the associated CVE entries. diff --git a/dependency-check-cli/pom.xml b/dependency-check-cli/pom.xml index 27aa6c55c..bbdefdf69 100644 --- a/dependency-check-cli/pom.xml +++ b/dependency-check-cli/pom.xml @@ -28,7 +28,7 @@ Copyright (c) 2012 - Jeremy Long. All Rights Reserved. dependency-check-cli jar - dependency-check-cli + Dependency-Check Command Line Dependency-Check-Maven is a Maven Plugin that attempts to detect publicly disclosed vulnerabilities contained within project dependencies. It does this by determining if there is a Common Platform Enumeration (CPE) identifier for a given dependency. If found, it will generate a report linking to the associated CVE entries. diff --git a/dependency-check-core/pom.xml b/dependency-check-core/pom.xml index 89823c2b0..343b3d6ed 100644 --- a/dependency-check-core/pom.xml +++ b/dependency-check-core/pom.xml @@ -28,7 +28,7 @@ along with Dependency-Check. If not, see . dependency-check-core jar - dependency-check-core + Dependency-Check Core diff --git a/dependency-check-jenkins/README.md b/dependency-check-jenkins/README.md new file mode 100644 index 000000000..0ef0d78f4 --- /dev/null +++ b/dependency-check-jenkins/README.md @@ -0,0 +1,34 @@ +Dependency-Check Jenkins Plugin +============================== + +Dependency-Check is a utility that attempts to detect publicly disclosed vulnerabilities contained within project dependencies. It does this by determining if there is a Common Platform Enumeration (CPE) identifier for a given dependency. If found, it will generate a report linking to the associated CVE entries. + +The Dependency-Check Jenkins Plugin features the ability to perform a dependency analysis build and later view results post build. The plugin is built using [analysis-core] and features many of the same features that Jenkins static analysis plugins offer, including thresholds, charts and the ability to view vulnerability information should a dependency have one identified. + +More information can be found on the [wiki]. + +Mailing List +------------ + +Subscribe: [dependency-check+subscribe@googlegroups.com] [subscribe] + +Post: [dependency-check@googlegroups.com] [post] + +Copyright & License +------------------- + +Dependency-Check is Copyright (c) 2012-2013 Jeremy Long. All Rights Reserved. + +Dependency-Check Jenkins Plugin is Copyright (c) 2013 Steve Springett. All Rights Reserved. + +Permission to modify and redistribute is granted under the terms of the GPLv3 license. See the [LICENSE.txt] [GPLv3] file for the full license. + +Dependency-Check makes use of several other open source libraries. Please see the [NOTICE.txt] [notices] file for more information. + + + [wiki]: https://github.com/jeremylong/dependency-check-jenkins/wiki + [analysis-core]: http://wiki.jenkins-ci.org/x/CwDgAQ + [subscribe]: mailto:dependency-check+subscribe@googlegroups.com + [post]: mailto:dependency-check@googlegroups.com + [GPLv3]: https://github.com/jeremylong/dependency-check-jenkins/blob/master/LICENSE.txt + [notices]: https://github.com/jeremylong/dependency-check-jenkins/blob/master/NOTICES.txt diff --git a/dependency-check-jenkins/pom.xml b/dependency-check-jenkins/pom.xml new file mode 100644 index 000000000..40fefb412 --- /dev/null +++ b/dependency-check-jenkins/pom.xml @@ -0,0 +1,58 @@ + + 4.0.0 + + This plug-in can independently execute a Dependency-Check analysis and visualize the results. + http://wiki.jenkins-ci.org/display/JENKINS/OWASP+Dependency-Check+Plugin + + org.owasp + dependency-check-parent + 1.0.2-SNAPSHOT + + + org.owasp + dependency-check-jenkins + Dependency-Check Jenkins Plugin + jar + 2012 + + OWASP + http://www.owasp.org + + + + + Steve Springett + steve.springett@owasp.org + OWASP + https://www.owasp.org/index.php/OWASP_Dependency_Check + + architect + developer + + + + + scm:git:git@github.com:jenkinsci/dependency-check-jenkins.git + https://github.com/jenkinsci/dependency-check-jenkins.git + scm:git:git@github.com:jenkinsci/dependency-check-jenkins.git + + + github + https://github.com/jenkinsci/dependency-check-jenkins/issues + + + + Dependency Check + dependency-check+subscribe@googlegroups.com + dependency-check+unsubscribe@googlegroups.com + dependency-check@googlegroups.com + https://groups.google.com/forum/?fromgroups#!forum/dependency-check + + + + + GNU General Public License version 3 + http://www.gnu.org/licenses/ + + + diff --git a/dependency-check-maven/pom.xml b/dependency-check-maven/pom.xml index 026e705e4..7348cecf6 100644 --- a/dependency-check-maven/pom.xml +++ b/dependency-check-maven/pom.xml @@ -30,7 +30,7 @@ Copyright (c) 2013 Jeremy Long. All Rights Reserved. dependency-check-maven maven-plugin - dependency-check-maven + Dependency-Check Maven Plugin Dependency-Check-Maven is a Maven Plugin that attempts to detect publicly disclosed vulnerabilities contained within project dependencies. It does this by determining if there is a Common Platform Enumeration (CPE) identifier for a given dependency. If found, it will generate a report linking to the associated CVE entries. 2013 diff --git a/pom.xml b/pom.xml index d7e75a523..703d8d879 100644 --- a/pom.xml +++ b/pom.xml @@ -35,8 +35,9 @@ along with Dependency-Check. If not, see . dependency-check-cli dependency-check-ant dependency-check-maven + dependency-check-jenkins - dependency-check-parent + Dependency-Check https://github.com/jeremylong/DependencyCheck.git Dependency-check is a utility that attempts to detect publicly disclosed vulnerabilities contained within project dependencies. It does this by determining if there is a Common Platform Enumeration (CPE) identifier for a given dependency. If found, it will generate a report linking to the associated CVE entries. 2012 From 12bdba9a9c020532c544bc57af526f62fbd4f4b6 Mon Sep 17 00:00:00 2001 From: Jeremy Long Date: Mon, 2 Sep 2013 12:42:44 -0400 Subject: [PATCH 45/50] updated site information Former-commit-id: 032930907c01ad54b0d03004ad7b5e73b541c397 --- dependency-check-ant/README.md | 46 +++++-------------- dependency-check-cli/README.md | 36 ++++----------- .../src/site/markdown/index.md | 2 +- dependency-check-jenkins/README.md | 27 ++++++----- dependency-check-jenkins/pom.xml | 15 ++---- dependency-check-jenkins/src/site/site.xml | 30 ++++++++++++ pom.xml | 2 +- src/site/markdown/index.md | 2 +- src/site/site.xml | 9 ++-- 9 files changed, 75 insertions(+), 94 deletions(-) create mode 100644 dependency-check-jenkins/src/site/site.xml diff --git a/dependency-check-ant/README.md b/dependency-check-ant/README.md index 2021cfcbd..4cae03c73 100644 --- a/dependency-check-ant/README.md +++ b/dependency-check-ant/README.md @@ -1,47 +1,23 @@ -Dependency-Check +Dependency-Check Ant Task ========= -Dependency-Check is a utility that attempts to detect publicly disclosed vulnerabilities contained within project dependencies. It does this by determining if there is a Common Platform Enumeration (CPE) identifier for a given dependency. If found, it will generate a report linking to the associated CVE entries. - -More information can be found on the [wiki]. - -Notice -- - -A very big release of new functionality and plugins will be made available during the BlackHat Arsenal on July 31st, 2013. If you are at BlackHat stop by and see the demos! - -Usage -- - -> $ mvn package - -> $ cd target - -> $ java -jar dependency-check-[version].jar -h - -> $ java -jar dependency-check-[version].jar -a Testing -out . -scan ./test-classes -scan ./lib - -Then load the resulting 'DependencyCheck-Report.html' into your favorite browser. +Dependency-Check Ant Task can be used to check the project dependencies for published security vulnerabilities. The checks +performed are a "best effort" and as such, there could be false positives as well as false negatives. However, +vulnerabilities in 3rd party components is a well-known problem and is currently documented in the 2013 OWASP +Top 10 as [A9 - Using Components with Known Vulnerabilities](https://www.owasp.org/index.php/Top_10_2013-A9-Using_Components_with_Known_Vulnerabilities). Mailing List -- +------------ -Subscribe: [dependency-check+subscribe@googlegroups.com] [subscribe] +Subscribe: [dependency-check+subscribe@googlegroups.com](mailto:dependency-check+subscribe@googlegroups.com) -Post: [dependency-check@googlegroups.com] [post] +Post: [dependency-check@googlegroups.com](mailto:dependency-check@googlegroups.com) Copyright & License -- +------------------- Dependency-Check is Copyright (c) 2012-2013 Jeremy Long. All Rights Reserved. -Permission to modify and redistribute is granted under the terms of the GPLv3 license. See the [LICENSE.txt] [GPLv3] file for the full license. +Permission to modify and redistribute is granted under the terms of the GPLv3 license. See the [LICENSE.txt](https://github.com/jeremylong/DependencyCheck/dependency-check-ant/blob/master/LICENSE.txt) file for the full license. -Dependency-Check makes use of several other open source libraries. Please see the [NOTICE.txt] [notices] file for more information. - - - [wiki]: https://github.com/jeremylong/DependencyCheck/wiki - [subscribe]: mailto:dependency-check+subscribe@googlegroups.com - [post]: mailto:dependency-check@googlegroups.com - [GPLv3]: https://github.com/jeremylong/DependencyCheck/blob/master/LICENSE.txt - [notices]: https://github.com/jeremylong/DependencyCheck/blob/master/NOTICES.txt \ No newline at end of file +Dependency-Check-Ant makes use of other open source libraries. Please see the [NOTICE.txt](https://github.com/jeremylong/DependencyCheck/dependency-check-ant/blob/master/NOTICES.txt) file for more information. diff --git a/dependency-check-cli/README.md b/dependency-check-cli/README.md index 2df244ef6..9b6d12969 100644 --- a/dependency-check-cli/README.md +++ b/dependency-check-cli/README.md @@ -1,41 +1,23 @@ -Dependency-Check +Dependency-Check Command Line ================ -Dependency-Check is a utility that attempts to detect publicly disclosed vulnerabilities contained within project dependencies. It does this by determining if there is a Common Platform Enumeration (CPE) identifier for a given dependency. If found, it will generate a report linking to the associated CVE entries. - -More information can be found on the [wiki]. - -Usage ------------- - -``` -$ mvn package -$ cd target/release/bin -$ dependency-check-[version].sh -h -$ dependency-check-[version].sh -a Testing -scan ../../test-classes -``` - -Then load the resulting 'DependencyCheck-Report.html' into your favorite browser. +Dependency-Check Command Line can be used to check project dependencies for published security vulnerabilities. The checks +performed are a "best effort" and as such, there could be false positives as well as false negatives. However, +vulnerabilities in 3rd party components is a well-known problem and is currently documented in the 2013 OWASP +Top 10 as [A9 - Using Components with Known Vulnerabilities](https://www.owasp.org/index.php/Top_10_2013-A9-Using_Components_with_Known_Vulnerabilities). Mailing List ------------ -Subscribe: [dependency-check+subscribe@googlegroups.com] [subscribe] +Subscribe: [dependency-check+subscribe@googlegroups.com](mailto:dependency-check+subscribe@googlegroups.com) -Post: [dependency-check@googlegroups.com] [post] +Post: [dependency-check@googlegroups.com](mailto:dependency-check@googlegroups.com) Copyright & License ------------ Dependency-Check is Copyright (c) 2012-2013 Jeremy Long. All Rights Reserved. -Permission to modify and redistribute is granted under the terms of the GPLv3 license. See the [LICENSE.txt] [GPLv3] file for the full license. +Permission to modify and redistribute is granted under the terms of the GPLv3 license. See the [LICENSE.txt](https://github.com/jeremylong/DependencyCheck/dependency-check-cli/blob/master/LICENSE.txt) file for the full license. -Dependency-Check makes use of several other open source libraries. Please see the [NOTICE.txt] [notices] file for more information. - - - [wiki]: https://github.com/jeremylong/DependencyCheck/wiki - [subscribe]: mailto:dependency-check+subscribe@googlegroups.com - [post]: mailto:dependency-check@googlegroups.com - [GPLv3]: https://github.com/jeremylong/DependencyCheck/blob/master/LICENSE.txt - [notices]: https://github.com/jeremylong/DependencyCheck/blob/master/NOTICES.txt \ No newline at end of file +Dependency-Check Command Line makes use of other open source libraries. Please see the [NOTICE.txt](https://github.com/jeremylong/DependencyCheck/dependency-check-cli/blob/master/NOTICES.txt) file for more information. diff --git a/dependency-check-core/src/site/markdown/index.md b/dependency-check-core/src/site/markdown/index.md index c2a1ddfc4..747bfd87c 100644 --- a/dependency-check-core/src/site/markdown/index.md +++ b/dependency-check-core/src/site/markdown/index.md @@ -13,4 +13,4 @@ The engine is currently exposed via: - [Command Line Tool](../dependency-check-cli/installation.html) - [Maven Plugin](../dependency-check-maven/usage.html) - [Ant Task](../dependency-check-ant/installation.html) -- Jenkins Plugin +- [Jenkins Plugin](../dependency-check-jenkins/index.html) diff --git a/dependency-check-jenkins/README.md b/dependency-check-jenkins/README.md index 0ef0d78f4..05451102b 100644 --- a/dependency-check-jenkins/README.md +++ b/dependency-check-jenkins/README.md @@ -1,34 +1,33 @@ Dependency-Check Jenkins Plugin ============================== -Dependency-Check is a utility that attempts to detect publicly disclosed vulnerabilities contained within project dependencies. It does this by determining if there is a Common Platform Enumeration (CPE) identifier for a given dependency. If found, it will generate a report linking to the associated CVE entries. +The Dependency-Check Jenkins Plugin features the ability to perform a dependency +analysis build and later view results post build. The plugin is built using [analysis-core] +and features many of the same features that Jenkins static analysis plugins offer, +including thresholds, charts and the ability to view vulnerability information should +a dependency have one identified. -The Dependency-Check Jenkins Plugin features the ability to perform a dependency analysis build and later view results post build. The plugin is built using [analysis-core] and features many of the same features that Jenkins static analysis plugins offer, including thresholds, charts and the ability to view vulnerability information should a dependency have one identified. +The main repository is located at [jenkins-cli/dependency-check-jenkins](https://github.com/jenkinsci/dependency-check-jenkins). + +The main site for documentation is located at [OWASP Dependency-Check-Jenkins](https://wiki.jenkins-ci.org/display/JENKINS/OWASP+Dependency-Check+Plugin). More information can be found on the [wiki]. Mailing List ------------ -Subscribe: [dependency-check+subscribe@googlegroups.com] [subscribe] +Subscribe: [dependency-check+subscribe@googlegroups.com](mailto:dependency-check+subscribe@googlegroups.com) -Post: [dependency-check@googlegroups.com] [post] +Post: [dependency-check@googlegroups.com](mailto:dependency-check@googlegroups.com) Copyright & License ------------------- -Dependency-Check is Copyright (c) 2012-2013 Jeremy Long. All Rights Reserved. - Dependency-Check Jenkins Plugin is Copyright (c) 2013 Steve Springett. All Rights Reserved. Permission to modify and redistribute is granted under the terms of the GPLv3 license. See the [LICENSE.txt] [GPLv3] file for the full license. -Dependency-Check makes use of several other open source libraries. Please see the [NOTICE.txt] [notices] file for more information. - - - [wiki]: https://github.com/jeremylong/dependency-check-jenkins/wiki + [wiki]: https://github.com/jenkinsci/dependency-check-jenkins/wiki [analysis-core]: http://wiki.jenkins-ci.org/x/CwDgAQ - [subscribe]: mailto:dependency-check+subscribe@googlegroups.com - [post]: mailto:dependency-check@googlegroups.com - [GPLv3]: https://github.com/jeremylong/dependency-check-jenkins/blob/master/LICENSE.txt - [notices]: https://github.com/jeremylong/dependency-check-jenkins/blob/master/NOTICES.txt + [GPLv3]: https://github.com/jenkinsci/dependency-check-jenkins/blob/master/LICENSE.txt + [notices]: https://github.com/jenkinsci/dependency-check-jenkins/blob/master/NOTICES.txt diff --git a/dependency-check-jenkins/pom.xml b/dependency-check-jenkins/pom.xml index 40fefb412..f3bc3d4c0 100644 --- a/dependency-check-jenkins/pom.xml +++ b/dependency-check-jenkins/pom.xml @@ -10,8 +10,8 @@ org.owasp - dependency-check-jenkins - Dependency-Check Jenkins Plugin + dependency-check-jenkins + Dependency-Check Jenkins Plugin jar 2012 @@ -33,22 +33,13 @@ scm:git:git@github.com:jenkinsci/dependency-check-jenkins.git - https://github.com/jenkinsci/dependency-check-jenkins.git + https://github.com/jenkinsci/dependency-check-jenkins scm:git:git@github.com:jenkinsci/dependency-check-jenkins.git github https://github.com/jenkinsci/dependency-check-jenkins/issues - - - Dependency Check - dependency-check+subscribe@googlegroups.com - dependency-check+unsubscribe@googlegroups.com - dependency-check@googlegroups.com - https://groups.google.com/forum/?fromgroups#!forum/dependency-check - - GNU General Public License version 3 diff --git a/dependency-check-jenkins/src/site/site.xml b/dependency-check-jenkins/src/site/site.xml new file mode 100644 index 000000000..f96a5ae97 --- /dev/null +++ b/dependency-check-jenkins/src/site/site.xml @@ -0,0 +1,30 @@ + + + + + dependency-check-jenkins + + + + + + + + \ No newline at end of file diff --git a/pom.xml b/pom.xml index 703d8d879..5dbaac655 100644 --- a/pom.xml +++ b/pom.xml @@ -78,7 +78,7 @@ along with Dependency-Check. If not, see . scm:git:git@github.com:jeremylong/DependencyCheck.git - https://github.com/jeremylong/DependencyCheck.git + https://github.com/jeremylong/DependencyCheck scm:git:git@github.com:jeremylong/DependencyCheck.git dependency-check-1.0.1 diff --git a/src/site/markdown/index.md b/src/site/markdown/index.md index acd272fb5..f00c39659 100644 --- a/src/site/markdown/index.md +++ b/src/site/markdown/index.md @@ -28,4 +28,4 @@ Dependency-check's core analysis library is exposed in various forms: - [Command Line Tool](dependency-check-cli/index.html) - [Maven Plugin](dependency-check-maven/usage.html) - [Ant Task](dependency-check-ant/installation.html) -- Jenkins Plugin +- [Jenkins Plugin](dependency-check-jenkins/index.html) diff --git a/src/site/site.xml b/src/site/site.xml index 41fdb930d..3dca91576 100644 --- a/src/site/site.xml +++ b/src/site/site.xml @@ -61,15 +61,18 @@ Copyright (c) 2013 Jeremy Long. All Rights Reserved. The core dependency-check library - + The command line interface for dependency-check. - + An Ant task to run dependency-check. - + A Maven plugin for dependency-check. + + A Jenkins plugin for dependency-check. +