diff --git a/NOTICES.txt b/NOTICES.txt index f3ccac1cf..a2184354c 100644 --- a/NOTICES.txt +++ b/NOTICES.txt @@ -4,8 +4,5 @@ Copyright (c) 2012 Jeremy Long. All Rights Reserved. This product includes software developed by The Apache Software Foundation (http://www.apache.org/). -This product includes software developed by -Joda.org (http://www.joda.org/). - This product includes software developed by Jquery.com (http://jquery.com/). \ No newline at end of file diff --git a/README.txt b/README.txt index b4dc2e4ea..e14638718 100644 --- a/README.txt +++ b/README.txt @@ -1,7 +1,7 @@ About: -DependencyCheck is a simple utility that attempts to detect publically disclosed +DependencyCheck is a utility that attempts to detect publically disclosed vulnerabilities contained within project dependencies. It does this by determining -if there is a Common Product Enumeration (CPE) identifier for a given dependency. +if there is a Common Platform Enumeration (CPE) identifier for a given dependency. If found, it will generate a report linking to the associated CVE entries. Usage: diff --git a/pom.xml b/pom.xml index 0b54d8e57..74a4c9202 100644 --- a/pom.xml +++ b/pom.xml @@ -27,8 +27,8 @@ along with DependencyCheck. If not, see . jar DependencyCheck - http://codesecure.blogspot.com - DependencyCheck is a simple utility that attempts to determine if there is a Common Product Enumeration (CPE) identifier for a given project dependency. If found, it will generate a report linking to the associated CVE entries. + https://github.com/jeremylong/DependencyCheck.git + DependencyCheck is a utility that attempts to detect publically disclosed vulnerabilities contained within project dependencies. It does this by determining if there is a Common Platform Enumeration (CPE) identifier for a given dependency. If found, it will generate a report linking to the associated CVE entries. Jeremy Long @@ -436,24 +436,5 @@ along with DependencyCheck. If not, see . javadoc provided - - joda-time - joda-time - 2.1 - - - joda-time - joda-time - 2.1 - javadoc - provided - - - joda-time - joda-time - 2.1 - sources - provided - diff --git a/src/main/java/org/codesecure/dependencycheck/data/cpe/Index.java b/src/main/java/org/codesecure/dependencycheck/data/cpe/Index.java index f49add80b..ff592f3d4 100644 --- a/src/main/java/org/codesecure/dependencycheck/data/cpe/Index.java +++ b/src/main/java/org/codesecure/dependencycheck/data/cpe/Index.java @@ -23,6 +23,7 @@ import java.io.FileInputStream; import java.io.FileNotFoundException; import java.io.FileOutputStream; import java.io.IOException; +import java.io.InputStream; import java.io.OutputStream; import java.io.OutputStreamWriter; import java.net.MalformedURLException; @@ -47,8 +48,6 @@ import org.codesecure.dependencycheck.utils.Downloader; import org.codesecure.dependencycheck.utils.Settings; import org.codesecure.dependencycheck.data.cpe.xml.Importer; import org.codesecure.dependencycheck.utils.DownloadFailedException; -import org.joda.time.DateTime; -import org.joda.time.Days; import org.xml.sax.SAXException; /** @@ -140,23 +139,24 @@ public class Index { * @throws IOException is thrown if a temporary file could not be created. */ public void updateIndexFromWeb() throws MalformedURLException, ParserConfigurationException, SAXException, IOException { - if (updateNeeded()) { + long timeStamp = updateNeeded(); + if (timeStamp > 0) { URL url = new URL(Settings.getString(Settings.KEYS.CPE_URL)); File outputPath = null; try { outputPath = File.createTempFile("cpe", ".xml"); - Downloader.fetchFile(url, outputPath); + Downloader.fetchFile(url, outputPath, true); Importer.importXML(outputPath.toString()); - writeLastUpdatedPropertyFile(); - + writeLastUpdatedPropertyFile(timeStamp); } catch (DownloadFailedException ex) { Logger.getLogger(Index.class.getName()).log(Level.SEVERE, null, ex); } finally { - boolean deleted = false; try { - deleted = outputPath.delete(); + if (outputPath != null && outputPath.exists()) { + outputPath.delete(); + } } finally { - if (!deleted) { + if (outputPath != null && outputPath.exists()) { outputPath.deleteOnExit(); } } @@ -164,12 +164,15 @@ public class Index { } } - private void writeLastUpdatedPropertyFile() { - DateTime now = new DateTime(); + /** + * Writes a properties file containing the last updated date to the CPE directory. + * @param timeStamp the timestamp to write. + */ + private void writeLastUpdatedPropertyFile(long timeStamp) { String dir = Settings.getString(Settings.KEYS.CPE_INDEX); File cpeProp = new File(dir + File.separatorChar + UPDATE_PROPERTIES_FILE); Properties prop = new Properties(); - prop.put(this.LAST_UPDATED, String.valueOf(now.getMillis())); + prop.put(Index.LAST_UPDATED, String.valueOf(timeStamp)); OutputStream os = null; try { os = new FileOutputStream(cpeProp); @@ -194,48 +197,91 @@ public class Index { } /** - * Determines if the index needs to be updated. + * Determines if the index needs to be updated. This is done by fetching the + * cpe.meta data and checking the lastModifiedDate. If the CPE data needs to + * be refreshed this method will return the timestamp of the new CPE. If an + * update is not required this function will return 0. * - * @return whether or not the CPE Index needs to be updated. + * @return the timestamp of the currently published CPE.xml if the index needs to be updated, otherwise returns 0.. + * @throws MalformedURLException is thrown if the URL for the CPE Meta data is incorrect. + * @throws DownloadFailedException is thrown if there is an error downloading the cpe.meta data file. */ - public boolean updateNeeded() { - boolean needed = false; - String lastUpdated = null; + public long updateNeeded() throws MalformedURLException, DownloadFailedException { + long retVal = 0; + long lastUpdated = 0; + long currentlyPublishedDate = retrieveCurrentCPETimestampFromWeb(); + if (currentlyPublishedDate == 0) { + throw new DownloadFailedException("Unable to retrieve valid timestamp from cpe.meta file"); + } + String dir = Settings.getString(Settings.KEYS.CPE_INDEX); File f = new File(dir); if (!f.exists()) { - needed = true; + retVal = currentlyPublishedDate; } else { File cpeProp = new File(dir + File.separatorChar + UPDATE_PROPERTIES_FILE); if (!cpeProp.exists()) { - needed = true; + retVal = currentlyPublishedDate; } else { Properties prop = new Properties(); - FileInputStream is = null; + InputStream is = null; try { is = new FileInputStream(cpeProp); prop.load(is); - lastUpdated = prop.getProperty(this.LAST_UPDATED); + lastUpdated = Long.parseLong(prop.getProperty(Index.LAST_UPDATED)); } catch (FileNotFoundException ex) { Logger.getLogger(Index.class.getName()).log(Level.FINEST, null, ex); } catch (IOException ex) { Logger.getLogger(Index.class.getName()).log(Level.FINEST, null, ex); - } - try { - long lastupdate = Long.parseLong(lastUpdated); - DateTime last = new DateTime(lastupdate); - DateTime now = new DateTime(); - Days d = Days.daysBetween(last, now); - int days = d.getDays(); - int freq = Settings.getInt(Settings.KEYS.CPE_DOWNLOAD_FREQUENCY); - if (days >= freq) { - needed = true; - } } catch (NumberFormatException ex) { - needed = true; + Logger.getLogger(Index.class.getName()).log(Level.FINEST, null, ex); + } + if (currentlyPublishedDate > lastUpdated) { + retVal = currentlyPublishedDate; } } } - return needed; + return retVal; + } + + /** + * Retrieves the timestamp from the CPE meta data file. + * @return the timestamp from the currently published cpe.meta. + * @throws MalformedURLException is thrown if the URL for the CPE Meta data is incorrect. + * @throws DownloadFailedException is thrown if there is an error downloading the cpe.meta data file. + */ + private long retrieveCurrentCPETimestampFromWeb() throws MalformedURLException, DownloadFailedException { + long timestamp = 0; + File tmp = null; + InputStream is = null; + try { + tmp = File.createTempFile("cpe", "meta"); + URL url = new URL(Settings.getString(Settings.KEYS.CPE_META_URL)); + Downloader.fetchFile(url, tmp); + Properties prop = new Properties(); + is = new FileInputStream(tmp); + prop.load(is); + timestamp = Long.parseLong(prop.getProperty("lastModifiedDate")); + } catch (IOException ex) { + throw new DownloadFailedException("Unable to create temporary file for CPE Meta File download.", ex); + } finally { + try { + if (is != null) { + try { + is.close(); + } catch (IOException ex) { + Logger.getLogger(Index.class.getName()).log(Level.FINEST, null, ex); + } + } + if (tmp != null && tmp.exists()) { + tmp.delete(); + } + } finally { + if (tmp != null && tmp.exists()) { + tmp.deleteOnExit(); + } + } + } + return timestamp; } } diff --git a/src/main/java/org/codesecure/dependencycheck/utils/Downloader.java b/src/main/java/org/codesecure/dependencycheck/utils/Downloader.java index 229197cec..3a984a1a5 100644 --- a/src/main/java/org/codesecure/dependencycheck/utils/Downloader.java +++ b/src/main/java/org/codesecure/dependencycheck/utils/Downloader.java @@ -30,6 +30,7 @@ import java.net.SocketAddress; import java.net.URL; import java.util.logging.Level; import java.util.logging.Logger; +import java.util.zip.GZIPInputStream; /** * A utility to download files from the Internet. @@ -51,8 +52,19 @@ public class Downloader { * @throws DownloadFailedException is thrown if there is an error downloading the file. */ public static void fetchFile(URL url, String outputPath) throws DownloadFailedException { + fetchFile(url, outputPath, false); + } + + /** + * Retrieves a file from a given URL and saves it to the outputPath. + * @param url the URL of the file to download. + * @param outputPath the path to the save the file to. + * @param unzip true/false indicating that the file being retrieved is gzipped and if true, should be uncompressed before writting to the file. + * @throws DownloadFailedException is thrown if there is an error downloading the file. + */ + public static void fetchFile(URL url, String outputPath, boolean unzip) throws DownloadFailedException { File f = new File(outputPath); - fetchFile(url, f); + fetchFile(url, f, unzip); } /** @@ -62,6 +74,17 @@ public class Downloader { * @throws DownloadFailedException is thrown if there is an error downloading the file. */ public static void fetchFile(URL url, File outputPath) throws DownloadFailedException { + fetchFile(url, outputPath, false); + } + + /** + * Retrieves a file from a given URL and saves it to the outputPath. + * @param url the URL of the file to download. + * @param outputPath the path to the save the file to. + * @param unzip true/false indicating that the file being retrieved is gzipped and if true, should be uncompressed before writting to the file. + * @throws DownloadFailedException is thrown if there is an error downloading the file. + */ + public static void fetchFile(URL url, File outputPath, boolean unzip) throws DownloadFailedException { HttpURLConnection conn = null; Proxy proxy = null; String proxyUrl = Settings.getString(Settings.KEYS.PROXY_URL); @@ -96,7 +119,12 @@ public class Downloader { try { //the following times out on some systems because the CPE is big. //InputStream reader = url.openStream(); - InputStream reader = conn.getInputStream(); + InputStream reader; + if (unzip) { + reader = new GZIPInputStream(conn.getInputStream()); + } else { + reader = conn.getInputStream(); + } writer = new BufferedOutputStream(new FileOutputStream(outputPath)); byte[] buffer = new byte[4096]; diff --git a/src/main/java/org/codesecure/dependencycheck/utils/Settings.java b/src/main/java/org/codesecure/dependencycheck/utils/Settings.java index a8194547a..c6b3985c2 100644 --- a/src/main/java/org/codesecure/dependencycheck/utils/Settings.java +++ b/src/main/java/org/codesecure/dependencycheck/utils/Settings.java @@ -47,7 +47,7 @@ public class Settings { /** * The properties key for the URL to the CPE. */ - public static final String CPE_DOWNLOAD_FREQUENCY = "cpe.downloadfrequency"; + public static final String CPE_META_URL = "cpe.meta.url"; /** * The properties key for the path where the CCE Lucene Index will be stored. */ @@ -69,8 +69,8 @@ public class Settings { */ public static final String CONNECTION_TIMEOUT = "connection.timeout"; } - private static final String PROPERTIES_FILE = "dependencycheck.properties"; - private static Settings instance = new Settings(); + private static final String PROPERTIES_FILE = "META-INF/dependencycheck.properties"; + private static final Settings INSTANCE = new Settings(); private Properties props = null; /** @@ -97,7 +97,7 @@ public class Settings { * @return the property from the properties file. */ public static String getString(String key, String defaultValue) { - String str = System.getProperty(key, instance.props.getProperty(key)); + String str = System.getProperty(key, INSTANCE.props.getProperty(key)); if (str == null) { str = defaultValue; } @@ -110,7 +110,7 @@ public class Settings { * @param value the value for the property. */ public static void setString(String key, String value) { - instance.props.setProperty(key, value); + INSTANCE.props.setProperty(key, value); } /** @@ -123,7 +123,7 @@ public class Settings { * @return the property from the properties file. */ public static String getString(String key) { - return System.getProperty(key, instance.props.getProperty(key)); + return System.getProperty(key, INSTANCE.props.getProperty(key)); } /** @@ -151,4 +151,4 @@ public class Settings { public static boolean getBoolean(String key) { return Boolean.parseBoolean(Settings.getString(key)); } -} +} \ No newline at end of file diff --git a/src/main/resources/META-INF/dependencycheck.properties b/src/main/resources/META-INF/dependencycheck.properties index 7527480e0..c77202aa4 100644 --- a/src/main/resources/META-INF/dependencycheck.properties +++ b/src/main/resources/META-INF/dependencycheck.properties @@ -2,8 +2,8 @@ application.name=${pom.name} application.version=${pom.version} cpe=store/cpe -cpe.url=http://static.nvd.nist.gov/feeds/xml/cpe/dictionary/official-cpe-dictionary_v2.2.xml -cpe.downloadfrequency=1 +cpe.url=http://static.nvd.nist.gov/feeds/xml/cpe/dictionary/official-cpe-dictionary_v2.2.xml.gz +cpe.meta.url=http://static.nvd.nist.gov/feeds/xml/cpe/dictionary/official-cpe-dictionary_v2.2.meta cve=store/cve osvdb=store/osvdb diff --git a/src/test/java/org/codesecure/dependencycheck/data/cpe/CPEQueryTest.java b/src/test/java/org/codesecure/dependencycheck/data/cpe/CPEQueryTest.java index 6f890a5b5..620976e46 100644 --- a/src/test/java/org/codesecure/dependencycheck/data/cpe/CPEQueryTest.java +++ b/src/test/java/org/codesecure/dependencycheck/data/cpe/CPEQueryTest.java @@ -21,16 +21,16 @@ import org.junit.Test; * @author jeremy */ public class CPEQueryTest extends BaseIndexTestCase { - + public CPEQueryTest(String testName) { super(testName); } - + @Override protected void setUp() throws Exception { super.setUp(); } - + @Override protected void tearDown() throws Exception { super.tearDown(); @@ -51,15 +51,15 @@ public class CPEQueryTest extends BaseIndexTestCase { String expResult = "cpe:/a:apache:struts:2.1.2"; List result = instance.searchCPE(vendor, product, version); assertEquals(expResult, result.get(0).getName()); - + //TODO - yeah, not a very good test as the results are the same with or without weighting... Set productWeightings = new HashSet(1); productWeightings.add("struts2"); Set vendorWeightings = new HashSet(1); vendorWeightings.add("apache"); - - result = instance.searchCPE(vendor, product, version,productWeightings,vendorWeightings); + + result = instance.searchCPE(vendor, product, version, productWeightings, vendorWeightings); assertEquals(expResult, result.get(0).getName()); vendor = "apache software foundation"; @@ -67,13 +67,13 @@ public class CPEQueryTest extends BaseIndexTestCase { version = "2.3.1.2"; //yes, this isn't right. we verify this with another method later - expResult = "cpe:/a:apache:struts"; + expResult = "cpe:/a:apache:struts"; result = instance.searchCPE(vendor, product, version); boolean startsWith = result.get(0).getName().startsWith(expResult); - assertTrue("CPE does not begin with apache struts",startsWith); + assertTrue("CPE does not begin with apache struts", startsWith); instance.close(); } - + /** * Tests of buildSearch of class CPEQuery. * @throws IOException is thrown when an IO Exception occurs. @@ -97,15 +97,15 @@ public class CPEQueryTest extends BaseIndexTestCase { String queryText = instance.buildSearch(vendor, product, version, null, null); String expResult = " product:( struts 2 core ) vendor:( apache software foundation ) version:(2.1.2)"; assertTrue(expResult.equals(queryText)); - + queryText = instance.buildSearch(vendor, product, version, null, productWeightings); expResult = " product:( struts^5 struts2^5 2 core ) vendor:( apache software foundation ) version:(2.1.2^0.2 )"; assertTrue(expResult.equals(queryText)); - - queryText = instance.buildSearch(vendor, product, version,vendorWeightings,null); + + queryText = instance.buildSearch(vendor, product, version, vendorWeightings, null); expResult = " product:( struts 2 core ) vendor:( apache^5 software foundation ) version:(2.1.2^0.2 )"; assertTrue(expResult.equals(queryText)); - + queryText = instance.buildSearch(vendor, product, version, vendorWeightings, productWeightings); expResult = " product:( struts^5 struts2^5 2 core ) vendor:( apache^5 software foundation ) version:(2.1.2^0.2 )"; assertTrue(expResult.equals(queryText)); @@ -126,7 +126,6 @@ public class CPEQueryTest extends BaseIndexTestCase { assertFalse(instance.isOpen()); } - /** * Test of determineCPE method, of class CPEQuery. * @throws Exception is thrown when an exception occurs @@ -143,7 +142,7 @@ public class CPEQueryTest extends BaseIndexTestCase { instance.determineCPE(depends); instance.close(); assertTrue(depends.getCPEs().contains(expResult)); - assertTrue(depends.getCPEs().size()==1); + assertTrue(depends.getCPEs().size() == 1); } @@ -163,7 +162,7 @@ public class CPEQueryTest extends BaseIndexTestCase { String expResult = "cpe:/a:apache:struts:2.1.2"; List result = instance.searchCPE(vendor, product, version); assertEquals(expResult, result.get(0).getName()); - + vendor = "apache software foundation"; product = "struts 2 core"; version = "2.3.1.2"; @@ -172,7 +171,7 @@ public class CPEQueryTest extends BaseIndexTestCase { result = instance.searchCPE(vendor, product, version); boolean startsWith = result.get(0).getName().startsWith(expResult); assertTrue("CPE Does not start with apache struts.", startsWith); - + instance.close(); } @@ -187,22 +186,21 @@ public class CPEQueryTest extends BaseIndexTestCase { String product = "struts 2 core"; String version = "2.1.2"; String expResult = "cpe:/a:apache:struts:2.1.2"; - + CPEQuery instance = new CPEQuery(); instance.open(); - + //TODO - yeah, not a very good test as the results are the same with or without weighting... Set productWeightings = new HashSet(1); productWeightings.add("struts2"); Set vendorWeightings = new HashSet(1); vendorWeightings.add("apache"); - - List result = instance.searchCPE(vendor, product, version,productWeightings,vendorWeightings); + + List result = instance.searchCPE(vendor, product, version, productWeightings, vendorWeightings); assertEquals(expResult, result.get(0).getName()); - + instance.close(); } - } diff --git a/src/test/java/org/codesecure/dependencycheck/data/cpe/IndexTest.java b/src/test/java/org/codesecure/dependencycheck/data/cpe/IndexTest.java new file mode 100644 index 000000000..c88d05c66 --- /dev/null +++ b/src/test/java/org/codesecure/dependencycheck/data/cpe/IndexTest.java @@ -0,0 +1,103 @@ +/* + * To change this template, choose Tools | Templates + * and open the template in the editor. + */ +package org.codesecure.dependencycheck.data.cpe; + +import org.codesecure.dependencycheck.data.BaseIndexTestCase; +import java.io.IOException; +import java.util.logging.Level; +import java.util.logging.Logger; +import org.apache.lucene.analysis.Analyzer; +import org.apache.lucene.index.CorruptIndexException; +import org.apache.lucene.store.Directory; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; +import static org.junit.Assert.*; + +/** + * + * @author Jeremy Long (jeremy.long@gmail.com) + */ +public class IndexTest extends BaseIndexTestCase { + + public IndexTest(String testCase) { + super(testCase); + } + + @BeforeClass + public static void setUpClass() throws Exception { + } + + @AfterClass + public static void tearDownClass() throws Exception { + } + + @Before + public void setUp() { + } + + @After + public void tearDown() { + } + + /** + * Test of open method, of class Index. + */ + @Test + public void testOpen() { + System.out.println("open"); + Index instance = new Index(); + try { + instance.open(); + } catch (IOException ex) { + fail(ex.getMessage()); + } + try { + instance.close(); + } catch (CorruptIndexException ex) { + fail(ex.getMessage()); + } catch (IOException ex) { + fail(ex.getMessage()); + } + } + + /** + * Test of getDirectory method, of class Index. + */ + @Test + public void testGetDirectory() throws Exception { + System.out.println("getDirectory"); + Directory result = Index.getDirectory(); + String exp = "\\target\\store\\cpe"; + // TODO review the generated test code and remove the default call to fail. + assertTrue(result.toString().contains(exp)); + } + + /** + * Test of updateIndexFromWeb method, of class Index. + */ + @Test + public void testUpdateIndexFromWeb() throws Exception { + System.out.println("updateIndexFromWeb"); + Index instance = new Index(); + instance.updateIndexFromWeb(); + } + + /** + * Test of updateNeeded method, of class Index. + */ + @Test + public void testUpdateNeeded() throws Exception { + System.out.println("updateNeeded"); + Index instance = new Index(); + long expResult = 0L; + long result = instance.updateNeeded(); + //if an exception is thrown this test fails. However, because it depends on the + // order of the tests what this will return I am just testing for the exception. + //assertTrue(expResult < result); + } +}