github-mirrors/DependencyCheck
1
0
Fork 0
mirror of https://github.com/ysoftdevs/DependencyCheck.git synced 2026-01-14 07:43:40 +01:00
Code Issues Packages Projects Releases Wiki Activity

improved downloading of CPE data

Browse Source
This commit is contained in:
Jeremy Long
2012-09-14 06:14:54 -04:00
parent 46f0f891f3
commit c089ac330a
9 changed files with 247 additions and 94 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
NOTICES.txt
View File

@@ -4,8 +4,5 @@ Copyright (c) 2012 Jeremy Long. All Rights Reserved.
This product includes software developed by
The Apache Software Foundation (http://www.apache.org/).
This product includes software developed by
Joda.org (http://www.joda.org/).
This product includes software developed by
Jquery.com (http://jquery.com/).

4
README.txt
View File

@@ -1,7 +1,7 @@
About:
DependencyCheck is a simple utility that attempts to detect publically disclosed
DependencyCheck is a utility that attempts to detect publically disclosed
vulnerabilities contained within project dependencies. It does this by determining
if there is a Common Product Enumeration (CPE) identifier for a given dependency.
if there is a Common Platform Enumeration (CPE) identifier for a given dependency.
If found, it will generate a report linking to the associated CVE entries.
Usage:

23
pom.xml
View File

@@ -27,8 +27,8 @@ along with DependencyCheck. If not, see <http://www.gnu.org/licenses/>.
<packaging>jar</packaging>
<name>DependencyCheck</name>
<url>http://codesecure.blogspot.com</url>
<description>DependencyCheck is a simple utility that attempts to determine if there is a Common Product Enumeration (CPE) identifier for a given project dependency. If found, it will generate a report linking to the associated CVE entries.</description>
<url>https://github.com/jeremylong/DependencyCheck.git</url>
<description>DependencyCheck is a utility that attempts to detect publically disclosed vulnerabilities contained within project dependencies. It does this by determining if there is a Common Platform Enumeration (CPE) identifier for a given dependency. If found, it will generate a report linking to the associated CVE entries.</description>
<organization>
<name>Jeremy Long</name>
</organization>
@@ -436,24 +436,5 @@ along with DependencyCheck. If not, see <http://www.gnu.org/licenses/>.
<classifier>javadoc</classifier>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>joda-time</groupId>
<artifactId>joda-time</artifactId>
<version>2.1</version>
</dependency>
<dependency>
<groupId>joda-time</groupId>
<artifactId>joda-time</artifactId>
<version>2.1</version>
<classifier>javadoc</classifier>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>joda-time</groupId>
<artifactId>joda-time</artifactId>
<version>2.1</version>
<classifier>sources</classifier>
<scope>provided</scope>
</dependency>
</dependencies>
</project>

114
src/main/java/org/codesecure/dependencycheck/data/cpe/Index.java
View File

@@ -23,6 +23,7 @@ import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.io.OutputStreamWriter;
import java.net.MalformedURLException;
@@ -47,8 +48,6 @@ import org.codesecure.dependencycheck.utils.Downloader;
import org.codesecure.dependencycheck.utils.Settings;
import org.codesecure.dependencycheck.data.cpe.xml.Importer;
import org.codesecure.dependencycheck.utils.DownloadFailedException;
import org.joda.time.DateTime;
import org.joda.time.Days;
import org.xml.sax.SAXException;
/**
@@ -140,23 +139,24 @@ public class Index {
* @throws IOException is thrown if a temporary file could not be created.
*/
public void updateIndexFromWeb() throws MalformedURLException, ParserConfigurationException, SAXException, IOException {
if (updateNeeded()) {
long timeStamp = updateNeeded();
if (timeStamp > 0) {
URL url = new URL(Settings.getString(Settings.KEYS.CPE_URL));
File outputPath = null;
try {
outputPath = File.createTempFile("cpe", ".xml");
Downloader.fetchFile(url, outputPath);
Downloader.fetchFile(url, outputPath, true);
Importer.importXML(outputPath.toString());
writeLastUpdatedPropertyFile();
writeLastUpdatedPropertyFile(timeStamp);
} catch (DownloadFailedException ex) {
Logger.getLogger(Index.class.getName()).log(Level.SEVERE, null, ex);
} finally {
boolean deleted = false;
try {
deleted = outputPath.delete();
if (outputPath != null && outputPath.exists()) {
outputPath.delete();
}
} finally {
if (!deleted) {
if (outputPath != null && outputPath.exists()) {
outputPath.deleteOnExit();
}
}
@@ -164,12 +164,15 @@ public class Index {
}
}
private void writeLastUpdatedPropertyFile() {
DateTime now = new DateTime();
/**
* Writes a properties file containing the last updated date to the CPE directory.
* @param timeStamp the timestamp to write.
*/
private void writeLastUpdatedPropertyFile(long timeStamp) {
String dir = Settings.getString(Settings.KEYS.CPE_INDEX);
File cpeProp = new File(dir + File.separatorChar + UPDATE_PROPERTIES_FILE);
Properties prop = new Properties();
prop.put(this.LAST_UPDATED, String.valueOf(now.getMillis()));
prop.put(Index.LAST_UPDATED, String.valueOf(timeStamp));
OutputStream os = null;
try {
os = new FileOutputStream(cpeProp);
@@ -194,48 +197,91 @@ public class Index {
}
/**
* Determines if the index needs to be updated.
* Determines if the index needs to be updated. This is done by fetching the
* cpe.meta data and checking the lastModifiedDate. If the CPE data needs to
* be refreshed this method will return the timestamp of the new CPE. If an
* update is not required this function will return 0.
*
* @return whether or not the CPE Index needs to be updated.
* @return the timestamp of the currently published CPE.xml if the index needs to be updated, otherwise returns 0..
* @throws MalformedURLException is thrown if the URL for the CPE Meta data is incorrect.
* @throws DownloadFailedException is thrown if there is an error downloading the cpe.meta data file.
*/
public boolean updateNeeded() {
boolean needed = false;
String lastUpdated = null;
public long updateNeeded() throws MalformedURLException, DownloadFailedException {
long retVal = 0;
long lastUpdated = 0;
long currentlyPublishedDate = retrieveCurrentCPETimestampFromWeb();
if (currentlyPublishedDate == 0) {
throw new DownloadFailedException("Unable to retrieve valid timestamp from cpe.meta file");
}
String dir = Settings.getString(Settings.KEYS.CPE_INDEX);
File f = new File(dir);
if (!f.exists()) {
needed = true;
retVal = currentlyPublishedDate;
} else {
File cpeProp = new File(dir + File.separatorChar + UPDATE_PROPERTIES_FILE);
if (!cpeProp.exists()) {
needed = true;
retVal = currentlyPublishedDate;
} else {
Properties prop = new Properties();
FileInputStream is = null;
InputStream is = null;
try {
is = new FileInputStream(cpeProp);
prop.load(is);
lastUpdated = prop.getProperty(this.LAST_UPDATED);
lastUpdated = Long.parseLong(prop.getProperty(Index.LAST_UPDATED));
} catch (FileNotFoundException ex) {
Logger.getLogger(Index.class.getName()).log(Level.FINEST, null, ex);
} catch (IOException ex) {
Logger.getLogger(Index.class.getName()).log(Level.FINEST, null, ex);
}
try {
long lastupdate = Long.parseLong(lastUpdated);
DateTime last = new DateTime(lastupdate);
DateTime now = new DateTime();
Days d = Days.daysBetween(last, now);
int days = d.getDays();
int freq = Settings.getInt(Settings.KEYS.CPE_DOWNLOAD_FREQUENCY);
if (days >= freq) {
needed = true;
}
} catch (NumberFormatException ex) {
needed = true;
Logger.getLogger(Index.class.getName()).log(Level.FINEST, null, ex);
}
if (currentlyPublishedDate > lastUpdated) {
retVal = currentlyPublishedDate;
}
}
}
return needed;
return retVal;
}
/**
* Retrieves the timestamp from the CPE meta data file.
* @return the timestamp from the currently published cpe.meta.
* @throws MalformedURLException is thrown if the URL for the CPE Meta data is incorrect.
* @throws DownloadFailedException is thrown if there is an error downloading the cpe.meta data file.
*/
private long retrieveCurrentCPETimestampFromWeb() throws MalformedURLException, DownloadFailedException {
long timestamp = 0;
File tmp = null;
InputStream is = null;
try {
tmp = File.createTempFile("cpe", "meta");
URL url = new URL(Settings.getString(Settings.KEYS.CPE_META_URL));
Downloader.fetchFile(url, tmp);
Properties prop = new Properties();
is = new FileInputStream(tmp);
prop.load(is);
timestamp = Long.parseLong(prop.getProperty("lastModifiedDate"));
} catch (IOException ex) {
throw new DownloadFailedException("Unable to create temporary file for CPE Meta File download.", ex);
} finally {
try {
if (is != null) {
try {
is.close();
} catch (IOException ex) {
Logger.getLogger(Index.class.getName()).log(Level.FINEST, null, ex);
}
}
if (tmp != null && tmp.exists()) {
tmp.delete();
}
} finally {
if (tmp != null && tmp.exists()) {
tmp.deleteOnExit();
}
}
}
return timestamp;
}
}

32
src/main/java/org/codesecure/dependencycheck/utils/Downloader.java
View File

@@ -30,6 +30,7 @@ import java.net.SocketAddress;
import java.net.URL;
import java.util.logging.Level;
import java.util.logging.Logger;
import java.util.zip.GZIPInputStream;
/**
* A utility to download files from the Internet.
@@ -51,8 +52,19 @@ public class Downloader {
* @throws DownloadFailedException is thrown if there is an error downloading the file.
*/
public static void fetchFile(URL url, String outputPath) throws DownloadFailedException {
fetchFile(url, outputPath, false);
}
/**
* Retrieves a file from a given URL and saves it to the outputPath.
* @param url the URL of the file to download.
* @param outputPath the path to the save the file to.
* @param unzip true/false indicating that the file being retrieved is gzipped and if true, should be uncompressed before writting to the file.
* @throws DownloadFailedException is thrown if there is an error downloading the file.
*/
public static void fetchFile(URL url, String outputPath, boolean unzip) throws DownloadFailedException {
File f = new File(outputPath);
fetchFile(url, f);
fetchFile(url, f, unzip);
}
/**
@@ -62,6 +74,17 @@ public class Downloader {
* @throws DownloadFailedException is thrown if there is an error downloading the file.
*/
public static void fetchFile(URL url, File outputPath) throws DownloadFailedException {
fetchFile(url, outputPath, false);
}
/**
* Retrieves a file from a given URL and saves it to the outputPath.
* @param url the URL of the file to download.
* @param outputPath the path to the save the file to.
* @param unzip true/false indicating that the file being retrieved is gzipped and if true, should be uncompressed before writting to the file.
* @throws DownloadFailedException is thrown if there is an error downloading the file.
*/
public static void fetchFile(URL url, File outputPath, boolean unzip) throws DownloadFailedException {
HttpURLConnection conn = null;
Proxy proxy = null;
String proxyUrl = Settings.getString(Settings.KEYS.PROXY_URL);
@@ -96,7 +119,12 @@ public class Downloader {
try {
//the following times out on some systems because the CPE is big.
//InputStream reader = url.openStream();
InputStream reader = conn.getInputStream();
InputStream reader;
if (unzip) {
reader = new GZIPInputStream(conn.getInputStream());
} else {
reader = conn.getInputStream();
}
writer = new BufferedOutputStream(new FileOutputStream(outputPath));
byte[] buffer = new byte[4096];

14
src/main/java/org/codesecure/dependencycheck/utils/Settings.java
View File

@@ -47,7 +47,7 @@ public class Settings {
/**
* The properties key for the URL to the CPE.
*/
public static final String CPE_DOWNLOAD_FREQUENCY = "cpe.downloadfrequency";
public static final String CPE_META_URL = "cpe.meta.url";
/**
* The properties key for the path where the CCE Lucene Index will be stored.
*/
@@ -69,8 +69,8 @@ public class Settings {
*/
public static final String CONNECTION_TIMEOUT = "connection.timeout";
}
private static final String PROPERTIES_FILE = "dependencycheck.properties";
private static Settings instance = new Settings();
private static final String PROPERTIES_FILE = "META-INF/dependencycheck.properties";
private static final Settings INSTANCE = new Settings();
private Properties props = null;
/**
@@ -97,7 +97,7 @@ public class Settings {
* @return the property from the properties file.
*/
public static String getString(String key, String defaultValue) {
String str = System.getProperty(key, instance.props.getProperty(key));
String str = System.getProperty(key, INSTANCE.props.getProperty(key));
if (str == null) {
str = defaultValue;
}
@@ -110,7 +110,7 @@ public class Settings {
* @param value the value for the property.
*/
public static void setString(String key, String value) {
instance.props.setProperty(key, value);
INSTANCE.props.setProperty(key, value);
}
/**
@@ -123,7 +123,7 @@ public class Settings {
* @return the property from the properties file.
*/
public static String getString(String key) {
return System.getProperty(key, instance.props.getProperty(key));
return System.getProperty(key, INSTANCE.props.getProperty(key));
}
/**
@@ -151,4 +151,4 @@ public class Settings {
public static boolean getBoolean(String key) {
return Boolean.parseBoolean(Settings.getString(key));
}
}
}

4
src/main/resources/META-INF/dependencycheck.properties
View File

@@ -2,8 +2,8 @@ application.name=${pom.name}
application.version=${pom.version}
cpe=store/cpe
cpe.url=http://static.nvd.nist.gov/feeds/xml/cpe/dictionary/official-cpe-dictionary_v2.2.xml
cpe.downloadfrequency=1
cpe.url=http://static.nvd.nist.gov/feeds/xml/cpe/dictionary/official-cpe-dictionary_v2.2.xml.gz
cpe.meta.url=http://static.nvd.nist.gov/feeds/xml/cpe/dictionary/official-cpe-dictionary_v2.2.meta
cve=store/cve
osvdb=store/osvdb

44
src/test/java/org/codesecure/dependencycheck/data/cpe/CPEQueryTest.java
View File

@@ -21,16 +21,16 @@ import org.junit.Test;
* @author jeremy
*/
public class CPEQueryTest extends BaseIndexTestCase {
public CPEQueryTest(String testName) {
super(testName);
}
@Override
protected void setUp() throws Exception {
super.setUp();
}
@Override
protected void tearDown() throws Exception {
super.tearDown();
@@ -51,15 +51,15 @@ public class CPEQueryTest extends BaseIndexTestCase {
String expResult = "cpe:/a:apache:struts:2.1.2";
List<Entry> result = instance.searchCPE(vendor, product, version);
assertEquals(expResult, result.get(0).getName());
//TODO - yeah, not a very good test as the results are the same with or without weighting...
Set<String> productWeightings = new HashSet<String>(1);
productWeightings.add("struts2");
Set<String> vendorWeightings = new HashSet<String>(1);
vendorWeightings.add("apache");
result = instance.searchCPE(vendor, product, version,productWeightings,vendorWeightings);
result = instance.searchCPE(vendor, product, version, productWeightings, vendorWeightings);
assertEquals(expResult, result.get(0).getName());
vendor = "apache software foundation";
@@ -67,13 +67,13 @@ public class CPEQueryTest extends BaseIndexTestCase {
version = "2.3.1.2";
//yes, this isn't right. we verify this with another method later
expResult = "cpe:/a:apache:struts";
expResult = "cpe:/a:apache:struts";
result = instance.searchCPE(vendor, product, version);
boolean startsWith = result.get(0).getName().startsWith(expResult);
assertTrue("CPE does not begin with apache struts",startsWith);
assertTrue("CPE does not begin with apache struts", startsWith);
instance.close();
}
/**
* Tests of buildSearch of class CPEQuery.
* @throws IOException is thrown when an IO Exception occurs.
@@ -97,15 +97,15 @@ public class CPEQueryTest extends BaseIndexTestCase {
String queryText = instance.buildSearch(vendor, product, version, null, null);
String expResult = " product:( struts 2 core ) vendor:( apache software foundation ) version:(2.1.2)";
assertTrue(expResult.equals(queryText));
queryText = instance.buildSearch(vendor, product, version, null, productWeightings);
expResult = " product:( struts^5 struts2^5 2 core ) vendor:( apache software foundation ) version:(2.1.2^0.2 )";
assertTrue(expResult.equals(queryText));
queryText = instance.buildSearch(vendor, product, version,vendorWeightings,null);
queryText = instance.buildSearch(vendor, product, version, vendorWeightings, null);
expResult = " product:( struts 2 core ) vendor:( apache^5 software foundation ) version:(2.1.2^0.2 )";
assertTrue(expResult.equals(queryText));
queryText = instance.buildSearch(vendor, product, version, vendorWeightings, productWeightings);
expResult = " product:( struts^5 struts2^5 2 core ) vendor:( apache^5 software foundation ) version:(2.1.2^0.2 )";
assertTrue(expResult.equals(queryText));
@@ -126,7 +126,6 @@ public class CPEQueryTest extends BaseIndexTestCase {
assertFalse(instance.isOpen());
}
/**
* Test of determineCPE method, of class CPEQuery.
* @throws Exception is thrown when an exception occurs
@@ -143,7 +142,7 @@ public class CPEQueryTest extends BaseIndexTestCase {
instance.determineCPE(depends);
instance.close();
assertTrue(depends.getCPEs().contains(expResult));
assertTrue(depends.getCPEs().size()==1);
assertTrue(depends.getCPEs().size() == 1);
}
@@ -163,7 +162,7 @@ public class CPEQueryTest extends BaseIndexTestCase {
String expResult = "cpe:/a:apache:struts:2.1.2";
List<Entry> result = instance.searchCPE(vendor, product, version);
assertEquals(expResult, result.get(0).getName());
vendor = "apache software foundation";
product = "struts 2 core";
version = "2.3.1.2";
@@ -172,7 +171,7 @@ public class CPEQueryTest extends BaseIndexTestCase {
result = instance.searchCPE(vendor, product, version);
boolean startsWith = result.get(0).getName().startsWith(expResult);
assertTrue("CPE Does not start with apache struts.", startsWith);
instance.close();
}
@@ -187,22 +186,21 @@ public class CPEQueryTest extends BaseIndexTestCase {
String product = "struts 2 core";
String version = "2.1.2";
String expResult = "cpe:/a:apache:struts:2.1.2";
CPEQuery instance = new CPEQuery();
instance.open();
//TODO - yeah, not a very good test as the results are the same with or without weighting...
Set<String> productWeightings = new HashSet<String>(1);
productWeightings.add("struts2");
Set<String> vendorWeightings = new HashSet<String>(1);
vendorWeightings.add("apache");
List<Entry> result = instance.searchCPE(vendor, product, version,productWeightings,vendorWeightings);
List<Entry> result = instance.searchCPE(vendor, product, version, productWeightings, vendorWeightings);
assertEquals(expResult, result.get(0).getName());
instance.close();
}
}

103
src/test/java/org/codesecure/dependencycheck/data/cpe/IndexTest.java Normal file
View File

@@ -0,0 +1,103 @@
/*
* To change this template, choose Tools | Templates
* and open the template in the editor.
*/
package org.codesecure.dependencycheck.data.cpe;
import org.codesecure.dependencycheck.data.BaseIndexTestCase;
import java.io.IOException;
import java.util.logging.Level;
import java.util.logging.Logger;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.index.CorruptIndexException;
import org.apache.lucene.store.Directory;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
import static org.junit.Assert.*;
/**
*
* @author Jeremy Long (jeremy.long@gmail.com)
*/
public class IndexTest extends BaseIndexTestCase {
public IndexTest(String testCase) {
super(testCase);
}
@BeforeClass
public static void setUpClass() throws Exception {
}
@AfterClass
public static void tearDownClass() throws Exception {
}
@Before
public void setUp() {
}
@After
public void tearDown() {
}
/**
* Test of open method, of class Index.
*/
@Test
public void testOpen() {
System.out.println("open");
Index instance = new Index();
try {
instance.open();
} catch (IOException ex) {
fail(ex.getMessage());
}
try {
instance.close();
} catch (CorruptIndexException ex) {
fail(ex.getMessage());
} catch (IOException ex) {
fail(ex.getMessage());
}
}
/**
* Test of getDirectory method, of class Index.
*/
@Test
public void testGetDirectory() throws Exception {
System.out.println("getDirectory");
Directory result = Index.getDirectory();
String exp = "\\target\\store\\cpe";
// TODO review the generated test code and remove the default call to fail.
assertTrue(result.toString().contains(exp));
}
/**
* Test of updateIndexFromWeb method, of class Index.
*/
@Test
public void testUpdateIndexFromWeb() throws Exception {
System.out.println("updateIndexFromWeb");
Index instance = new Index();
instance.updateIndexFromWeb();
}
/**
* Test of updateNeeded method, of class Index.
*/
@Test
public void testUpdateNeeded() throws Exception {
System.out.println("updateNeeded");
Index instance = new Index();
long expResult = 0L;
long result = instance.updateNeeded();
//if an exception is thrown this test fails. However, because it depends on the
// order of the tests what this will return I am just testing for the exception.
//assertTrue(expResult < result);
}
}