github-mirrors/DependencyCheck
1
0
Fork 0
mirror of https://github.com/ysoftdevs/DependencyCheck.git synced 2026-03-17 23:04:07 +01:00
Code Issues Packages Projects Releases Wiki Activity

Update NVD CVE timestamp checking

Browse Source 
Former-commit-id: a0b977d3b3066ff369967c4b6abad2a8d2ca0eeb
This commit is contained in:
Jeremy Long
2012-12-30 08:53:14 -05:00
parent f5b48f5390
commit bea19ad8ce
8 changed files with 298 additions and 519 deletions
Download Patch File Download Diff File Expand all files Collapse all files

351
src/main/java/org/codesecure/dependencycheck/data/cpe/xml/CPEHandler.java
View File

@@ -1,351 +0,0 @@
package org.codesecure.dependencycheck.data.cpe.xml;
/*
* This file is part of DependencyCheck.
*
* DependencyCheck is free software: you can redistribute it and/or modify it
* under the terms of the GNU General Public License as published by the Free
* Software Foundation, either version 3 of the License, or (at your option) any
* later version.
*
* DependencyCheck is distributed in the hope that it will be useful, but
* WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
* details.
*
* You should have received a copy of the GNU General Public License along with
* DependencyCheck. If not, see http://www.gnu.org/licenses/.
*
* Copyright (c) 2012 Jeremy Long. All Rights Reserved.
*/
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.text.ParseException;
import java.util.logging.Level;
import java.util.logging.Logger;
import org.apache.lucene.index.CorruptIndexException;
import org.codesecure.dependencycheck.data.cpe.Entry;
import org.xml.sax.Attributes;
import org.xml.sax.SAXException;
import org.xml.sax.helpers.DefaultHandler;
/**
* A SAX Handler that will parse the CPE XML Listing.
*
* @author Jeremy Long (jeremy.long@gmail.com)
*/
public class CPEHandler extends DefaultHandler {
private static final String CURRENT_SCHEMA_VERSION = "2.2";
EntrySaveDelegate saveDelegate = null;
Entry entry = null;
boolean languageIsUS = false;
StringBuilder nodeText = null;
boolean skip = false;
Element current = new Element();
/**
* Register a EntrySaveDelegate object. When the last node of an entry is
* reached if a save delegate has been registered the save method will be
* invoked.
*
* @param delegate the delegate used to save an entry
*/
public void registerSaveDelegate(EntrySaveDelegate delegate) {
this.saveDelegate = delegate;
}
@Override
public void startElement(String uri, String localName, String qName, Attributes attributes) throws SAXException {
nodeText = null;
current.setNode(qName);
if (current.isCpeItemNode()) {
entry = new Entry();
String temp = attributes.getValue("deprecated");
String name = attributes.getValue("name");
skip = (temp != null && temp.equals("true"));
try {
if (!skip && name.startsWith("cpe:/a:")) {
entry.parseName(name);
} else {
skip = true;
}
} catch (UnsupportedEncodingException ex) {
throw new SAXException(ex);
}
} else if (current.isTitleNode()) {
nodeText = new StringBuilder(100);
if ("en-US".equalsIgnoreCase(attributes.getValue("xml:lang"))) {
languageIsUS = true;
} else {
languageIsUS = false;
}
} else if (current.isMetaNode()) {
try {
entry.setModificationDate(attributes.getValue("modification-date"));
} catch (ParseException ex) {
Logger.getLogger(CPEHandler.class.getName()).log(Level.SEVERE, null, ex);
}
entry.setStatus(attributes.getValue("status"));
entry.setNvdId(attributes.getValue("nvd-id"));
} else if (current.isSchemaVersionNode()) {
nodeText = new StringBuilder(3);
} else if (current.isTimestampNode()) {
nodeText = new StringBuilder(24);
}
// } else if (current.isCpeListNode()) {
// //do nothing
// } else if (current.isNotesNode()) {
// //do nothing
// } else if (current.isNoteNode()) {
// //do nothing
// } else if (current.isCheckNode()) {
// //do nothing
// } else if (current.isGeneratorNode()) {
// //do nothing
// } else if (current.isProductNameNode()) {
// //do nothing
// } else if (current.isProductVersionNode()) {
// //do nothing
}
@Override
public void characters(char[] ch, int start, int length) throws SAXException {
//nodeText += new String(ch, start, length);
if (nodeText != null) {
nodeText.append(ch, start, length);
}
}
@Override
public void endElement(String uri, String localName, String qName) throws SAXException {
current.setNode(qName);
if (current.isCpeItemNode()) {
if (saveDelegate != null && !skip) {
try {
saveDelegate.saveEntry(entry);
} catch (CorruptIndexException ex) {
Logger.getLogger(CPEHandler.class.getName()).log(Level.SEVERE, null, ex);
throw new SAXException(ex);
} catch (IOException ex) {
Logger.getLogger(CPEHandler.class.getName()).log(Level.SEVERE, null, ex);
throw new SAXException(ex);
}
entry = null;
}
} else if (current.isTitleNode()) {
if (languageIsUS) {
entry.setTitle(nodeText.toString());
}
} else if (current.isSchemaVersionNode() && !CURRENT_SCHEMA_VERSION.equals(nodeText.toString())) {
throw new SAXException("ERROR: Invalid Schema Version, expected: "
+ CURRENT_SCHEMA_VERSION + ", file is: " + nodeText);
}
// } else if (current.isCpeListNode()) {
// //do nothing
// } else if (current.isMetaNode()) {
// //do nothing
// } else if (current.isNotesNode()) {
// //do nothing
// } else if (current.isNoteNode()) {
// //do nothing
// } else if (current.isCheckNode()) {
// //do nothing
// } else if (current.isGeneratorNode()) {
// //do nothing
// } else if (current.isProductNameNode()) {
// //do nothing
// } else if (current.isProductVersionNode()) {
// //do nothing
// else if (current.isTimestampNode()) {
// //do nothing
// } else {
// throw new SAXException("ERROR STATE: Unexpected qName '" + qName + "'");
// }
}
// <editor-fold defaultstate="collapsed" desc="The Element Class that maintains state information about the current node">
/**
* A simple class to maintain information about the current element while
* parsing the CPE XML.
*/
protected class Element {
/**
* A node type in the CPE Schema 2.2
*/
public static final String CPE_LIST = "cpe-list";
/**
* A node type in the CPE Schema 2.2
*/
public static final String CPE_ITEM = "cpe-item";
/**
* A node type in the CPE Schema 2.2
*/
public static final String TITLE = "title";
/**
* A node type in the CPE Schema 2.2
*/
public static final String NOTES = "notes";
/**
* A node type in the CPE Schema 2.2
*/
public static final String NOTE = "note";
/**
* A node type in the CPE Schema 2.2
*/
public static final String CHECK = "check";
/**
* A node type in the CPE Schema 2.2
*/
public static final String META = "meta:item-metadata";
/**
* A node type in the CPE Schema 2.2
*/
public static final String GENERATOR = "generator";
/**
* A node type in the CPE Schema 2.2
*/
public static final String PRODUCT_NAME = "product_name";
/**
* A node type in the CPE Schema 2.2
*/
public static final String PRODUCT_VERSION = "product_version";
/**
* A node type in the CPE Schema 2.2
*/
public static final String SCHEMA_VERSION = "schema_version";
/**
* A node type in the CPE Schema 2.2
*/
public static final String TIMESTAMP = "timestamp";
private String node = null;
/**
* Gets the value of node
*
* @return the value of node
*/
public String getNode() {
return this.node;
}
/**
* Sets the value of node
*
* @param node new value of node
*/
public void setNode(String node) {
this.node = node;
}
/**
* Checks if the handler is at the CPE_LIST node
*
* @return true or false
*/
public boolean isCpeListNode() {
return CPE_LIST.equals(node);
}
/**
* Checks if the handler is at the CPE_ITEM node
*
* @return true or false
*/
public boolean isCpeItemNode() {
return CPE_ITEM.equals(node);
}
/**
* Checks if the handler is at the TITLE node
*
* @return true or false
*/
public boolean isTitleNode() {
return TITLE.equals(node);
}
/**
* Checks if the handler is at the NOTES node
*
* @return true or false
*/
public boolean isNotesNode() {
return NOTES.equals(node);
}
/**
* Checks if the handler is at the NOTE node
*
* @return true or false
*/
public boolean isNoteNode() {
return NOTE.equals(node);
}
/**
* Checks if the handler is at the CHECK node
*
* @return true or false
*/
public boolean isCheckNode() {
return CHECK.equals(node);
}
/**
* Checks if the handler is at the META node
*
* @return true or false
*/
public boolean isMetaNode() {
return META.equals(node);
}
/**
* Checks if the handler is at the GENERATOR node
*
* @return true or false
*/
public boolean isGeneratorNode() {
return GENERATOR.equals(node);
}
/**
* Checks if the handler is at the PRODUCT_NAME node
*
* @return true or false
*/
public boolean isProductNameNode() {
return PRODUCT_NAME.equals(node);
}
/**
* Checks if the handler is at the PRODUCT_VERSION node
*
* @return true or false
*/
public boolean isProductVersionNode() {
return PRODUCT_VERSION.equals(node);
}
/**
* Checks if the handler is at the SCHEMA_VERSION node
*
* @return true or false
*/
public boolean isSchemaVersionNode() {
return SCHEMA_VERSION.equals(node);
}
/**
* Checks if the handler is at the TIMESTAMP node
*
* @return true or false
*/
public boolean isTimestampNode() {
return TIMESTAMP.equals(node);
}
}
// </editor-fold>
}

280
src/main/java/org/codesecure/dependencycheck/data/nvdcve/xml/IndexUpdater.java
View File

@@ -24,28 +24,24 @@ import java.io.FileNotFoundException;
import java.io.FileOutputStream; import java.io.FileOutputStream;
import java.io.IOException; import java.io.IOException;
import java.io.InputStream; import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.OutputStream; import java.io.OutputStream;
import java.io.OutputStreamWriter; import java.io.OutputStreamWriter;
import org.codesecure.dependencycheck.data.CachedWebDataSource; import org.codesecure.dependencycheck.data.CachedWebDataSource;
import java.net.MalformedURLException; import java.net.MalformedURLException;
import java.net.URL; import java.net.URL;
import java.util.Calendar;
import java.util.Date; import java.util.Date;
import java.util.GregorianCalendar;
import java.util.HashMap; import java.util.HashMap;
import java.util.Map; import java.util.Map;
import java.util.Properties; import java.util.Properties;
import java.util.logging.Level; import java.util.logging.Level;
import java.util.logging.Logger; import java.util.logging.Logger;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.apache.lucene.index.CorruptIndexException; import org.apache.lucene.index.CorruptIndexException;
import org.codesecure.dependencycheck.data.nvdcve.Index; import org.codesecure.dependencycheck.data.nvdcve.Index;
import org.codesecure.dependencycheck.data.UpdateException; import org.codesecure.dependencycheck.data.UpdateException;
import org.codesecure.dependencycheck.utils.DownloadFailedException; import org.codesecure.dependencycheck.utils.DownloadFailedException;
import org.codesecure.dependencycheck.utils.Downloader; import org.codesecure.dependencycheck.utils.Downloader;
import org.codesecure.dependencycheck.utils.FileUtils; import org.codesecure.dependencycheck.utils.FileUtils;
import org.codesecure.dependencycheck.utils.InvalidSettingException;
import org.codesecure.dependencycheck.utils.Settings; import org.codesecure.dependencycheck.utils.Settings;
/** /**
@@ -283,7 +279,12 @@ public class IndexUpdater extends Index implements CachedWebDataSource {
} catch (InvalidDataException ex) { } catch (InvalidDataException ex) {
Logger.getLogger(IndexUpdater.class.getName()).log(Level.SEVERE, null, ex); Logger.getLogger(IndexUpdater.class.getName()).log(Level.SEVERE, null, ex);
throw new DownloadFailedException("Unable to retrieve valid timestamp from nvd cve downloads page", ex); throw new DownloadFailedException("Unable to retrieve valid timestamp from nvd cve downloads page", ex);
} catch (InvalidSettingException ex) {
Logger.getLogger(IndexUpdater.class.getName()).log(Level.SEVERE, null, ex);
throw new DownloadFailedException("Invalid settings", ex);
} }
if (currentlyPublished == null) { if (currentlyPublished == null) {
throw new DownloadFailedException("Unable to retrieve valid timestamp from nvd cve downloads page"); throw new DownloadFailedException("Unable to retrieve valid timestamp from nvd cve downloads page");
} }
@@ -401,139 +402,176 @@ public class IndexUpdater extends Index implements CachedWebDataSource {
* Retrieves the timestamps from the NVD CVE meta data file. * Retrieves the timestamps from the NVD CVE meta data file.
* *
* @return the timestamp from the currently published nvdcve downloads page * @return the timestamp from the currently published nvdcve downloads page
* @throws MalformedURLException is thrown if the URL for the NVD CCE Meta * @throws MalformedURLException thrown if the URL for the NVD CCE Meta
* data is incorrect. * data is incorrect.
* @throws DownloadFailedException is thrown if there is an error * @throws DownloadFailedException thrown if there is an error
* downloading the nvd cve meta data file * downloading the nvd cve meta data file
* @throws InvalidDataException is thrown if there is an exception parsing * @throws InvalidDataException thrown if there is an exception parsing
* the timestamps * the timestamps
* @throws InvalidSettingException thrown if the settings are invalid
*/ */
protected Map<String, NvdCveUrl> retrieveCurrentTimestampsFromWeb() throws MalformedURLException, DownloadFailedException, InvalidDataException { protected Map<String, NvdCveUrl> retrieveCurrentTimestampsFromWeb()
throws MalformedURLException, DownloadFailedException, InvalidDataException, InvalidSettingException {
Map<String, NvdCveUrl> map = new HashMap<String, NvdCveUrl>(); Map<String, NvdCveUrl> map = new HashMap<String, NvdCveUrl>();
String retrieveUrl = Settings.getString(Settings.KEYS.CVE_MODIFIED_20_URL);
File tmp = null; NvdCveUrl item = new NvdCveUrl();
try { item.setNeedsUpdate(false); //the others default to true, to make life easier later this should default to false.
tmp = File.createTempFile("cve", "meta"); item.id = "modified";
URL url = new URL(Settings.getString(Settings.KEYS.CVE_META_URL)); item.timestamp = Downloader.getLastModified(new URL(retrieveUrl));
Downloader.fetchFile(url, tmp); map.put("modified", item);
String html = readFile(tmp);
String retrieveUrl = Settings.getString(Settings.KEYS.CVE_MODIFIED_URL);
NvdCveUrl cve = createNvdCveUrl("modified", retrieveUrl, html);
cve.setNeedsUpdate(false); //the others default to true, to make life easier later this should default to false.
map.put("modified", cve);
int max = Settings.getInt(Settings.KEYS.CVE_URL_COUNT); int max = Settings.getInt(Settings.KEYS.CVE_URL_COUNT);
for (int i = 1; i <= max; i++) { for (int i = 1; i <= max; i++) {
retrieveUrl = Settings.getString(Settings.KEYS.CVE_BASE_URL + i); retrieveUrl = Settings.getString(Settings.KEYS.CVE_BASE_URL + Settings.KEYS.CVE_SCHEMA_2_0 + i);
String key = Integer.toString(i); item = new NvdCveUrl();
cve = createNvdCveUrl(key, retrieveUrl, html); item.id = Integer.toString(i);
map.put(key, cve); item.url = retrieveUrl;
} item.timestamp = Downloader.getLastModified(new URL(retrieveUrl));
} catch (IOException ex) { map.put(item.id, item);
throw new DownloadFailedException("Unable to create temporary file for NVD CVE Meta File download.", ex);
} finally {
try {
if (tmp != null && tmp.exists()) {
tmp.delete();
}
} finally {
if (tmp != null && tmp.exists()) {
tmp.deleteOnExit();
}
}
} }
return map; return map;
} }
/** //<editor-fold defaultstate="collapsed" desc="old dead code">
* Creates a new NvdCveUrl object from the provide id, url, and text/html // /**
* from the NVD CVE downloads page. // * Retrieves the timestamps from the NVD CVE meta data file.
* // *
* @param id the name of this NVD CVE Url // * @return the timestamp from the currently published nvdcve downloads page
* @param retrieveUrl the URL to download the file from // * @throws MalformedURLException is thrown if the URL for the NVD CCE Meta
* @param text a bit of HTML from the NVD CVE downloads page that contains // * data is incorrect.
* the URL and the last updated timestamp. // * @throws DownloadFailedException is thrown if there is an error
* @return a shiny new NvdCveUrl object. // * downloading the nvd cve meta data file
* @throws InvalidDataException is thrown if the timestamp could not be // * @throws InvalidDataException is thrown if there is an exception parsing
* extracted from the provided text. // * the timestamps
*/ // */
private NvdCveUrl createNvdCveUrl(String id, String retrieveUrl, String text) throws InvalidDataException { // protected Map<String, NvdCveUrl> retrieveCurrentTimestampsFromWeb() throws MalformedURLException, DownloadFailedException, InvalidDataException {
Pattern pattern = Pattern.compile(Pattern.quote(retrieveUrl) + ".+?\\<br"); // Map<String, NvdCveUrl> map = new HashMap<String, NvdCveUrl>();
Matcher m = pattern.matcher(text); //
NvdCveUrl item = new NvdCveUrl(); // File tmp = null;
item.id = id; // try {
item.url = retrieveUrl; // tmp = File.createTempFile("cve", "meta");
if (m.find()) { // URL url = new URL(Settings.getString(Settings.KEYS.CVE_META_URL));
String line = m.group(); // Downloader.fetchFile(url, tmp);
int pos = line.indexOf("Updated:"); // String html = readFile(tmp);
if (pos > 0) { //
pos += 9; // String retrieveUrl = Settings.getString(Settings.KEYS.CVE_MODIFIED_20_URL);
try { // NvdCveUrl cve = createNvdCveUrl("modified", retrieveUrl, html);
String timestampstr = line.substring(pos, line.length() - 3).replace("at ", ""); // cve.setNeedsUpdate(false); //the others default to true, to make life easier later this should default to false.
long timestamp = getEpochTimeFromDateTime(timestampstr); // map.put("modified", cve);
item.setTimestamp(timestamp); // int max = Settings.getInt(Settings.KEYS.CVE_URL_COUNT);
} catch (NumberFormatException ex) { // for (int i = 1; i <= max; i++) {
throw new InvalidDataException("NVD CVE Meta file does not contain a valid timestamp for '" + retrieveUrl + "'.", ex); // retrieveUrl = Settings.getString(Settings.KEYS.CVE_BASE_URL + Settings.KEYS.CVE_SCHEMA_2_0 + i);
} // String key = Integer.toString(i);
} else { // cve = createNvdCveUrl(key, retrieveUrl, html);
throw new InvalidDataException("NVD CVE Meta file does not contain the updated timestamp for '" + retrieveUrl + "'."); // map.put(key, cve);
} // }
} else { // } catch (IOException ex) {
throw new InvalidDataException("NVD CVE Meta file does not contain the url for '" + retrieveUrl + "'."); // throw new DownloadFailedException("Unable to create temporary file for NVD CVE Meta File download.", ex);
} // } finally {
return item; // try {
} // if (tmp != null && tmp.exists()) {
// tmp.delete();
/** // }
* Parses a timestamp in the format of "MM/dd/yy hh:mm" into a calendar // } finally {
* object and returns the epoch time. Note, this removes the millisecond // if (tmp != null && tmp.exists()) {
* portion of the epoch time so all numbers returned should end in 000. // tmp.deleteOnExit();
* // }
* @param timestamp a string in the format of "MM/dd/yy hh:mm" // }
* @return a Calendar object. // }
* @throws NumberFormatException if the timestamp was parsed incorrectly. // return map;
*/ // }
private long getEpochTimeFromDateTime(String timestamp) throws NumberFormatException { //
Calendar c = new GregorianCalendar(); // /**
int month = Integer.parseInt(timestamp.substring(0, 2)); // * Creates a new NvdCveUrl object from the provide id, url, and text/html
int date = Integer.parseInt(timestamp.substring(3, 5)); // * from the NVD CVE downloads page.
int year = 2000 + Integer.parseInt(timestamp.substring(6, 8)); // *
int hourOfDay = Integer.parseInt(timestamp.substring(9, 11)); // * @param id the name of this NVD CVE Url
int minute = Integer.parseInt(timestamp.substring(12, 14)); // * @param retrieveUrl the URL to download the file from
c.set(year, month, date, hourOfDay, minute, 0); // * @param text a bit of HTML from the NVD CVE downloads page that contains
long t = c.getTimeInMillis(); // * the URL and the last updated timestamp.
t = (t / 1000) * 1000; // * @return a shiny new NvdCveUrl object.
return t; // * @throws InvalidDataException is thrown if the timestamp could not be
} // * extracted from the provided text.
// */
/** // private NvdCveUrl createNvdCveUrl(String id, String retrieveUrl, String text) throws InvalidDataException {
* Reads a file into a string. // Pattern pattern = Pattern.compile(Pattern.quote(retrieveUrl) + ".+?\\<br");
* // Matcher m = pattern.matcher(text);
* @param file the file to be read. // NvdCveUrl item = new NvdCveUrl();
* @return the contents of the file. // item.id = id;
* @throws IOException is thrown if an IOExcpetion occurs. // item.url = retrieveUrl;
*/ // if (m.find()) {
private String readFile(File file) throws IOException { // String line = m.group();
InputStreamReader stream = new InputStreamReader(new FileInputStream(file), "UTF-8"); // int pos = line.indexOf("Updated:");
StringBuilder str = new StringBuilder((int) file.length()); // if (pos > 0) {
try { // pos += 9;
char[] buf = new char[8096]; // try {
int read = stream.read(buf, 0, 8096); // String timestampstr = line.substring(pos, line.length() - 3).replace("at ", "");
while (read > 0) { // long timestamp = getEpochTimeFromDateTime(timestampstr);
str.append(buf, 0, read); // item.setTimestamp(timestamp);
read = stream.read(buf, 0, 8096); // } catch (NumberFormatException ex) {
} // throw new InvalidDataException("NVD CVE Meta file does not contain a valid timestamp for '" + retrieveUrl + "'.", ex);
} finally { // }
stream.close(); // } else {
} // throw new InvalidDataException("NVD CVE Meta file does not contain the updated timestamp for '" + retrieveUrl + "'.");
return str.toString(); // }
} // } else {
// throw new InvalidDataException("NVD CVE Meta file does not contain the url for '" + retrieveUrl + "'.");
// }
// return item;
// }
//
// /**
// * Parses a timestamp in the format of "MM/dd/yy hh:mm" into a calendar
// * object and returns the epoch time. Note, this removes the millisecond
// * portion of the epoch time so all numbers returned should end in 000.
// *
// * @param timestamp a string in the format of "MM/dd/yy hh:mm"
// * @return a Calendar object.
// * @throws NumberFormatException if the timestamp was parsed incorrectly.
// */
// private long getEpochTimeFromDateTime(String timestamp) throws NumberFormatException {
// Calendar c = new GregorianCalendar();
// int month = Integer.parseInt(timestamp.substring(0, 2));
// int date = Integer.parseInt(timestamp.substring(3, 5));
// int year = 2000 + Integer.parseInt(timestamp.substring(6, 8));
// int hourOfDay = Integer.parseInt(timestamp.substring(9, 11));
// int minute = Integer.parseInt(timestamp.substring(12, 14));
// c.set(year, month, date, hourOfDay, minute, 0);
// long t = c.getTimeInMillis();
// t = (t / 1000) * 1000;
// return t;
// }
//
// /**
// * Reads a file into a string.
// *
// * @param file the file to be read.
// * @return the contents of the file.
// * @throws IOException is thrown if an IOExcpetion occurs.
// */
// private String readFile(File file) throws IOException {
// InputStreamReader stream = new InputStreamReader(new FileInputStream(file), "UTF-8");
// StringBuilder str = new StringBuilder((int) file.length());
// try {
// char[] buf = new char[8096];
// int read = stream.read(buf, 0, 8096);
// while (read > 0) {
// str.append(buf, 0, read);
// read = stream.read(buf, 0, 8096);
// }
// } finally {
// stream.close();
// }
// return str.toString();
// }
//</editor-fold>
/** /**
* A pojo that contains the Url and timestamp of the current NvdCve XML * A pojo that contains the Url and timestamp of the current NvdCve XML
* files. * files.
*/ */
protected class NvdCveUrl { protected static class NvdCveUrl {
/** /**
* an id. * an id.

84
src/main/java/org/codesecure/dependencycheck/utils/Downloader.java
View File

@@ -97,22 +97,8 @@ public class Downloader {
*/ */
public static void fetchFile(URL url, File outputPath, boolean unzip) throws DownloadFailedException { public static void fetchFile(URL url, File outputPath, boolean unzip) throws DownloadFailedException {
HttpURLConnection conn = null; HttpURLConnection conn = null;
Proxy proxy = null;
String proxyUrl = Settings.getString(Settings.KEYS.PROXY_URL);
try { try {
if (proxyUrl != null) { conn = Downloader.getConnection(url);
int proxyPort = Settings.getInt(Settings.KEYS.PROXY_PORT);
SocketAddress addr = new InetSocketAddress(proxyUrl, proxyPort);
proxy = new Proxy(Proxy.Type.HTTP, addr);
conn = (HttpURLConnection) url.openConnection(proxy);
} else {
conn = (HttpURLConnection) url.openConnection();
}
if (Settings.getString(Settings.KEYS.CONNECTION_TIMEOUT) != null) {
int timeout = Settings.getInt(Settings.KEYS.CONNECTION_TIMEOUT);
conn.setConnectTimeout(timeout);
}
conn.setRequestProperty("Accept-Encoding", "gzip, deflate"); conn.setRequestProperty("Accept-Encoding", "gzip, deflate");
conn.connect(); conn.connect();
} catch (IOException ex) { } catch (IOException ex) {
@@ -173,4 +159,72 @@ public class Downloader {
} }
} }
} }
/**
* Makes an HTTP Head request to retrieve the last modified date of the given URL.
*
* @param url the URL to retrieve the timestamp from
* @return an epoch timestamp
* @throws DownloadFailedException is thrown if an exception occurs making the HTTP request
*/
public static long getLastModified(URL url) throws DownloadFailedException {
HttpURLConnection conn = null;
long timestamp = 0;
try {
conn = Downloader.getConnection(url);
conn.setRequestMethod("HEAD");
conn.connect();
timestamp = conn.getLastModified();
} catch (Exception ex) {
throw new DownloadFailedException("Error making HTTP HEAD request.", ex);
} finally {
if (conn != null) {
try {
conn.disconnect();
} finally {
conn = null;
}
}
}
return timestamp;
}
/**
* Utility method to get an HttpURLConnectoin. If the app is configured to
* use a proxy this method will retrieve the proxy settings and use them
* when setting up the connection.
*
* @param url the url to connect to
* @return an HttpURLConnection
* @throws DownloadFailedException thrown if there is an exception
*/
private static HttpURLConnection getConnection(URL url) throws DownloadFailedException {
HttpURLConnection conn = null;
Proxy proxy = null;
String proxyUrl = Settings.getString(Settings.KEYS.PROXY_URL);
try {
if (proxyUrl != null) {
int proxyPort = Settings.getInt(Settings.KEYS.PROXY_PORT);
SocketAddress addr = new InetSocketAddress(proxyUrl, proxyPort);
proxy = new Proxy(Proxy.Type.HTTP, addr);
conn = (HttpURLConnection) url.openConnection(proxy);
} else {
conn = (HttpURLConnection) url.openConnection();
}
if (Settings.getString(Settings.KEYS.CONNECTION_TIMEOUT) != null) {
int timeout = Settings.getInt(Settings.KEYS.CONNECTION_TIMEOUT);
conn.setConnectTimeout(timeout);
}
} catch (IOException ex) {
try {
if (conn != null) {
conn.disconnect();
}
} finally {
conn = null;
}
throw new DownloadFailedException("Error getting connection.", ex);
}
return conn;
}
} }

23
src/main/java/org/codesecure/dependencycheck/utils/Settings.java
View File

@@ -70,9 +70,14 @@ public class Settings {
public static final String CVE_META_URL = "cve.url.meta"; public static final String CVE_META_URL = "cve.url.meta";
/** /**
* The properties key for the URL to retrieve the recently modified and * The properties key for the URL to retrieve the recently modified and
* added CVE entries (last 8 days). * added CVE entries (last 8 days) using the 2.0 schema.
*/ */
public static final String CVE_MODIFIED_URL = "cve.url.modified"; public static final String CVE_MODIFIED_20_URL = "cve.url-2.0.modified";
/**
* The properties key for the URL to retrieve the recently modified and
* added CVE entries (last 8 days) using the 1.2 schema.
*/
public static final String CVE_MODIFIED_12_URL = "cve.url-1.2.modified";
/** /**
* The properties key for the URL to retrieve the recently modified and * The properties key for the URL to retrieve the recently modified and
* added CVE entries (last 8 days). * added CVE entries (last 8 days).
@@ -86,9 +91,19 @@ public class Settings {
public static final String CVE_URL_COUNT = "cve.url.count"; public static final String CVE_URL_COUNT = "cve.url.count";
/** /**
* The properties key for the "base" property key for the CVE URLs (e.g. * The properties key for the "base" property key for the CVE URLs (e.g.
* cve.url.1, cve.url.2, cve.url.n). * cve.url-2.0.1, cve.url-1.2.2, cve.url.n).
*/ */
public static final String CVE_BASE_URL = "cve.url."; public static final String CVE_BASE_URL = "cve.url-";
/**
* The properties key for the CVE schema version 1.2
*/
public static final String CVE_SCHEMA_1_2 = "1.2.";
/**
* The properties key for the CVE schema version 2.0
*/
public static final String CVE_SCHEMA_2_0 = "2.0.";
/** /**
* The properties key for the proxy url. * The properties key for the proxy url.
*/ */

45
src/main/resources/configuration/dependencycheck.properties
View File

@@ -7,12 +7,15 @@ cpe=data/cpe
cpe.url=http://static.nvd.nist.gov/feeds/xml/cpe/dictionary/official-cpe-dictionary_v2.2.xml.gz cpe.url=http://static.nvd.nist.gov/feeds/xml/cpe/dictionary/official-cpe-dictionary_v2.2.xml.gz
# the path to the cpe meta data file. # the path to the cpe meta data file.
cpe.meta.url=http://static.nvd.nist.gov/feeds/xml/cpe/dictionary/official-cpe-dictionary_v2.2.meta cpe.meta.url=http://static.nvd.nist.gov/feeds/xml/cpe/dictionary/official-cpe-dictionary_v2.2.meta
# the path to the lucene index to store the nvd cve data # the path to the lucene index to store the nvd cve data
cve=data/cve cve=data/cve
# the path to the nvd cve "meta" page where the timestamps for the last update files can be found. # the path to the nvd cve "meta" page where the timestamps for the last update files can be found.
cve.url.meta=http://nvd.nist.gov/download.cfm #cve.url.meta=http://nvd.nist.gov/download.cfm
# the path to the modified nvd cve xml file. # the path to the modified nvd cve xml file.
cve.url.modified=http://static.nvd.nist.gov/feeds/xml/cve/nvdcve-2.0-modified.xml cve.url-2.0.modified=http://static.nvd.nist.gov/feeds/xml/cve/nvdcve-2.0-modified.xml
# the number of days that the modified nvd cve data holds data for. We don't need # the number of days that the modified nvd cve data holds data for. We don't need
# to update the other files if we are within this timespan. Per NIST this file # to update the other files if we are within this timespan. Per NIST this file
@@ -20,15 +23,29 @@ cve.url.modified=http://static.nvd.nist.gov/feeds/xml/cve/nvdcve-2.0-modified.xm
cve.url.modified.validfordays=7 cve.url.modified.validfordays=7
# the number of cve.urls # the number of cve.urls
cve.url.count=11 cve.url.count=11
# the paths to the various nvd cve files. # the paths to the various nvd cve files (schema version 2.0)
cve.url.1=http://static.nvd.nist.gov/feeds/xml/cve/nvdcve-2.0-2002.xml cve.url-2.0.1=http://static.nvd.nist.gov/feeds/xml/cve/nvdcve-2.0-2002.xml
cve.url.2=http://static.nvd.nist.gov/feeds/xml/cve/nvdcve-2.0-2003.xml cve.url-2.0.2=http://static.nvd.nist.gov/feeds/xml/cve/nvdcve-2.0-2003.xml
cve.url.3=http://static.nvd.nist.gov/feeds/xml/cve/nvdcve-2.0-2004.xml cve.url-2.0.3=http://static.nvd.nist.gov/feeds/xml/cve/nvdcve-2.0-2004.xml
cve.url.4=http://static.nvd.nist.gov/feeds/xml/cve/nvdcve-2.0-2005.xml cve.url-2.0.4=http://static.nvd.nist.gov/feeds/xml/cve/nvdcve-2.0-2005.xml
cve.url.5=http://static.nvd.nist.gov/feeds/xml/cve/nvdcve-2.0-2006.xml cve.url-2.0.5=http://static.nvd.nist.gov/feeds/xml/cve/nvdcve-2.0-2006.xml
cve.url.6=http://static.nvd.nist.gov/feeds/xml/cve/nvdcve-2.0-2007.xml cve.url-2.0.6=http://static.nvd.nist.gov/feeds/xml/cve/nvdcve-2.0-2007.xml
cve.url.7=http://static.nvd.nist.gov/feeds/xml/cve/nvdcve-2.0-2008.xml cve.url-2.0.7=http://static.nvd.nist.gov/feeds/xml/cve/nvdcve-2.0-2008.xml
cve.url.8=http://static.nvd.nist.gov/feeds/xml/cve/nvdcve-2.0-2009.xml cve.url-2.0.8=http://static.nvd.nist.gov/feeds/xml/cve/nvdcve-2.0-2009.xml
cve.url.9=http://static.nvd.nist.gov/feeds/xml/cve/nvdcve-2.0-2010.xml cve.url-2.0.9=http://static.nvd.nist.gov/feeds/xml/cve/nvdcve-2.0-2010.xml
cve.url.10=http://static.nvd.nist.gov/feeds/xml/cve/nvdcve-2.0-2011.xml cve.url-2.0.10=http://static.nvd.nist.gov/feeds/xml/cve/nvdcve-2.0-2011.xml
cve.url.11=http://static.nvd.nist.gov/feeds/xml/cve/nvdcve-2.0-2012.xml cve.url-2.0.11=http://static.nvd.nist.gov/feeds/xml/cve/nvdcve-2.0-2012.xml
# the paths to the various nvd cve files (schema version 1.2).
cve.url-1.2.modified=http://nvd.nist.gov/download/nvdcve-modified.xml
cve.url-1.2.1=http://nvd.nist.gov/download/nvdcve-2002.xml
cve.url-1.2.2=http://nvd.nist.gov/download/nvdcve-2003.xml
cve.url-1.2.3=http://nvd.nist.gov/download/nvdcve-2004.xml
cve.url-1.2.4=http://nvd.nist.gov/download/nvdcve-2005.xml
cve.url-1.2.5=http://nvd.nist.gov/download/nvdcve-2006.xml
cve.url-1.2.6=http://nvd.nist.gov/download/nvdcve-2007.xml
cve.url-1.2.7=http://nvd.nist.gov/download/nvdcve-2008.xml
cve.url-1.2.8=http://nvd.nist.gov/download/nvdcve-2009.xml
cve.url-1.2.9=http://nvd.nist.gov/download/nvdcve-2010.xml
cve.url-1.2.10=http://nvd.nist.gov/download/nvdcve-2011.xml
cve.url-1.2.11=http://nvd.nist.gov/download/nvdcve-2012.xml

2
src/test/java/org/codesecure/dependencycheck/dependency/DependencyTest.java
View File

@@ -5,8 +5,6 @@
package org.codesecure.dependencycheck.dependency; package org.codesecure.dependencycheck.dependency;
import java.io.File; import java.io.File;
import org.codesecure.dependencycheck.dependency.Dependency;
import org.codesecure.dependencycheck.dependency.Evidence;
import java.util.List; import java.util.List;
import org.junit.After; import org.junit.After;
import org.junit.AfterClass; import org.junit.AfterClass;

9
src/test/java/org/codesecure/dependencycheck/utils/DownloaderIntegrationTest.java
View File

@@ -10,6 +10,7 @@ import org.junit.AfterClass;
import org.junit.Before; import org.junit.Before;
import org.junit.BeforeClass; import org.junit.BeforeClass;
import org.junit.Test; import org.junit.Test;
import static org.junit.Assert.*;
/** /**
* *
@@ -58,4 +59,12 @@ public class DownloaderIntegrationTest {
Downloader.fetchFile(url, outputPath, false); Downloader.fetchFile(url, outputPath, false);
} }
@Test
public void testGetLastModified() throws Exception {
System.out.println("getLastModified");
URL url = new URL("http://nvd.nist.gov/download/nvdcve-2012.xml");
long timestamp = Downloader.getLastModified(url);
assertTrue("timestamp equal to zero?", timestamp>0);
}
} }

1
src/test/java/org/codesecure/dependencycheck/utils/SettingsTest.java
View File

@@ -6,7 +6,6 @@ package org.codesecure.dependencycheck.utils;
import java.io.File; import java.io.File;
import java.io.IOException; import java.io.IOException;
import java.io.InputStream;
import java.net.URISyntaxException; import java.net.URISyntaxException;
import junit.framework.TestCase; import junit.framework.TestCase;
import org.junit.Test; import org.junit.Test;