diff --git a/src/main/java/org/codesecure/dependencycheck/data/cpe/xml/CPEHandler.java b/src/main/java/org/codesecure/dependencycheck/data/cpe/xml/CPEHandler.java deleted file mode 100644 index 076bf406c..000000000 --- a/src/main/java/org/codesecure/dependencycheck/data/cpe/xml/CPEHandler.java +++ /dev/null @@ -1,351 +0,0 @@ -package org.codesecure.dependencycheck.data.cpe.xml; -/* - * This file is part of DependencyCheck. - * - * DependencyCheck is free software: you can redistribute it and/or modify it - * under the terms of the GNU General Public License as published by the Free - * Software Foundation, either version 3 of the License, or (at your option) any - * later version. - * - * DependencyCheck is distributed in the hope that it will be useful, but - * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more - * details. - * - * You should have received a copy of the GNU General Public License along with - * DependencyCheck. If not, see http://www.gnu.org/licenses/. - * - * Copyright (c) 2012 Jeremy Long. All Rights Reserved. - */ - -import java.io.IOException; -import java.io.UnsupportedEncodingException; -import java.text.ParseException; -import java.util.logging.Level; -import java.util.logging.Logger; -import org.apache.lucene.index.CorruptIndexException; -import org.codesecure.dependencycheck.data.cpe.Entry; -import org.xml.sax.Attributes; -import org.xml.sax.SAXException; -import org.xml.sax.helpers.DefaultHandler; - -/** - * A SAX Handler that will parse the CPE XML Listing. - * - * @author Jeremy Long (jeremy.long@gmail.com) - */ -public class CPEHandler extends DefaultHandler { - - private static final String CURRENT_SCHEMA_VERSION = "2.2"; - EntrySaveDelegate saveDelegate = null; - Entry entry = null; - boolean languageIsUS = false; - StringBuilder nodeText = null; - boolean skip = false; - Element current = new Element(); - - /** - * Register a EntrySaveDelegate object. When the last node of an entry is - * reached if a save delegate has been registered the save method will be - * invoked. - * - * @param delegate the delegate used to save an entry - */ - public void registerSaveDelegate(EntrySaveDelegate delegate) { - this.saveDelegate = delegate; - } - - @Override - public void startElement(String uri, String localName, String qName, Attributes attributes) throws SAXException { - nodeText = null; - current.setNode(qName); - if (current.isCpeItemNode()) { - entry = new Entry(); - String temp = attributes.getValue("deprecated"); - String name = attributes.getValue("name"); - skip = (temp != null && temp.equals("true")); - try { - if (!skip && name.startsWith("cpe:/a:")) { - entry.parseName(name); - } else { - skip = true; - } - } catch (UnsupportedEncodingException ex) { - throw new SAXException(ex); - } - } else if (current.isTitleNode()) { - nodeText = new StringBuilder(100); - if ("en-US".equalsIgnoreCase(attributes.getValue("xml:lang"))) { - languageIsUS = true; - } else { - languageIsUS = false; - } - } else if (current.isMetaNode()) { - try { - entry.setModificationDate(attributes.getValue("modification-date")); - } catch (ParseException ex) { - Logger.getLogger(CPEHandler.class.getName()).log(Level.SEVERE, null, ex); - } - entry.setStatus(attributes.getValue("status")); - entry.setNvdId(attributes.getValue("nvd-id")); - } else if (current.isSchemaVersionNode()) { - nodeText = new StringBuilder(3); - } else if (current.isTimestampNode()) { - nodeText = new StringBuilder(24); - } -// } else if (current.isCpeListNode()) { -// //do nothing -// } else if (current.isNotesNode()) { -// //do nothing -// } else if (current.isNoteNode()) { -// //do nothing -// } else if (current.isCheckNode()) { -// //do nothing -// } else if (current.isGeneratorNode()) { -// //do nothing -// } else if (current.isProductNameNode()) { -// //do nothing -// } else if (current.isProductVersionNode()) { -// //do nothing - } - - @Override - public void characters(char[] ch, int start, int length) throws SAXException { - //nodeText += new String(ch, start, length); - if (nodeText != null) { - nodeText.append(ch, start, length); - } - } - - @Override - public void endElement(String uri, String localName, String qName) throws SAXException { - current.setNode(qName); - if (current.isCpeItemNode()) { - if (saveDelegate != null && !skip) { - try { - saveDelegate.saveEntry(entry); - } catch (CorruptIndexException ex) { - Logger.getLogger(CPEHandler.class.getName()).log(Level.SEVERE, null, ex); - throw new SAXException(ex); - } catch (IOException ex) { - Logger.getLogger(CPEHandler.class.getName()).log(Level.SEVERE, null, ex); - throw new SAXException(ex); - } - entry = null; - } - } else if (current.isTitleNode()) { - if (languageIsUS) { - entry.setTitle(nodeText.toString()); - } - } else if (current.isSchemaVersionNode() && !CURRENT_SCHEMA_VERSION.equals(nodeText.toString())) { - throw new SAXException("ERROR: Invalid Schema Version, expected: " - + CURRENT_SCHEMA_VERSION + ", file is: " + nodeText); - } -// } else if (current.isCpeListNode()) { -// //do nothing -// } else if (current.isMetaNode()) { -// //do nothing -// } else if (current.isNotesNode()) { -// //do nothing -// } else if (current.isNoteNode()) { -// //do nothing -// } else if (current.isCheckNode()) { -// //do nothing -// } else if (current.isGeneratorNode()) { -// //do nothing -// } else if (current.isProductNameNode()) { -// //do nothing -// } else if (current.isProductVersionNode()) { -// //do nothing -// else if (current.isTimestampNode()) { -// //do nothing -// } else { -// throw new SAXException("ERROR STATE: Unexpected qName '" + qName + "'"); -// } - } - - // - /** - * A simple class to maintain information about the current element while - * parsing the CPE XML. - */ - protected class Element { - - /** - * A node type in the CPE Schema 2.2 - */ - public static final String CPE_LIST = "cpe-list"; - /** - * A node type in the CPE Schema 2.2 - */ - public static final String CPE_ITEM = "cpe-item"; - /** - * A node type in the CPE Schema 2.2 - */ - public static final String TITLE = "title"; - /** - * A node type in the CPE Schema 2.2 - */ - public static final String NOTES = "notes"; - /** - * A node type in the CPE Schema 2.2 - */ - public static final String NOTE = "note"; - /** - * A node type in the CPE Schema 2.2 - */ - public static final String CHECK = "check"; - /** - * A node type in the CPE Schema 2.2 - */ - public static final String META = "meta:item-metadata"; - /** - * A node type in the CPE Schema 2.2 - */ - public static final String GENERATOR = "generator"; - /** - * A node type in the CPE Schema 2.2 - */ - public static final String PRODUCT_NAME = "product_name"; - /** - * A node type in the CPE Schema 2.2 - */ - public static final String PRODUCT_VERSION = "product_version"; - /** - * A node type in the CPE Schema 2.2 - */ - public static final String SCHEMA_VERSION = "schema_version"; - /** - * A node type in the CPE Schema 2.2 - */ - public static final String TIMESTAMP = "timestamp"; - private String node = null; - - /** - * Gets the value of node - * - * @return the value of node - */ - public String getNode() { - return this.node; - } - - /** - * Sets the value of node - * - * @param node new value of node - */ - public void setNode(String node) { - this.node = node; - } - - /** - * Checks if the handler is at the CPE_LIST node - * - * @return true or false - */ - public boolean isCpeListNode() { - return CPE_LIST.equals(node); - } - - /** - * Checks if the handler is at the CPE_ITEM node - * - * @return true or false - */ - public boolean isCpeItemNode() { - return CPE_ITEM.equals(node); - } - - /** - * Checks if the handler is at the TITLE node - * - * @return true or false - */ - public boolean isTitleNode() { - return TITLE.equals(node); - } - - /** - * Checks if the handler is at the NOTES node - * - * @return true or false - */ - public boolean isNotesNode() { - return NOTES.equals(node); - } - - /** - * Checks if the handler is at the NOTE node - * - * @return true or false - */ - public boolean isNoteNode() { - return NOTE.equals(node); - } - - /** - * Checks if the handler is at the CHECK node - * - * @return true or false - */ - public boolean isCheckNode() { - return CHECK.equals(node); - } - - /** - * Checks if the handler is at the META node - * - * @return true or false - */ - public boolean isMetaNode() { - return META.equals(node); - } - - /** - * Checks if the handler is at the GENERATOR node - * - * @return true or false - */ - public boolean isGeneratorNode() { - return GENERATOR.equals(node); - } - - /** - * Checks if the handler is at the PRODUCT_NAME node - * - * @return true or false - */ - public boolean isProductNameNode() { - return PRODUCT_NAME.equals(node); - } - - /** - * Checks if the handler is at the PRODUCT_VERSION node - * - * @return true or false - */ - public boolean isProductVersionNode() { - return PRODUCT_VERSION.equals(node); - } - - /** - * Checks if the handler is at the SCHEMA_VERSION node - * - * @return true or false - */ - public boolean isSchemaVersionNode() { - return SCHEMA_VERSION.equals(node); - } - - /** - * Checks if the handler is at the TIMESTAMP node - * - * @return true or false - */ - public boolean isTimestampNode() { - return TIMESTAMP.equals(node); - } - } - // -} diff --git a/src/main/java/org/codesecure/dependencycheck/data/nvdcve/xml/IndexUpdater.java b/src/main/java/org/codesecure/dependencycheck/data/nvdcve/xml/IndexUpdater.java index 17552b852..db8f10983 100644 --- a/src/main/java/org/codesecure/dependencycheck/data/nvdcve/xml/IndexUpdater.java +++ b/src/main/java/org/codesecure/dependencycheck/data/nvdcve/xml/IndexUpdater.java @@ -24,28 +24,24 @@ import java.io.FileNotFoundException; import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; -import java.io.InputStreamReader; import java.io.OutputStream; import java.io.OutputStreamWriter; import org.codesecure.dependencycheck.data.CachedWebDataSource; import java.net.MalformedURLException; import java.net.URL; -import java.util.Calendar; import java.util.Date; -import java.util.GregorianCalendar; import java.util.HashMap; import java.util.Map; import java.util.Properties; import java.util.logging.Level; import java.util.logging.Logger; -import java.util.regex.Matcher; -import java.util.regex.Pattern; import org.apache.lucene.index.CorruptIndexException; import org.codesecure.dependencycheck.data.nvdcve.Index; import org.codesecure.dependencycheck.data.UpdateException; import org.codesecure.dependencycheck.utils.DownloadFailedException; import org.codesecure.dependencycheck.utils.Downloader; import org.codesecure.dependencycheck.utils.FileUtils; +import org.codesecure.dependencycheck.utils.InvalidSettingException; import org.codesecure.dependencycheck.utils.Settings; /** @@ -283,7 +279,12 @@ public class IndexUpdater extends Index implements CachedWebDataSource { } catch (InvalidDataException ex) { Logger.getLogger(IndexUpdater.class.getName()).log(Level.SEVERE, null, ex); throw new DownloadFailedException("Unable to retrieve valid timestamp from nvd cve downloads page", ex); + + } catch (InvalidSettingException ex) { + Logger.getLogger(IndexUpdater.class.getName()).log(Level.SEVERE, null, ex); + throw new DownloadFailedException("Invalid settings", ex); } + if (currentlyPublished == null) { throw new DownloadFailedException("Unable to retrieve valid timestamp from nvd cve downloads page"); } @@ -401,139 +402,176 @@ public class IndexUpdater extends Index implements CachedWebDataSource { * Retrieves the timestamps from the NVD CVE meta data file. * * @return the timestamp from the currently published nvdcve downloads page - * @throws MalformedURLException is thrown if the URL for the NVD CCE Meta + * @throws MalformedURLException thrown if the URL for the NVD CCE Meta * data is incorrect. - * @throws DownloadFailedException is thrown if there is an error + * @throws DownloadFailedException thrown if there is an error * downloading the nvd cve meta data file - * @throws InvalidDataException is thrown if there is an exception parsing + * @throws InvalidDataException thrown if there is an exception parsing * the timestamps + * @throws InvalidSettingException thrown if the settings are invalid */ - protected Map retrieveCurrentTimestampsFromWeb() throws MalformedURLException, DownloadFailedException, InvalidDataException { + protected Map retrieveCurrentTimestampsFromWeb() + throws MalformedURLException, DownloadFailedException, InvalidDataException, InvalidSettingException { + Map map = new HashMap(); + String retrieveUrl = Settings.getString(Settings.KEYS.CVE_MODIFIED_20_URL); - File tmp = null; - try { - tmp = File.createTempFile("cve", "meta"); - URL url = new URL(Settings.getString(Settings.KEYS.CVE_META_URL)); - Downloader.fetchFile(url, tmp); - String html = readFile(tmp); + NvdCveUrl item = new NvdCveUrl(); + item.setNeedsUpdate(false); //the others default to true, to make life easier later this should default to false. + item.id = "modified"; + item.timestamp = Downloader.getLastModified(new URL(retrieveUrl)); + map.put("modified", item); - String retrieveUrl = Settings.getString(Settings.KEYS.CVE_MODIFIED_URL); - NvdCveUrl cve = createNvdCveUrl("modified", retrieveUrl, html); - cve.setNeedsUpdate(false); //the others default to true, to make life easier later this should default to false. - map.put("modified", cve); - int max = Settings.getInt(Settings.KEYS.CVE_URL_COUNT); - for (int i = 1; i <= max; i++) { - retrieveUrl = Settings.getString(Settings.KEYS.CVE_BASE_URL + i); - String key = Integer.toString(i); - cve = createNvdCveUrl(key, retrieveUrl, html); - map.put(key, cve); - } - } catch (IOException ex) { - throw new DownloadFailedException("Unable to create temporary file for NVD CVE Meta File download.", ex); - } finally { - try { - if (tmp != null && tmp.exists()) { - tmp.delete(); - } - } finally { - if (tmp != null && tmp.exists()) { - tmp.deleteOnExit(); - } - } + int max = Settings.getInt(Settings.KEYS.CVE_URL_COUNT); + for (int i = 1; i <= max; i++) { + retrieveUrl = Settings.getString(Settings.KEYS.CVE_BASE_URL + Settings.KEYS.CVE_SCHEMA_2_0 + i); + item = new NvdCveUrl(); + item.id = Integer.toString(i); + item.url = retrieveUrl; + item.timestamp = Downloader.getLastModified(new URL(retrieveUrl)); + map.put(item.id, item); } return map; } - /** - * Creates a new NvdCveUrl object from the provide id, url, and text/html - * from the NVD CVE downloads page. - * - * @param id the name of this NVD CVE Url - * @param retrieveUrl the URL to download the file from - * @param text a bit of HTML from the NVD CVE downloads page that contains - * the URL and the last updated timestamp. - * @return a shiny new NvdCveUrl object. - * @throws InvalidDataException is thrown if the timestamp could not be - * extracted from the provided text. - */ - private NvdCveUrl createNvdCveUrl(String id, String retrieveUrl, String text) throws InvalidDataException { - Pattern pattern = Pattern.compile(Pattern.quote(retrieveUrl) + ".+?\\ 0) { - pos += 9; - try { - String timestampstr = line.substring(pos, line.length() - 3).replace("at ", ""); - long timestamp = getEpochTimeFromDateTime(timestampstr); - item.setTimestamp(timestamp); - } catch (NumberFormatException ex) { - throw new InvalidDataException("NVD CVE Meta file does not contain a valid timestamp for '" + retrieveUrl + "'.", ex); - } - } else { - throw new InvalidDataException("NVD CVE Meta file does not contain the updated timestamp for '" + retrieveUrl + "'."); - } - } else { - throw new InvalidDataException("NVD CVE Meta file does not contain the url for '" + retrieveUrl + "'."); - } - return item; - } - - /** - * Parses a timestamp in the format of "MM/dd/yy hh:mm" into a calendar - * object and returns the epoch time. Note, this removes the millisecond - * portion of the epoch time so all numbers returned should end in 000. - * - * @param timestamp a string in the format of "MM/dd/yy hh:mm" - * @return a Calendar object. - * @throws NumberFormatException if the timestamp was parsed incorrectly. - */ - private long getEpochTimeFromDateTime(String timestamp) throws NumberFormatException { - Calendar c = new GregorianCalendar(); - int month = Integer.parseInt(timestamp.substring(0, 2)); - int date = Integer.parseInt(timestamp.substring(3, 5)); - int year = 2000 + Integer.parseInt(timestamp.substring(6, 8)); - int hourOfDay = Integer.parseInt(timestamp.substring(9, 11)); - int minute = Integer.parseInt(timestamp.substring(12, 14)); - c.set(year, month, date, hourOfDay, minute, 0); - long t = c.getTimeInMillis(); - t = (t / 1000) * 1000; - return t; - } - - /** - * Reads a file into a string. - * - * @param file the file to be read. - * @return the contents of the file. - * @throws IOException is thrown if an IOExcpetion occurs. - */ - private String readFile(File file) throws IOException { - InputStreamReader stream = new InputStreamReader(new FileInputStream(file), "UTF-8"); - StringBuilder str = new StringBuilder((int) file.length()); - try { - char[] buf = new char[8096]; - int read = stream.read(buf, 0, 8096); - while (read > 0) { - str.append(buf, 0, read); - read = stream.read(buf, 0, 8096); - } - } finally { - stream.close(); - } - return str.toString(); - } - + // +// /** +// * Retrieves the timestamps from the NVD CVE meta data file. +// * +// * @return the timestamp from the currently published nvdcve downloads page +// * @throws MalformedURLException is thrown if the URL for the NVD CCE Meta +// * data is incorrect. +// * @throws DownloadFailedException is thrown if there is an error +// * downloading the nvd cve meta data file +// * @throws InvalidDataException is thrown if there is an exception parsing +// * the timestamps +// */ +// protected Map retrieveCurrentTimestampsFromWeb() throws MalformedURLException, DownloadFailedException, InvalidDataException { +// Map map = new HashMap(); +// +// File tmp = null; +// try { +// tmp = File.createTempFile("cve", "meta"); +// URL url = new URL(Settings.getString(Settings.KEYS.CVE_META_URL)); +// Downloader.fetchFile(url, tmp); +// String html = readFile(tmp); +// +// String retrieveUrl = Settings.getString(Settings.KEYS.CVE_MODIFIED_20_URL); +// NvdCveUrl cve = createNvdCveUrl("modified", retrieveUrl, html); +// cve.setNeedsUpdate(false); //the others default to true, to make life easier later this should default to false. +// map.put("modified", cve); +// int max = Settings.getInt(Settings.KEYS.CVE_URL_COUNT); +// for (int i = 1; i <= max; i++) { +// retrieveUrl = Settings.getString(Settings.KEYS.CVE_BASE_URL + Settings.KEYS.CVE_SCHEMA_2_0 + i); +// String key = Integer.toString(i); +// cve = createNvdCveUrl(key, retrieveUrl, html); +// map.put(key, cve); +// } +// } catch (IOException ex) { +// throw new DownloadFailedException("Unable to create temporary file for NVD CVE Meta File download.", ex); +// } finally { +// try { +// if (tmp != null && tmp.exists()) { +// tmp.delete(); +// } +// } finally { +// if (tmp != null && tmp.exists()) { +// tmp.deleteOnExit(); +// } +// } +// } +// return map; +// } +// +// /** +// * Creates a new NvdCveUrl object from the provide id, url, and text/html +// * from the NVD CVE downloads page. +// * +// * @param id the name of this NVD CVE Url +// * @param retrieveUrl the URL to download the file from +// * @param text a bit of HTML from the NVD CVE downloads page that contains +// * the URL and the last updated timestamp. +// * @return a shiny new NvdCveUrl object. +// * @throws InvalidDataException is thrown if the timestamp could not be +// * extracted from the provided text. +// */ +// private NvdCveUrl createNvdCveUrl(String id, String retrieveUrl, String text) throws InvalidDataException { +// Pattern pattern = Pattern.compile(Pattern.quote(retrieveUrl) + ".+?\\ 0) { +// pos += 9; +// try { +// String timestampstr = line.substring(pos, line.length() - 3).replace("at ", ""); +// long timestamp = getEpochTimeFromDateTime(timestampstr); +// item.setTimestamp(timestamp); +// } catch (NumberFormatException ex) { +// throw new InvalidDataException("NVD CVE Meta file does not contain a valid timestamp for '" + retrieveUrl + "'.", ex); +// } +// } else { +// throw new InvalidDataException("NVD CVE Meta file does not contain the updated timestamp for '" + retrieveUrl + "'."); +// } +// } else { +// throw new InvalidDataException("NVD CVE Meta file does not contain the url for '" + retrieveUrl + "'."); +// } +// return item; +// } +// +// /** +// * Parses a timestamp in the format of "MM/dd/yy hh:mm" into a calendar +// * object and returns the epoch time. Note, this removes the millisecond +// * portion of the epoch time so all numbers returned should end in 000. +// * +// * @param timestamp a string in the format of "MM/dd/yy hh:mm" +// * @return a Calendar object. +// * @throws NumberFormatException if the timestamp was parsed incorrectly. +// */ +// private long getEpochTimeFromDateTime(String timestamp) throws NumberFormatException { +// Calendar c = new GregorianCalendar(); +// int month = Integer.parseInt(timestamp.substring(0, 2)); +// int date = Integer.parseInt(timestamp.substring(3, 5)); +// int year = 2000 + Integer.parseInt(timestamp.substring(6, 8)); +// int hourOfDay = Integer.parseInt(timestamp.substring(9, 11)); +// int minute = Integer.parseInt(timestamp.substring(12, 14)); +// c.set(year, month, date, hourOfDay, minute, 0); +// long t = c.getTimeInMillis(); +// t = (t / 1000) * 1000; +// return t; +// } +// +// /** +// * Reads a file into a string. +// * +// * @param file the file to be read. +// * @return the contents of the file. +// * @throws IOException is thrown if an IOExcpetion occurs. +// */ +// private String readFile(File file) throws IOException { +// InputStreamReader stream = new InputStreamReader(new FileInputStream(file), "UTF-8"); +// StringBuilder str = new StringBuilder((int) file.length()); +// try { +// char[] buf = new char[8096]; +// int read = stream.read(buf, 0, 8096); +// while (read > 0) { +// str.append(buf, 0, read); +// read = stream.read(buf, 0, 8096); +// } +// } finally { +// stream.close(); +// } +// return str.toString(); +// } + // /** * A pojo that contains the Url and timestamp of the current NvdCve XML * files. */ - protected class NvdCveUrl { + protected static class NvdCveUrl { /** * an id. diff --git a/src/main/java/org/codesecure/dependencycheck/utils/Downloader.java b/src/main/java/org/codesecure/dependencycheck/utils/Downloader.java index 05ebebbfa..5bd86e7bd 100644 --- a/src/main/java/org/codesecure/dependencycheck/utils/Downloader.java +++ b/src/main/java/org/codesecure/dependencycheck/utils/Downloader.java @@ -97,22 +97,8 @@ public class Downloader { */ public static void fetchFile(URL url, File outputPath, boolean unzip) throws DownloadFailedException { HttpURLConnection conn = null; - Proxy proxy = null; - String proxyUrl = Settings.getString(Settings.KEYS.PROXY_URL); - try { - if (proxyUrl != null) { - int proxyPort = Settings.getInt(Settings.KEYS.PROXY_PORT); - SocketAddress addr = new InetSocketAddress(proxyUrl, proxyPort); - proxy = new Proxy(Proxy.Type.HTTP, addr); - conn = (HttpURLConnection) url.openConnection(proxy); - } else { - conn = (HttpURLConnection) url.openConnection(); - } - if (Settings.getString(Settings.KEYS.CONNECTION_TIMEOUT) != null) { - int timeout = Settings.getInt(Settings.KEYS.CONNECTION_TIMEOUT); - conn.setConnectTimeout(timeout); - } + conn = Downloader.getConnection(url); conn.setRequestProperty("Accept-Encoding", "gzip, deflate"); conn.connect(); } catch (IOException ex) { @@ -148,13 +134,13 @@ public class Downloader { throw new DownloadFailedException("Error saving downloaded file.", ex); } finally { if (writer != null) { - try { - writer.close(); - writer = null; - } catch (Exception ex) { - Logger.getLogger(Downloader.class.getName()).log(Level.FINEST, - "Error closing the writter in Downloader.", ex); - } + try { + writer.close(); + writer = null; + } catch (Exception ex) { + Logger.getLogger(Downloader.class.getName()).log(Level.FINEST, + "Error closing the writter in Downloader.", ex); + } } if (reader != null) { try { @@ -162,8 +148,8 @@ public class Downloader { reader = null; } catch (Exception ex) { - Logger.getLogger(Downloader.class.getName()).log(Level.FINEST, - "Error closing the reader in Downloader.", ex); + Logger.getLogger(Downloader.class.getName()).log(Level.FINEST, + "Error closing the reader in Downloader.", ex); } } try { @@ -173,4 +159,72 @@ public class Downloader { } } } + + /** + * Makes an HTTP Head request to retrieve the last modified date of the given URL. + * + * @param url the URL to retrieve the timestamp from + * @return an epoch timestamp + * @throws DownloadFailedException is thrown if an exception occurs making the HTTP request + */ + public static long getLastModified(URL url) throws DownloadFailedException { + HttpURLConnection conn = null; + long timestamp = 0; + try { + conn = Downloader.getConnection(url); + conn.setRequestMethod("HEAD"); + conn.connect(); + timestamp = conn.getLastModified(); + } catch (Exception ex) { + throw new DownloadFailedException("Error making HTTP HEAD request.", ex); + } finally { + if (conn != null) { + try { + conn.disconnect(); + } finally { + conn = null; + } + } + } + return timestamp; + } + + /** + * Utility method to get an HttpURLConnectoin. If the app is configured to + * use a proxy this method will retrieve the proxy settings and use them + * when setting up the connection. + * + * @param url the url to connect to + * @return an HttpURLConnection + * @throws DownloadFailedException thrown if there is an exception + */ + private static HttpURLConnection getConnection(URL url) throws DownloadFailedException { + HttpURLConnection conn = null; + Proxy proxy = null; + String proxyUrl = Settings.getString(Settings.KEYS.PROXY_URL); + try { + if (proxyUrl != null) { + int proxyPort = Settings.getInt(Settings.KEYS.PROXY_PORT); + SocketAddress addr = new InetSocketAddress(proxyUrl, proxyPort); + proxy = new Proxy(Proxy.Type.HTTP, addr); + conn = (HttpURLConnection) url.openConnection(proxy); + } else { + conn = (HttpURLConnection) url.openConnection(); + } + if (Settings.getString(Settings.KEYS.CONNECTION_TIMEOUT) != null) { + int timeout = Settings.getInt(Settings.KEYS.CONNECTION_TIMEOUT); + conn.setConnectTimeout(timeout); + } + } catch (IOException ex) { + try { + if (conn != null) { + conn.disconnect(); + } + } finally { + conn = null; + } + throw new DownloadFailedException("Error getting connection.", ex); + } + return conn; + } } diff --git a/src/main/java/org/codesecure/dependencycheck/utils/Settings.java b/src/main/java/org/codesecure/dependencycheck/utils/Settings.java index 2c816fd6e..3b0365628 100644 --- a/src/main/java/org/codesecure/dependencycheck/utils/Settings.java +++ b/src/main/java/org/codesecure/dependencycheck/utils/Settings.java @@ -70,9 +70,14 @@ public class Settings { public static final String CVE_META_URL = "cve.url.meta"; /** * The properties key for the URL to retrieve the recently modified and - * added CVE entries (last 8 days). + * added CVE entries (last 8 days) using the 2.0 schema. */ - public static final String CVE_MODIFIED_URL = "cve.url.modified"; + public static final String CVE_MODIFIED_20_URL = "cve.url-2.0.modified"; + /** + * The properties key for the URL to retrieve the recently modified and + * added CVE entries (last 8 days) using the 1.2 schema. + */ + public static final String CVE_MODIFIED_12_URL = "cve.url-1.2.modified"; /** * The properties key for the URL to retrieve the recently modified and * added CVE entries (last 8 days). @@ -86,9 +91,19 @@ public class Settings { public static final String CVE_URL_COUNT = "cve.url.count"; /** * The properties key for the "base" property key for the CVE URLs (e.g. - * cve.url.1, cve.url.2, cve.url.n). + * cve.url-2.0.1, cve.url-1.2.2, cve.url.n). */ - public static final String CVE_BASE_URL = "cve.url."; + public static final String CVE_BASE_URL = "cve.url-"; + /** + * The properties key for the CVE schema version 1.2 + */ + public static final String CVE_SCHEMA_1_2 = "1.2."; + /** + * The properties key for the CVE schema version 2.0 + */ + public static final String CVE_SCHEMA_2_0 = "2.0."; + + /** * The properties key for the proxy url. */ diff --git a/src/main/resources/configuration/dependencycheck.properties b/src/main/resources/configuration/dependencycheck.properties index 7d657a35e..f8a5504df 100644 --- a/src/main/resources/configuration/dependencycheck.properties +++ b/src/main/resources/configuration/dependencycheck.properties @@ -7,12 +7,15 @@ cpe=data/cpe cpe.url=http://static.nvd.nist.gov/feeds/xml/cpe/dictionary/official-cpe-dictionary_v2.2.xml.gz # the path to the cpe meta data file. cpe.meta.url=http://static.nvd.nist.gov/feeds/xml/cpe/dictionary/official-cpe-dictionary_v2.2.meta + + # the path to the lucene index to store the nvd cve data cve=data/cve # the path to the nvd cve "meta" page where the timestamps for the last update files can be found. -cve.url.meta=http://nvd.nist.gov/download.cfm +#cve.url.meta=http://nvd.nist.gov/download.cfm + # the path to the modified nvd cve xml file. -cve.url.modified=http://static.nvd.nist.gov/feeds/xml/cve/nvdcve-2.0-modified.xml +cve.url-2.0.modified=http://static.nvd.nist.gov/feeds/xml/cve/nvdcve-2.0-modified.xml # the number of days that the modified nvd cve data holds data for. We don't need # to update the other files if we are within this timespan. Per NIST this file @@ -20,15 +23,29 @@ cve.url.modified=http://static.nvd.nist.gov/feeds/xml/cve/nvdcve-2.0-modified.xm cve.url.modified.validfordays=7 # the number of cve.urls cve.url.count=11 -# the paths to the various nvd cve files. -cve.url.1=http://static.nvd.nist.gov/feeds/xml/cve/nvdcve-2.0-2002.xml -cve.url.2=http://static.nvd.nist.gov/feeds/xml/cve/nvdcve-2.0-2003.xml -cve.url.3=http://static.nvd.nist.gov/feeds/xml/cve/nvdcve-2.0-2004.xml -cve.url.4=http://static.nvd.nist.gov/feeds/xml/cve/nvdcve-2.0-2005.xml -cve.url.5=http://static.nvd.nist.gov/feeds/xml/cve/nvdcve-2.0-2006.xml -cve.url.6=http://static.nvd.nist.gov/feeds/xml/cve/nvdcve-2.0-2007.xml -cve.url.7=http://static.nvd.nist.gov/feeds/xml/cve/nvdcve-2.0-2008.xml -cve.url.8=http://static.nvd.nist.gov/feeds/xml/cve/nvdcve-2.0-2009.xml -cve.url.9=http://static.nvd.nist.gov/feeds/xml/cve/nvdcve-2.0-2010.xml -cve.url.10=http://static.nvd.nist.gov/feeds/xml/cve/nvdcve-2.0-2011.xml -cve.url.11=http://static.nvd.nist.gov/feeds/xml/cve/nvdcve-2.0-2012.xml +# the paths to the various nvd cve files (schema version 2.0) +cve.url-2.0.1=http://static.nvd.nist.gov/feeds/xml/cve/nvdcve-2.0-2002.xml +cve.url-2.0.2=http://static.nvd.nist.gov/feeds/xml/cve/nvdcve-2.0-2003.xml +cve.url-2.0.3=http://static.nvd.nist.gov/feeds/xml/cve/nvdcve-2.0-2004.xml +cve.url-2.0.4=http://static.nvd.nist.gov/feeds/xml/cve/nvdcve-2.0-2005.xml +cve.url-2.0.5=http://static.nvd.nist.gov/feeds/xml/cve/nvdcve-2.0-2006.xml +cve.url-2.0.6=http://static.nvd.nist.gov/feeds/xml/cve/nvdcve-2.0-2007.xml +cve.url-2.0.7=http://static.nvd.nist.gov/feeds/xml/cve/nvdcve-2.0-2008.xml +cve.url-2.0.8=http://static.nvd.nist.gov/feeds/xml/cve/nvdcve-2.0-2009.xml +cve.url-2.0.9=http://static.nvd.nist.gov/feeds/xml/cve/nvdcve-2.0-2010.xml +cve.url-2.0.10=http://static.nvd.nist.gov/feeds/xml/cve/nvdcve-2.0-2011.xml +cve.url-2.0.11=http://static.nvd.nist.gov/feeds/xml/cve/nvdcve-2.0-2012.xml + +# the paths to the various nvd cve files (schema version 1.2). +cve.url-1.2.modified=http://nvd.nist.gov/download/nvdcve-modified.xml +cve.url-1.2.1=http://nvd.nist.gov/download/nvdcve-2002.xml +cve.url-1.2.2=http://nvd.nist.gov/download/nvdcve-2003.xml +cve.url-1.2.3=http://nvd.nist.gov/download/nvdcve-2004.xml +cve.url-1.2.4=http://nvd.nist.gov/download/nvdcve-2005.xml +cve.url-1.2.5=http://nvd.nist.gov/download/nvdcve-2006.xml +cve.url-1.2.6=http://nvd.nist.gov/download/nvdcve-2007.xml +cve.url-1.2.7=http://nvd.nist.gov/download/nvdcve-2008.xml +cve.url-1.2.8=http://nvd.nist.gov/download/nvdcve-2009.xml +cve.url-1.2.9=http://nvd.nist.gov/download/nvdcve-2010.xml +cve.url-1.2.10=http://nvd.nist.gov/download/nvdcve-2011.xml +cve.url-1.2.11=http://nvd.nist.gov/download/nvdcve-2012.xml \ No newline at end of file diff --git a/src/test/java/org/codesecure/dependencycheck/dependency/DependencyTest.java b/src/test/java/org/codesecure/dependencycheck/dependency/DependencyTest.java index 63940595d..69c8c5dde 100644 --- a/src/test/java/org/codesecure/dependencycheck/dependency/DependencyTest.java +++ b/src/test/java/org/codesecure/dependencycheck/dependency/DependencyTest.java @@ -5,8 +5,6 @@ package org.codesecure.dependencycheck.dependency; import java.io.File; -import org.codesecure.dependencycheck.dependency.Dependency; -import org.codesecure.dependencycheck.dependency.Evidence; import java.util.List; import org.junit.After; import org.junit.AfterClass; diff --git a/src/test/java/org/codesecure/dependencycheck/utils/DownloaderIntegrationTest.java b/src/test/java/org/codesecure/dependencycheck/utils/DownloaderIntegrationTest.java index f426777a7..29b6959f9 100644 --- a/src/test/java/org/codesecure/dependencycheck/utils/DownloaderIntegrationTest.java +++ b/src/test/java/org/codesecure/dependencycheck/utils/DownloaderIntegrationTest.java @@ -10,6 +10,7 @@ import org.junit.AfterClass; import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; +import static org.junit.Assert.*; /** * @@ -58,4 +59,12 @@ public class DownloaderIntegrationTest { Downloader.fetchFile(url, outputPath, false); } + + @Test + public void testGetLastModified() throws Exception { + System.out.println("getLastModified"); + URL url = new URL("http://nvd.nist.gov/download/nvdcve-2012.xml"); + long timestamp = Downloader.getLastModified(url); + assertTrue("timestamp equal to zero?", timestamp>0); + } } diff --git a/src/test/java/org/codesecure/dependencycheck/utils/SettingsTest.java b/src/test/java/org/codesecure/dependencycheck/utils/SettingsTest.java index 4edcd7032..24f1befc9 100644 --- a/src/test/java/org/codesecure/dependencycheck/utils/SettingsTest.java +++ b/src/test/java/org/codesecure/dependencycheck/utils/SettingsTest.java @@ -6,7 +6,6 @@ package org.codesecure.dependencycheck.utils; import java.io.File; import java.io.IOException; -import java.io.InputStream; import java.net.URISyntaxException; import junit.framework.TestCase; import org.junit.Test;