diff --git a/pom.xml b/pom.xml index 7e70ff461..098a34817 100644 --- a/pom.xml +++ b/pom.xml @@ -417,5 +417,11 @@ along with DependencyCheck. If not, see . + diff --git a/src/main/java/org/codesecure/dependencycheck/data/nvdcve/Index.java b/src/main/java/org/codesecure/dependencycheck/data/nvdcve/Index.java index 43f64aa0b..f3e9d7856 100644 --- a/src/main/java/org/codesecure/dependencycheck/data/nvdcve/Index.java +++ b/src/main/java/org/codesecure/dependencycheck/data/nvdcve/Index.java @@ -121,13 +121,16 @@ public class Index extends AbstractIndex implements CachedWebDataSource { for (NvdCveUrl cve : update.values()) { if (cve.getNeedsUpdate()) { count += 1; - Logger.getLogger(Index.class.getName()).log(Level.WARNING, "Updating NVD CVE (" + count + " of " + maxUpdates + ") :" + cve.getUrl()); + Logger.getLogger(Index.class.getName()).log(Level.WARNING, "Updating NVD CVE (" + count + " of " + maxUpdates + ")"); URL url = new URL(cve.getUrl()); File outputPath = null; try { + Logger.getLogger(Index.class.getName()).log(Level.WARNING, "Downloading " + cve.getUrl()); outputPath = File.createTempFile("cve" + cve.getId() + "_", ".xml"); Downloader.fetchFile(url, outputPath, false); + Logger.getLogger(Index.class.getName()).log(Level.WARNING, "Processing " + cve.getUrl()); Importer.importXML(outputPath.toString()); + Logger.getLogger(Index.class.getName()).log(Level.WARNING, "Completed updated " + count + " of " + maxUpdates); } catch (FileNotFoundException ex) { //Logger.getLogger(Index.class.getName()).log(Level.SEVERE, null, ex); throw new UpdateException(ex); diff --git a/src/main/java/org/codesecure/dependencycheck/data/nvdcve/xml/Indexer.java b/src/main/java/org/codesecure/dependencycheck/data/nvdcve/xml/Indexer.java index 6f4a19108..74ee2c82d 100644 --- a/src/main/java/org/codesecure/dependencycheck/data/nvdcve/xml/Indexer.java +++ b/src/main/java/org/codesecure/dependencycheck/data/nvdcve/xml/Indexer.java @@ -106,7 +106,7 @@ public class Indexer extends Index implements EntrySaveDelegate { doc.add(name); Field description = new Field(Fields.DESCRIPTION, vulnerability.getSummary(), Field.Store.NO, Field.Index.ANALYZED); - name.setIndexOptions(IndexOptions.DOCS_ONLY); + description.setIndexOptions(IndexOptions.DOCS_ONLY); doc.add(description); diff --git a/src/main/java/org/codesecure/dependencycheck/data/nvdcve/xml/NvdCveParser.java b/src/main/java/org/codesecure/dependencycheck/data/nvdcve/xml/NvdCveParser.java new file mode 100644 index 000000000..5648978f8 --- /dev/null +++ b/src/main/java/org/codesecure/dependencycheck/data/nvdcve/xml/NvdCveParser.java @@ -0,0 +1,109 @@ +/* + * To change this template, choose Tools | Templates + * and open the template in the editor. + */ +package org.codesecure.dependencycheck.data.nvdcve.xml; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileNotFoundException; +import java.io.FileReader; +import java.io.IOException; +import java.util.logging.Level; +import java.util.logging.Logger; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; +import org.apache.lucene.index.FieldInfo.IndexOptions; +import org.apache.lucene.index.Term; +import org.codesecure.dependencycheck.data.lucene.LuceneUtils; +import org.codesecure.dependencycheck.data.nvdcve.Fields; +import org.codesecure.dependencycheck.data.nvdcve.Index; + +/** + * + * @author Jeremy Long (jeremy.long@gmail.com) + */ +public class NvdCveParser extends Index { + + public void parse(File file) { + FileReader fr = null; + BufferedReader br = null; + Pattern rxEntry = Pattern.compile("^\\s*\\([^\\<]+"); + try { + fr = new FileReader(file); + br = new BufferedReader(fr); + StringBuilder sb = new StringBuilder(7000); + String str = null; + String id = null; + Document doc = new Document(); + while ((str = br.readLine()) != null) { + sb.append(str); + //facts occur more often, do them first. + Matcher matcherFact = rxFact.matcher(str); + if (matcherFact.matches()) { + addVulnerableCpe(matcherFact.group(0), doc); + continue; + } + Matcher matcherEntry = rxEntry.matcher(str); + if (matcherEntry.matches()) { + id = matcherEntry.group(0); + Field name = new Field(Fields.CVE_ID, id, Field.Store.NO, Field.Index.ANALYZED); + name.setIndexOptions(IndexOptions.DOCS_ONLY); + doc.add(name); + continue; + } + Matcher matcherSummary = rxSummary.matcher(str); + if (matcherSummary.matches()) { + String summary = matcherSummary.group(0); + Field description = new Field(Fields.DESCRIPTION, summary, Field.Store.NO, Field.Index.ANALYZED); + description.setIndexOptions(IndexOptions.DOCS_ONLY); + doc.add(description); + continue; + } + Matcher matcherEntryEnd = rxEntryEnd.matcher(str); + if (matcherEntryEnd.matches()) { + + Field xml = new Field(Fields.XML, sb.toString(), Field.Store.YES, Field.Index.NO); + doc.add(xml); + + Term name = new Term(Fields.CVE_ID, LuceneUtils.escapeLuceneQuery(id)); + indexWriter.updateDocument(name, doc); + + doc = new Document(); + + } + } + + + } catch (FileNotFoundException ex) { + Logger.getLogger(NvdCveParser.class.getName()).log(Level.SEVERE, null, ex); + } catch (IOException ex) { + Logger.getLogger(NvdCveParser.class.getName()).log(Level.SEVERE, null, ex); + } finally { + try { + fr.close(); + } catch (IOException ex) { + Logger.getLogger(NvdCveParser.class.getName()).log(Level.SEVERE, null, ex); + } + try { + if (br != null) { + br.close(); + } + } catch (IOException ex) { + Logger.getLogger(NvdCveParser.class.getName()).log(Level.SEVERE, null, ex); + } + } + } + + + private void addVulnerableCpe(String cpe, Document doc) { + Field vulnerable = new Field(Fields.VULNERABLE_CPE, cpe, Field.Store.NO, Field.Index.ANALYZED); + vulnerable.setIndexOptions(IndexOptions.DOCS_ONLY); + doc.add(vulnerable); + } +} diff --git a/src/main/java/org/codesecure/dependencycheck/data/nvdcve/xml/package-info.java b/src/main/java/org/codesecure/dependencycheck/data/nvdcve/xml/package-info.java index d0e4ee5f6..4453648c7 100644 --- a/src/main/java/org/codesecure/dependencycheck/data/nvdcve/xml/package-info.java +++ b/src/main/java/org/codesecure/dependencycheck/data/nvdcve/xml/package-info.java @@ -4,7 +4,13 @@ * org.codesecure.dependencycheck.data.nvdcve.xml * * - * Contains classes used to parse the NVD CVE XML file. + *

Contains classes used to parse the NVD CVE XML file.

+ *

The basic use is that the Importer is called to import + * an NVD CVE file. The Importer instantiates an Indexer object + * (which extends Index). The Indexer creates a partial-unmarshalling + * SAX parser (implemented in the NvdCveXmlFilter) that extracts + * VulnerabilityTypes (aka Entry) from the NVD CVE data file and + * stores these into a Lucene Index.

* * */ diff --git a/src/main/java/org/codesecure/dependencycheck/utils/Downloader.java b/src/main/java/org/codesecure/dependencycheck/utils/Downloader.java index 54ef4facb..84f199d6b 100644 --- a/src/main/java/org/codesecure/dependencycheck/utils/Downloader.java +++ b/src/main/java/org/codesecure/dependencycheck/utils/Downloader.java @@ -112,7 +112,7 @@ public class Downloader { int timeout = Settings.getInt(Settings.KEYS.CONNECTION_TIMEOUT); conn.setConnectTimeout(timeout); } - + conn.setRequestProperty("Accept-Encoding", "gzip, deflate"); conn.connect(); } catch (IOException ex) { try { @@ -124,11 +124,10 @@ public class Downloader { } throw new DownloadFailedException("Error downloading file.", ex); } - + String encoding = conn.getContentEncoding(); + BufferedOutputStream writer = null; try { - //the following times out on some systems because the CPE is big. - //InputStream reader = url.openStream(); InputStream reader; if (unzip) { reader = new GZIPInputStream(conn.getInputStream()); diff --git a/src/test/java/org/codesecure/dependencycheck/utils/DownloaderIntegrationTest.java b/src/test/java/org/codesecure/dependencycheck/utils/DownloaderIntegrationTest.java new file mode 100644 index 000000000..17e2d6203 --- /dev/null +++ b/src/test/java/org/codesecure/dependencycheck/utils/DownloaderIntegrationTest.java @@ -0,0 +1,61 @@ +/* + * To change this template, choose Tools | Templates + * and open the template in the editor. + */ +package org.codesecure.dependencycheck.utils; + +import java.net.URL; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + +/** + * + * @author Jeremy Long (jeremy.long@gmail.com) + */ +public class DownloaderIntegrationTest { + + public DownloaderIntegrationTest() { + } + + @BeforeClass + public static void setUpClass() throws Exception { + } + + @AfterClass + public static void tearDownClass() throws Exception { + } + + @Before + public void setUp() { + } + + @After + public void tearDown() { + } + + + /** + * Test of fetchFile method, of class Downloader. + * @throws Exception thrown when an excpetion occurs. + */ + @Test + public void testFetchFile_URL_String() throws Exception { + System.out.println("fetchFile"); + +// Settings.setString(Settings.KEYS.PROXY_URL, "test"); +// Settings.setString(Settings.KEYS.PROXY_PORT, "80"); +// Settings.setString(Settings.KEYS.CONNECTION_TIMEOUT, "1000"); + + Settings.setString(Settings.KEYS.PROXY_PORT, "8080"); + Settings.setString(Settings.KEYS.PROXY_URL, "127.0.0.1"); + + //URL url = new URL(Settings.getString(Settings.KEYS.CPE_URL)); + URL url = new URL("http://static.nvd.nist.gov/feeds/xml/cve/nvdcve-2.0-modified.xml"); + + String outputPath = "target\\downloaded_cpe.xml"; + Downloader.fetchFile(url, outputPath); + } +} diff --git a/src/test/java/org/codesecure/dependencycheck/utils/DownloaderTest.java b/src/test/java/org/codesecure/dependencycheck/utils/DownloaderTest.java deleted file mode 100644 index 4a067a8ef..000000000 --- a/src/test/java/org/codesecure/dependencycheck/utils/DownloaderTest.java +++ /dev/null @@ -1,57 +0,0 @@ -/* - * To change this template, choose Tools | Templates - * and open the template in the editor. - */ -package org.codesecure.dependencycheck.utils; - -import java.net.URL; -import org.junit.After; -import org.junit.AfterClass; -import org.junit.Before; -import org.junit.BeforeClass; -import org.junit.Test; - -/** - * - * @author Jeremy Long (jeremy.long@gmail.com) - */ -public class DownloaderTest { - - public DownloaderTest() { - } - - @BeforeClass - public static void setUpClass() throws Exception { - } - - @AfterClass - public static void tearDownClass() throws Exception { - } - - @Before - public void setUp() { - } - - @After - public void tearDown() { - } - - -//This test is being removed because it is a bit too slow. -// /** -// * Test of fetchFile method, of class Downloader. -// * @throws Exception thrown when an excpetion occurs. -// */ -// @Test -// public void testFetchFile_URL_String() throws Exception { -// System.out.println("fetchFile"); -// -//// Settings.setString(Settings.KEYS.PROXY_URL, "test"); -//// Settings.setString(Settings.KEYS.PROXY_PORT, "80"); -//// Settings.setString(Settings.KEYS.CONNECTION_TIMEOUT, "1000"); -// -// URL url = new URL(Settings.getString(Settings.KEYS.CPE_URL)); -// String outputPath = "target\\downloaded_cpe.xml"; -// Downloader.fetchFile(url, outputPath); -// } -}