github-mirrors/DependencyCheck
1
0
Fork 0
mirror of https://github.com/ysoftdevs/DependencyCheck.git synced 2026-03-21 16:49:43 +01:00
Code Issues Packages Projects Releases Wiki Activity

performance enhancement for nvd cve import.

Browse Source 
Former-commit-id: 5a40d5798c804c4632e6ed2dd04d6d7c9e0f51b8
This commit is contained in:
Jeremy Long
2012-10-30 21:13:47 -04:00
parent c694461abc
commit 7971c42814
10 changed files with 174 additions and 173 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
README.txt
View File

@@ -7,16 +7,11 @@ If found, it will generate a report linking to the associated CVE entries.
Usage: Usage:
$ mvn package $ mvn package
$ cd target $ cd target
$ java -jar DependencyCheck-0.2.0.jar -h $ java -jar DependencyCheck-0.2.1.jar -h
$ java -jar DependencyCheck-0.2.0.jar -a Testing -out . -scan ./test-classes/org.mortbay.jetty.jar -scan ./test-classes/struts2-core-2.1.2.jar -scan ./lib $ java -jar DependencyCheck-0.2.1.jar -a Testing -out . -scan ./test-classes/org.mortbay.jetty.jar -scan ./test-classes/struts2-core-2.1.2.jar -scan ./lib
Then load the resulting 'Testing.html' into your favorite browser. Then load the resulting 'Testing.html' into your favorite browser.
Important note - DependencyCheck should be run to analyze a project at least once every week.
The reason for this is that it downloads data from the National Vulnerability Database hosted
by NIST. If more then a week goes by without DependencyCheck updating the data, a full update
can take an 90 minutes or more (a lot of data needs to be downloaded and processed).
Author: Jeremy Long (jeremy.long@gmail.com) Author: Jeremy Long (jeremy.long@gmail.com)
Copyright (c) 2012 Jeremy Long. All Rights Reserved. Copyright (c) 2012 Jeremy Long. All Rights Reserved.

2
pom.xml
View File

@@ -23,7 +23,7 @@ along with DependencyCheck. If not, see <http://www.gnu.org/licenses/>.
<groupId>org.codesecure</groupId> <groupId>org.codesecure</groupId>
<artifactId>DependencyCheck</artifactId> <artifactId>DependencyCheck</artifactId>
<version>0.2.0</version> <version>0.2.1</version>
<packaging>jar</packaging> <packaging>jar</packaging>
<name>DependencyCheck</name> <name>DependencyCheck</name>

12
src/main/java/org/codesecure/dependencycheck/data/nvdcve/Index.java
View File

@@ -26,8 +26,6 @@ import java.util.logging.Level;
import java.util.logging.Logger; import java.util.logging.Logger;
import java.util.regex.Matcher; import java.util.regex.Matcher;
import java.util.regex.Pattern; import java.util.regex.Pattern;
import javax.xml.bind.JAXBException;
import javax.xml.parsers.ParserConfigurationException;
import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.KeywordAnalyzer; import org.apache.lucene.analysis.KeywordAnalyzer;
import org.apache.lucene.analysis.PerFieldAnalyzerWrapper; import org.apache.lucene.analysis.PerFieldAnalyzerWrapper;
@@ -42,7 +40,6 @@ import org.codesecure.dependencycheck.data.nvdcve.xml.Importer;
import org.codesecure.dependencycheck.utils.DownloadFailedException; import org.codesecure.dependencycheck.utils.DownloadFailedException;
import org.codesecure.dependencycheck.utils.Downloader; import org.codesecure.dependencycheck.utils.Downloader;
import org.codesecure.dependencycheck.utils.Settings; import org.codesecure.dependencycheck.utils.Settings;
import org.xml.sax.SAXException;
/** /**
* The Index class is used to utilize and maintain the NVD CVE Index. * The Index class is used to utilize and maintain the NVD CVE Index.
@@ -134,15 +131,6 @@ public class Index extends AbstractIndex implements CachedWebDataSource {
} catch (FileNotFoundException ex) { } catch (FileNotFoundException ex) {
//Logger.getLogger(Index.class.getName()).log(Level.SEVERE, null, ex); //Logger.getLogger(Index.class.getName()).log(Level.SEVERE, null, ex);
throw new UpdateException(ex); throw new UpdateException(ex);
} catch (JAXBException ex) {
//Logger.getLogger(Index.class.getName()).log(Level.SEVERE, null, ex);
throw new UpdateException(ex);
} catch (ParserConfigurationException ex) {
//Logger.getLogger(Index.class.getName()).log(Level.SEVERE, null, ex);
throw new UpdateException(ex);
} catch (SAXException ex) {
//Logger.getLogger(Index.class.getName()).log(Level.SEVERE, null, ex);
throw new UpdateException(ex);
} catch (IOException ex) { } catch (IOException ex) {
//Logger.getLogger(Index.class.getName()).log(Level.SEVERE, null, ex); //Logger.getLogger(Index.class.getName()).log(Level.SEVERE, null, ex);
throw new UpdateException(ex); throw new UpdateException(ex);

24
src/main/java/org/codesecure/dependencycheck/data/nvdcve/xml/Importer.java
View File

@@ -21,14 +21,7 @@ package org.codesecure.dependencycheck.data.nvdcve.xml;
import java.io.*; import java.io.*;
import java.util.logging.Level; import java.util.logging.Level;
import java.util.logging.Logger; import java.util.logging.Logger;
import javax.xml.bind.JAXBContext;
import javax.xml.bind.JAXBException;
import javax.xml.parsers.ParserConfigurationException;
import javax.xml.parsers.SAXParserFactory;
import org.apache.lucene.index.CorruptIndexException; import org.apache.lucene.index.CorruptIndexException;
import org.xml.sax.InputSource;
import org.xml.sax.SAXException;
import org.xml.sax.XMLReader;
/** /**
* Imports a NVD CVE XML file into the Lucene NVD CVE Index. * Imports a NVD CVE XML file into the Lucene NVD CVE Index.
@@ -47,13 +40,6 @@ public class Importer {
* Imports the NVD CVE XML File into the Lucene Index. * Imports the NVD CVE XML File into the Lucene Index.
* *
* @param file containing the path to the NVD CVE XML file. * @param file containing the path to the NVD CVE XML file.
* @throws ParserConfigurationException is thrown if the parser is
* misconfigured.
* @throws FileNotFoundException is thrown when there is a
* FileNotFoundException.
* @throws IOException is thrown when there is an IOException.
* @throws JAXBException is thrown when there is a JAXBException.
* @throws SAXException is thrown when there is a SAXException.
*/ */
public static void importXML(File file) { public static void importXML(File file) {
NvdCveParser indexer = null; NvdCveParser indexer = null;
@@ -109,16 +95,8 @@ public class Importer {
* Imports the CPE XML File into the Lucene Index. * Imports the CPE XML File into the Lucene Index.
* *
* @param path the path to the CPE XML file. * @param path the path to the CPE XML file.
* @throws ParserConfigurationException is thrown if the parser is
* misconfigured.
* @throws FileNotFoundException is thrown when there is a
* FileNotFoundException.
* @throws IOException is thrown when there is an IOException.
* @throws JAXBException is thrown when there is a JAXBException.
* @throws SAXException is thrown when there is a SAXException.
*/ */
public static void importXML(String path) throws FileNotFoundException, IOException, JAXBException, public static void importXML(String path) {
ParserConfigurationException, SAXException {
File f = new File(path); File f = new File(path);
if (!f.exists()) { if (!f.exists()) {
f.mkdirs(); f.mkdirs();

55
src/main/java/org/codesecure/dependencycheck/data/nvdcve/xml/NvdCveParser.java
View File

@@ -1,8 +1,22 @@
/*
* To change this template, choose Tools | Templates
* and open the template in the editor.
*/
package org.codesecure.dependencycheck.data.nvdcve.xml; package org.codesecure.dependencycheck.data.nvdcve.xml;
/*
* This file is part of DependencyCheck.
*
* DependencyCheck is free software: you can redistribute it and/or modify it
* under the terms of the GNU General Public License as published by the Free
* Software Foundation, either version 3 of the License, or (at your option) any
* later version.
*
* DependencyCheck is distributed in the hope that it will be useful, but
* WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
* details.
*
* You should have received a copy of the GNU General Public License along with
* DependencyCheck. If not, see http://www.gnu.org/licenses/.
*
* Copyright (c) 2012 Jeremy Long. All Rights Reserved.
*/
import java.io.BufferedReader; import java.io.BufferedReader;
import java.io.File; import java.io.File;
@@ -17,7 +31,6 @@ import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field; import org.apache.lucene.document.Field;
import org.apache.lucene.index.FieldInfo.IndexOptions; import org.apache.lucene.index.FieldInfo.IndexOptions;
import org.apache.lucene.index.Term; import org.apache.lucene.index.Term;
import org.codesecure.dependencycheck.data.lucene.LuceneUtils;
import org.codesecure.dependencycheck.data.nvdcve.Fields; import org.codesecure.dependencycheck.data.nvdcve.Fields;
import org.codesecure.dependencycheck.data.nvdcve.Index; import org.codesecure.dependencycheck.data.nvdcve.Index;
@@ -27,6 +40,14 @@ import org.codesecure.dependencycheck.data.nvdcve.Index;
*/ */
public class NvdCveParser extends Index { public class NvdCveParser extends Index {
/**
* Parses an NVD CVE xml file using a buffered readerd. This
* method maybe more fragile then using a partial-unmarshalling SAX
* Parser (aka the deprecated NvdCveXmlFilter) - but this method is
* orders of magnitude faster.
*
* @param file the reference to the NVD CVE file
*/
public void parse(File file) { public void parse(File file) {
FileReader fr = null; FileReader fr = null;
BufferedReader br = null; BufferedReader br = null;
@@ -66,8 +87,22 @@ public class NvdCveParser extends Index {
id = matcherEntry.group(1); id = matcherEntry.group(1);
sb.append("<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"yes\"?>"); sb.append("<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"yes\"?>");
sb.append("<vulnerabilityType xmlns=\"http://scap.nist.gov/schema/vulnerability/0.4\" xmlns:vuln=\"http://scap.nist.gov/schema/vulnerability/0.4\" xmlns:cpe-lang=\"http://cpe.mitre.org/language/2.0\" xmlns:cvss=\"http://scap.nist.gov/schema/cvss-v2/0.2\" xmlns:scap-core=\"http://scap.nist.gov/schema/scap-core/0.1\" xmlns:patch=\"http://scap.nist.gov/schema/patch/0.1\" xmlns:ns6=\"http://scap.nist.gov/schema/cve/0.1\" xmlns:cce=\"http://scap.nist.gov/schema/cce/0.1\" xmlns:vulnerability=\"http://scap.nist.gov/schema/feed/vulnerability/2.0\""); sb.append("<vulnerabilityType ");
sb.append(" id=\"").append(id).append("\">"); //sb.append("xmlns=\"http://scap.nist.gov/schema/feed/vulnerability/2.0\" ");
//sb.append("xmlns:vuln=\"http://scap.nist.gov/schema/vulnerability/0.4\" ");
sb.append("xmlns=\"http://scap.nist.gov/schema/vulnerability/0.4\" ");
sb.append("xmlns:vuln=\"http://scap.nist.gov/schema/vulnerability/0.4\" ");
//sb.append("xmlns:vulnerability=\"http://scap.nist.gov/schema/feed/vulnerability/2.0\" ");
sb.append("xmlns:cpe-lang=\"http://cpe.mitre.org/language/2.0\" ");
sb.append("xmlns:cvss2=\"http://scap.nist.gov/schema/cvss-v2/0.2\" ");
sb.append("xmlns:cvss=\"http://scap.nist.gov/schema/cvss-v2/0.2\" ");
sb.append("xmlns:scap-core=\"http://scap.nist.gov/schema/scap-core/0.1\" ");
sb.append("xmlns:scap_core=\"http://scap.nist.gov/schema/scap-core/0.1\" ");
sb.append("xmlns:patch=\"http://scap.nist.gov/schema/patch/0.1\" ");
sb.append("xmlns:cve=\"http://scap.nist.gov/schema/cve/0.1\" ");
sb.append("xmlns:cce=\"http://scap.nist.gov/schema/cce/0.1\" ");
sb.append("id=\"").append(id).append("\">");
//sb.append(str); //need to do the above to get the correct schema generated from files. //sb.append(str); //need to do the above to get the correct schema generated from files.
Field name = new Field(Fields.CVE_ID, id, Field.Store.NO, Field.Index.ANALYZED); Field name = new Field(Fields.CVE_ID, id, Field.Store.NO, Field.Index.ANALYZED);
@@ -125,7 +160,11 @@ public class NvdCveParser extends Index {
} }
} }
/**
* Adds a CPE to the Lucene Document
* @param cpe a string representing a CPE
* @param doc a lucene document
*/
private void addVulnerableCpe(String cpe, Document doc) { private void addVulnerableCpe(String cpe, Document doc) {
Field vulnerable = new Field(Fields.VULNERABLE_CPE, cpe, Field.Store.NO, Field.Index.ANALYZED); Field vulnerable = new Field(Fields.VULNERABLE_CPE, cpe, Field.Store.NO, Field.Index.ANALYZED);
vulnerable.setIndexOptions(IndexOptions.DOCS_ONLY); vulnerable.setIndexOptions(IndexOptions.DOCS_ONLY);

1
src/main/java/org/codesecure/dependencycheck/data/nvdcve/xml/NvdCveXmlFilter.java
View File

@@ -46,6 +46,7 @@ import org.xml.sax.helpers.XMLFilterImpl;
* *
* @author Jeremy * @author Jeremy
*/ */
@Deprecated
public class NvdCveXmlFilter extends XMLFilterImpl { public class NvdCveXmlFilter extends XMLFilterImpl {
EntrySaveDelegate saveDelegate = null; EntrySaveDelegate saveDelegate = null;

8
src/main/resources/templates/HtmlReport.vsl
View File

@@ -353,7 +353,7 @@ Copyright (c) 2012 Jeremy Long. All Rights Reserved.
#end #end
#end #end
<h4 id="header$cnt" class="subsectionheader white">Identifiers</h4> <h4 id="header$cnt" class="subsectionheader white">Identifiers</h4>
##:&nbsp;<a href="http://web.nvd.nist.gov/view/vuln/search-results?cpe=$esc.url($cpevalue)" target="blank">$esc.html($cpevalue)</a></h4> ##:&nbsp;<a href="http://web.nvd.nist.gov/view/vuln/search-results?cpe=$esc.url($cpevalue)" target="_blank">$esc.html($cpevalue)</a></h4>
<div id="content$cnt" class="subsectioncontent standardsubsection"> <div id="content$cnt" class="subsectioncontent standardsubsection">
#if($cpeCount>1) #if($cpeCount>1)
Several possible CPEs where identified. If one of the following are correct please update the configuration Several possible CPEs where identified. If one of the following are correct please update the configuration
@@ -366,7 +366,7 @@ Copyright (c) 2012 Jeremy Long. All Rights Reserved.
<ul> <ul>
#foreach($id in $dependency.getIdentifiers()) #foreach($id in $dependency.getIdentifiers())
##yes, we are HTML Encoding the href. this is okay. We can't URL encode as we have to trust the analyzer here... ##yes, we are HTML Encoding the href. this is okay. We can't URL encode as we have to trust the analyzer here...
<li><b>$esc.html($id.type):</b>&nbsp;$esc.html($id.title)&nbsp;:&nbsp;<a href="$esc.html($id.url)" target="blank">$esc.html($id.value)</a> <li><b>$esc.html($id.type):</b>&nbsp;$esc.html($id.title)&nbsp;:&nbsp;<a href="$esc.html($id.url)" target="_blank">$esc.html($id.value)</a>
#if( $id.descrription ) #if( $id.descrription )
<br/>$esc.html($id.description) <br/>$esc.html($id.description)
#end #end
@@ -380,12 +380,12 @@ Copyright (c) 2012 Jeremy Long. All Rights Reserved.
<h4 id="header$cnt" class="subsectionheader white">Published Vulnerabilities</h4> <h4 id="header$cnt" class="subsectionheader white">Published Vulnerabilities</h4>
<div id="content$cnt" class="subsectioncontent standardsubsection"> <div id="content$cnt" class="subsectioncontent standardsubsection">
#foreach($vuln in $dependency.getVulnerabilities()) #foreach($vuln in $dependency.getVulnerabilities())
<p><b><a target="blank" href="http://web.nvd.nist.gov/view/vuln/detail?vulnId=$esc.url($vuln.name)">$esc.html($vuln.name)</a></b></p> <p><b><a target="_blank" href="http://web.nvd.nist.gov/view/vuln/detail?vulnId=$esc.url($vuln.name)">$esc.html($vuln.name)</a></b></p>
<p>$esc.html($vuln.description) <p>$esc.html($vuln.description)
#if ($vuln.getReferences().size()>0) #if ($vuln.getReferences().size()>0)
<ul> <ul>
#foreach($ref in $vuln.getReferences()) #foreach($ref in $vuln.getReferences())
<li>$esc.html($ref.source) - <a target="blank" href="$esc.html($ref.url)">$ref.name</a></li> <li>$esc.html($ref.source) - <a target="_blank" href="$esc.html($ref.url)">$ref.name</a></li>
#end #end
</ul> </ul>
#end #end

196
src/test/java/org/codesecure/dependencycheck/data/nvdcve/xml/NvdCveXmlFilterTest.java
View File

@@ -3,101 +3,101 @@
* and open the template in the editor. * and open the template in the editor.
*/ */
package org.codesecure.dependencycheck.data.nvdcve.xml; package org.codesecure.dependencycheck.data.nvdcve.xml;
//
import java.io.BufferedInputStream; //import java.io.BufferedInputStream;
import java.io.DataInputStream; //import java.io.DataInputStream;
import java.io.File; //import java.io.File;
import java.io.FileReader; //import java.io.FileReader;
import java.io.IOException; //import java.io.IOException;
import java.io.InputStream; //import java.io.InputStream;
import java.io.Reader; //import java.io.Reader;
import java.net.MalformedURLException; //import java.net.MalformedURLException;
import java.util.logging.Level; //import java.util.logging.Level;
import java.util.logging.Logger; //import java.util.logging.Logger;
import javax.xml.bind.JAXBContext; //import javax.xml.bind.JAXBContext;
import javax.xml.bind.JAXBException; //import javax.xml.bind.JAXBException;
import javax.xml.parsers.ParserConfigurationException; //import javax.xml.parsers.ParserConfigurationException;
import javax.xml.parsers.SAXParserFactory; //import javax.xml.parsers.SAXParserFactory;
import org.apache.lucene.index.CorruptIndexException; //import org.apache.lucene.index.CorruptIndexException;
import org.codesecure.dependencycheck.data.nvdcve.InvalidDataException; //import org.codesecure.dependencycheck.data.nvdcve.InvalidDataException;
import org.codesecure.dependencycheck.data.nvdcve.generated.VulnerabilityType; //import org.codesecure.dependencycheck.data.nvdcve.generated.VulnerabilityType;
import org.junit.After; //import org.junit.After;
import org.junit.AfterClass; //import org.junit.AfterClass;
import org.junit.Before; //import org.junit.Before;
import org.junit.BeforeClass; //import org.junit.BeforeClass;
import org.junit.Test; //import org.junit.Test;
import static org.junit.Assert.*; //import static org.junit.Assert.*;
import org.xml.sax.Attributes; //import org.xml.sax.Attributes;
import org.xml.sax.InputSource; //import org.xml.sax.InputSource;
import org.xml.sax.Locator; //import org.xml.sax.Locator;
import org.xml.sax.SAXException; //import org.xml.sax.SAXException;
import org.xml.sax.XMLReader; //import org.xml.sax.XMLReader;
//
/** ///**
* // *
* @author Jeremy // * @author Jeremy
*/ // */
public class NvdCveXmlFilterTest { //public class NvdCveXmlFilterTest {
//
public NvdCveXmlFilterTest() { // public NvdCveXmlFilterTest() {
} // }
//
@BeforeClass // @BeforeClass
public static void setUpClass() { // public static void setUpClass() {
} // }
//
@AfterClass // @AfterClass
public static void tearDownClass() { // public static void tearDownClass() {
} // }
//
@Before // @Before
public void setUp() { // public void setUp() {
} // }
//
@After // @After
public void tearDown() { // public void tearDown() {
} // }
//
/** // /**
* Test of process method, of class NvdCveXmlFilter. // * Test of process method, of class NvdCveXmlFilter.
*/ // */
@Test // @Test
public void testFilter() throws InvalidDataException { // public void testFilter() throws InvalidDataException {
Indexer indexer = null; // Indexer indexer = null;
try { // try {
System.out.println("filter"); // System.out.println("filter");
//
SAXParserFactory factory = SAXParserFactory.newInstance(); // SAXParserFactory factory = SAXParserFactory.newInstance();
factory.setNamespaceAware(true); // factory.setNamespaceAware(true);
XMLReader reader = factory.newSAXParser().getXMLReader(); // XMLReader reader = factory.newSAXParser().getXMLReader();
//
JAXBContext context = JAXBContext.newInstance("org.codesecure.dependencycheck.data.nvdcve.generated"); // JAXBContext context = JAXBContext.newInstance("org.codesecure.dependencycheck.data.nvdcve.generated");
NvdCveXmlFilter filter = new NvdCveXmlFilter(context); // NvdCveXmlFilter filter = new NvdCveXmlFilter(context);
//
indexer = new Indexer(); // indexer = new Indexer();
indexer.openIndexWriter(); // indexer.openIndexWriter();
//
filter.registerSaveDelegate(indexer); // filter.registerSaveDelegate(indexer);
//
reader.setContentHandler(filter); // reader.setContentHandler(filter);
File file = new File(this.getClass().getClassLoader().getResource("nvdcve-2.0-2012.xml").getPath()); // File file = new File(this.getClass().getClassLoader().getResource("nvdcve-2.0-2012.xml").getPath());
Reader fileReader = new FileReader(file); // Reader fileReader = new FileReader(file);
InputSource is = new InputSource(fileReader); // InputSource is = new InputSource(fileReader);
reader.parse(is); // reader.parse(is);
} catch (JAXBException ex) { // } catch (JAXBException ex) {
throw new InvalidDataException("JAXBException", ex); // throw new InvalidDataException("JAXBException", ex);
} catch (SAXException ex) { // } catch (SAXException ex) {
throw new InvalidDataException("SAXException", ex); // throw new InvalidDataException("SAXException", ex);
} catch (ParserConfigurationException ex) { // } catch (ParserConfigurationException ex) {
throw new InvalidDataException("ParserConfigurationException", ex); // throw new InvalidDataException("ParserConfigurationException", ex);
} catch (CorruptIndexException ex) { // } catch (CorruptIndexException ex) {
throw new InvalidDataException("CorruptIndexException", ex); // throw new InvalidDataException("CorruptIndexException", ex);
} catch (IOException ex) { // } catch (IOException ex) {
throw new InvalidDataException("IOException", ex); // throw new InvalidDataException("IOException", ex);
} finally { // } finally {
if (indexer != null) { // if (indexer != null) {
indexer.close(); // indexer.close();
} // }
} // }
} // }
} //}