github-mirrors/DependencyCheck
1
0
Fork 0
mirror of https://github.com/ysoftdevs/DependencyCheck.git synced 2026-01-14 15:53:36 +01:00
Code Issues Packages Projects Releases Wiki Activity

minor updates

Browse Source 
Former-commit-id: e5bac8c3d6caab97b70210568369b51d11558741
This commit is contained in:
Jeremy Long
2012-10-29 21:47:26 -04:00
parent b157f7ad47
commit 04d82554e8
8 changed files with 191 additions and 64 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
pom.xml
View File

@@ -417,5 +417,11 @@ along with DependencyCheck. If not, see <http://www.gnu.org/licenses/>.
</exclusion>
</exclusions>
</dependency>
<!--
<dependency>
<groupId>org.fusesource.hawtdb</groupId>
<artifactId>hawtdb</artifactId>
<version>1.6</version>
</dependency>-->
</dependencies>
</project>

5
src/main/java/org/codesecure/dependencycheck/data/nvdcve/Index.java
View File

@@ -121,13 +121,16 @@ public class Index extends AbstractIndex implements CachedWebDataSource {
for (NvdCveUrl cve : update.values()) {
if (cve.getNeedsUpdate()) {
count += 1;
Logger.getLogger(Index.class.getName()).log(Level.WARNING, "Updating NVD CVE (" + count + " of " + maxUpdates + ") :" + cve.getUrl());
Logger.getLogger(Index.class.getName()).log(Level.WARNING, "Updating NVD CVE (" + count + " of " + maxUpdates + ")");
URL url = new URL(cve.getUrl());
File outputPath = null;
try {
Logger.getLogger(Index.class.getName()).log(Level.WARNING, "Downloading " + cve.getUrl());
outputPath = File.createTempFile("cve" + cve.getId() + "_", ".xml");
Downloader.fetchFile(url, outputPath, false);
Logger.getLogger(Index.class.getName()).log(Level.WARNING, "Processing " + cve.getUrl());
Importer.importXML(outputPath.toString());
Logger.getLogger(Index.class.getName()).log(Level.WARNING, "Completed updated " + count + " of " + maxUpdates);
} catch (FileNotFoundException ex) {
//Logger.getLogger(Index.class.getName()).log(Level.SEVERE, null, ex);
throw new UpdateException(ex);

2
src/main/java/org/codesecure/dependencycheck/data/nvdcve/xml/Indexer.java
View File

@@ -106,7 +106,7 @@ public class Indexer extends Index implements EntrySaveDelegate {
doc.add(name);
Field description = new Field(Fields.DESCRIPTION, vulnerability.getSummary(), Field.Store.NO, Field.Index.ANALYZED);
name.setIndexOptions(IndexOptions.DOCS_ONLY);
description.setIndexOptions(IndexOptions.DOCS_ONLY);
doc.add(description);

109
src/main/java/org/codesecure/dependencycheck/data/nvdcve/xml/NvdCveParser.java Normal file
View File

@@ -0,0 +1,109 @@
/*
* To change this template, choose Tools | Templates
* and open the template in the editor.
*/
package org.codesecure.dependencycheck.data.nvdcve.xml;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileReader;
import java.io.IOException;
import java.util.logging.Level;
import java.util.logging.Logger;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.index.FieldInfo.IndexOptions;
import org.apache.lucene.index.Term;
import org.codesecure.dependencycheck.data.lucene.LuceneUtils;
import org.codesecure.dependencycheck.data.nvdcve.Fields;
import org.codesecure.dependencycheck.data.nvdcve.Index;
/**
*
* @author Jeremy Long (jeremy.long@gmail.com)
*/
public class NvdCveParser extends Index {
public void parse(File file) {
FileReader fr = null;
BufferedReader br = null;
Pattern rxEntry = Pattern.compile("^\\s*\\<entry\\s*id\\=\\\"([^\\\"]+)\\\"");
Pattern rxEntryEnd = Pattern.compile("^\\s*\\</entry");
Pattern rxFact = Pattern.compile("^\\s*\\<cpe\\-lang\\:fact\\-ref name=\\\"([^\\\"]+)");
Pattern rxSummary = Pattern.compile("^\\s*\\<vuln:summary>([^\\<]+");
try {
fr = new FileReader(file);
br = new BufferedReader(fr);
StringBuilder sb = new StringBuilder(7000);
String str = null;
String id = null;
Document doc = new Document();
while ((str = br.readLine()) != null) {
sb.append(str);
//facts occur more often, do them first.
Matcher matcherFact = rxFact.matcher(str);
if (matcherFact.matches()) {
addVulnerableCpe(matcherFact.group(0), doc);
continue;
}
Matcher matcherEntry = rxEntry.matcher(str);
if (matcherEntry.matches()) {
id = matcherEntry.group(0);
Field name = new Field(Fields.CVE_ID, id, Field.Store.NO, Field.Index.ANALYZED);
name.setIndexOptions(IndexOptions.DOCS_ONLY);
doc.add(name);
continue;
}
Matcher matcherSummary = rxSummary.matcher(str);
if (matcherSummary.matches()) {
String summary = matcherSummary.group(0);
Field description = new Field(Fields.DESCRIPTION, summary, Field.Store.NO, Field.Index.ANALYZED);
description.setIndexOptions(IndexOptions.DOCS_ONLY);
doc.add(description);
continue;
}
Matcher matcherEntryEnd = rxEntryEnd.matcher(str);
if (matcherEntryEnd.matches()) {
Field xml = new Field(Fields.XML, sb.toString(), Field.Store.YES, Field.Index.NO);
doc.add(xml);
Term name = new Term(Fields.CVE_ID, LuceneUtils.escapeLuceneQuery(id));
indexWriter.updateDocument(name, doc);
doc = new Document();
}
}
} catch (FileNotFoundException ex) {
Logger.getLogger(NvdCveParser.class.getName()).log(Level.SEVERE, null, ex);
} catch (IOException ex) {
Logger.getLogger(NvdCveParser.class.getName()).log(Level.SEVERE, null, ex);
} finally {
try {
fr.close();
} catch (IOException ex) {
Logger.getLogger(NvdCveParser.class.getName()).log(Level.SEVERE, null, ex);
}
try {
if (br != null) {
br.close();
}
} catch (IOException ex) {
Logger.getLogger(NvdCveParser.class.getName()).log(Level.SEVERE, null, ex);
}
}
}
private void addVulnerableCpe(String cpe, Document doc) {
Field vulnerable = new Field(Fields.VULNERABLE_CPE, cpe, Field.Store.NO, Field.Index.ANALYZED);
vulnerable.setIndexOptions(IndexOptions.DOCS_ONLY);
doc.add(vulnerable);
}
}

8
src/main/java/org/codesecure/dependencycheck/data/nvdcve/xml/package-info.java
View File

@@ -4,7 +4,13 @@
* <title>org.codesecure.dependencycheck.data.nvdcve.xml</title>
* </head>
* <body>
* Contains classes used to parse the NVD CVE XML file.
* <p>Contains classes used to parse the NVD CVE XML file.</p>
* <p>The basic use is that the Importer is called to import
* an NVD CVE file. The Importer instantiates an Indexer object
* (which extends Index). The Indexer creates a partial-unmarshalling
* SAX parser (implemented in the NvdCveXmlFilter) that extracts
* VulnerabilityTypes (aka Entry) from the NVD CVE data file and
* stores these into a Lucene Index.</p>
* </body>
* </html>
*/

7
src/main/java/org/codesecure/dependencycheck/utils/Downloader.java
View File

@@ -112,7 +112,7 @@ public class Downloader {
int timeout = Settings.getInt(Settings.KEYS.CONNECTION_TIMEOUT);
conn.setConnectTimeout(timeout);
}
conn.setRequestProperty("Accept-Encoding", "gzip, deflate");
conn.connect();
} catch (IOException ex) {
try {
@@ -124,11 +124,10 @@ public class Downloader {
}
throw new DownloadFailedException("Error downloading file.", ex);
}
String encoding = conn.getContentEncoding();
BufferedOutputStream writer = null;
try {
//the following times out on some systems because the CPE is big.
//InputStream reader = url.openStream();
InputStream reader;
if (unzip) {
reader = new GZIPInputStream(conn.getInputStream());

61
src/test/java/org/codesecure/dependencycheck/utils/DownloaderIntegrationTest.java Normal file
View File

@@ -0,0 +1,61 @@
/*
* To change this template, choose Tools | Templates
* and open the template in the editor.
*/
package org.codesecure.dependencycheck.utils;
import java.net.URL;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
/**
*
* @author Jeremy Long (jeremy.long@gmail.com)
*/
public class DownloaderIntegrationTest {
public DownloaderIntegrationTest() {
}
@BeforeClass
public static void setUpClass() throws Exception {
}
@AfterClass
public static void tearDownClass() throws Exception {
}
@Before
public void setUp() {
}
@After
public void tearDown() {
}
/**
* Test of fetchFile method, of class Downloader.
* @throws Exception thrown when an excpetion occurs.
*/
@Test
public void testFetchFile_URL_String() throws Exception {
System.out.println("fetchFile");
// Settings.setString(Settings.KEYS.PROXY_URL, "test");
// Settings.setString(Settings.KEYS.PROXY_PORT, "80");
// Settings.setString(Settings.KEYS.CONNECTION_TIMEOUT, "1000");
Settings.setString(Settings.KEYS.PROXY_PORT, "8080");
Settings.setString(Settings.KEYS.PROXY_URL, "127.0.0.1");
//URL url = new URL(Settings.getString(Settings.KEYS.CPE_URL));
URL url = new URL("http://static.nvd.nist.gov/feeds/xml/cve/nvdcve-2.0-modified.xml");
String outputPath = "target\\downloaded_cpe.xml";
Downloader.fetchFile(url, outputPath);
}
}

57
src/test/java/org/codesecure/dependencycheck/utils/DownloaderTest.java
View File

@@ -1,57 +0,0 @@
/*
* To change this template, choose Tools | Templates
* and open the template in the editor.
*/
package org.codesecure.dependencycheck.utils;
import java.net.URL;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
/**
*
* @author Jeremy Long (jeremy.long@gmail.com)
*/
public class DownloaderTest {
public DownloaderTest() {
}
@BeforeClass
public static void setUpClass() throws Exception {
}
@AfterClass
public static void tearDownClass() throws Exception {
}
@Before
public void setUp() {
}
@After
public void tearDown() {
}
//This test is being removed because it is a bit too slow.
// /**
// * Test of fetchFile method, of class Downloader.
// * @throws Exception thrown when an excpetion occurs.
// */
// @Test
// public void testFetchFile_URL_String() throws Exception {
// System.out.println("fetchFile");
//
//// Settings.setString(Settings.KEYS.PROXY_URL, "test");
//// Settings.setString(Settings.KEYS.PROXY_PORT, "80");
//// Settings.setString(Settings.KEYS.CONNECTION_TIMEOUT, "1000");
//
// URL url = new URL(Settings.getString(Settings.KEYS.CPE_URL));
// String outputPath = "target\\downloaded_cpe.xml";
// Downloader.fetchFile(url, outputPath);
// }
}