Compare commits

..

5 Commits

Author SHA1 Message Date
Jeremy Long
76899996c2 version 0.2.3.2
Former-commit-id: 474018fefa56379cb88cfd232c4a88e8ce9b15af
2012-12-22 06:16:46 -05:00
Jeremy Long
283dcae297 added find bugs and fixed some bugs
Former-commit-id: 2266d86317f4fb20b7d3262b41b14d962916078f
2012-12-22 06:15:39 -05:00
Jeremy Long
566f36e577 continued removal of cpe.xml - it is incomplete for our purpose
Former-commit-id: 83d10942664962f0f530b4694a96c1f4f2783d43
2012-12-22 04:30:26 -05:00
Jeremy Long
6e23fd7251 bug fixes
Former-commit-id: 770436e4eff1f331b122eecd16081194987ba3f9
2012-12-20 21:39:02 -05:00
Jeremy Long
a16bcfbc10 upgrade to lucene 4.0
Former-commit-id: 0822d5816b603d8017b2fe8aa2a592aa3263c51c
2012-12-16 21:26:30 -05:00
26 changed files with 281 additions and 197 deletions

View File

@@ -7,8 +7,8 @@ If found, it will generate a report linking to the associated CVE entries.
Usage:
$ mvn package
$ cd target
$ java -jar DependencyCheck-0.2.3.1.jar -h
$ java -jar DependencyCheck-0.2.3.1.jar -a Testing -out . -scan ./test-classes/org.mortbay.jetty.jar -scan ./test-classes/struts2-core-2.1.2.jar -scan ./lib
$ java -jar DependencyCheck-0.2.3.2.jar -h
$ java -jar DependencyCheck-0.2.3.2.jar -a Testing -out . -scan ./test-classes/org.mortbay.jetty.jar -scan ./test-classes/struts2-core-2.1.2.jar -scan ./lib
Then load the resulting 'DependencyCheck-Report.html' into your favorite browser.

25
pom.xml
View File

@@ -23,7 +23,7 @@ along with DependencyCheck. If not, see <http://www.gnu.org/licenses/>.
<groupId>org.codesecure</groupId>
<artifactId>DependencyCheck</artifactId>
<version>0.2.3.1</version>
<version>0.2.3.2</version>
<packaging>jar</packaging>
<name>DependencyCheck</name>
@@ -331,6 +331,11 @@ along with DependencyCheck. If not, see <http://www.gnu.org/licenses/>.
</excludes>
</configuration>
</plugin>
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>taglist-maven-plugin</artifactId>
<version>2.4</version>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-javadoc-plugin</artifactId>
@@ -367,6 +372,11 @@ along with DependencyCheck. If not, see <http://www.gnu.org/licenses/>.
</reportSet>
</reportSets>
</plugin>
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>findbugs-maven-plugin</artifactId>
<version>2.5.2</version>
</plugin>
</reportPlugins>
</configuration>
</plugin>
@@ -389,7 +399,18 @@ along with DependencyCheck. If not, see <http://www.gnu.org/licenses/>.
<dependency>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-core</artifactId>
<version>3.5.0</version>
<version>4.0.0</version>
<!--<version>3.5.0</version>-->
</dependency>
<dependency>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-analyzers-common</artifactId>
<version>4.0.0</version>
</dependency>
<dependency>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-queryparser</artifactId>
<version>4.0.0</version>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>

View File

@@ -229,7 +229,7 @@ public class JarAnalyzer extends AbstractAnalyzer {
}
} else if (!entry.isDirectory() && "pom.properties".equals(entryName)) {
if (pomProperties == null) {
Reader reader = new InputStreamReader(zin);
Reader reader = new InputStreamReader(zin, "UTF-8");
pomProperties = new Properties();
pomProperties.load(reader);
zin.closeEntry();
@@ -482,6 +482,12 @@ public class JarAnalyzer extends AbstractAnalyzer {
protected void parseManifest(Dependency dependency) throws IOException {
JarFile jar = new JarFile(dependency.getActualFilePath());
Manifest manifest = jar.getManifest();
if (manifest == null) {
Logger.getLogger(JarAnalyzer.class.getName()).log(Level.SEVERE,
"Jar file '{0}' does not contain a manifest.",
dependency.getFileName());
return;
}
Attributes atts = manifest.getMainAttributes();
EvidenceCollection vendorEvidence = dependency.getVendorEvidence();

View File

@@ -27,8 +27,10 @@ import java.util.StringTokenizer;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.index.CorruptIndexException;
import org.apache.lucene.queryParser.ParseException;
import org.apache.lucene.queryParser.QueryParser;
import org.apache.lucene.queryparser.classic.ParseException;
import org.apache.lucene.queryparser.classic.QueryParser;
//TODO convert to the analyzing query parser
//import org.apache.lucene.queryparser.analyzing.AnalyzingQueryParser;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.ScoreDoc;
@@ -98,8 +100,7 @@ public class CPEAnalyzer implements org.codesecure.dependencycheck.analyzer.Anal
cpe.open();
indexSearcher = cpe.getIndexSearcher();
Analyzer analyzer = cpe.getAnalyzer();
//TITLE is the default field because it contains venddor, product, and version all in one.
queryParser = new QueryParser(Version.LUCENE_35, Fields.TITLE, analyzer);
queryParser = new QueryParser(Version.LUCENE_40, Fields.NAME, analyzer);
}
/**
@@ -166,13 +167,11 @@ public class CPEAnalyzer implements org.codesecure.dependencycheck.analyzer.Anal
dependency.addIdentifier(
"cpe",
e.getName(),
e.getTitle(),
"http://web.nvd.nist.gov/view/vuln/search?cpe="
+ URLEncoder.encode(e.getName(), "UTF-8"));
}
}
if (!found) {
int round = ctr % 3;
if (round == 0) {

View File

@@ -46,7 +46,6 @@ public class Entry {
Entry entry = new Entry();
try {
entry.parseName(doc.get(Fields.NAME));
entry.setTitle(doc.get(Fields.TITLE));
} catch (UnsupportedEncodingException ex) {
Logger.getLogger(Entry.class.getName()).log(Level.SEVERE, null, ex);
entry.name = doc.get(Fields.NAME);
@@ -55,6 +54,7 @@ public class Entry {
}
/**
* The title of the CPE
* @deprecated This field is no longer used
*/
protected String title;
@@ -62,6 +62,7 @@ public class Entry {
* Get the value of title
*
* @return the value of title
* @deprecated This field is no longer used
*/
public String getTitle() {
return title;
@@ -71,6 +72,7 @@ public class Entry {
* Set the value of title
*
* @param title new value of title
* @deprecated This field is no longer used
*/
public void setTitle(String title) {
this.title = title;
@@ -99,6 +101,7 @@ public class Entry {
}
/**
* The status of the CPE Entry.
* @deprecated This field is no longer used
*/
protected String status;
@@ -106,6 +109,7 @@ public class Entry {
* Get the value of status
*
* @return the value of status
* @deprecated This field is no longer used
*/
public String getStatus() {
return status;
@@ -115,31 +119,35 @@ public class Entry {
* Set the value of status
*
* @param status new value of status
* @deprecated This field is no longer used
*/
public void setStatus(String status) {
this.status = status;
}
/**
* The modification date of the CPE Entry.
* @deprecated This field is no longer used
*/
protected Date modificationDate;
private Date modificationDate;
/**
* Get the value of modificationDate
*
* @return the value of modificationDate
* @deprecated This field is no longer used
*/
public Date getModificationDate() {
return modificationDate;
return (Date) modificationDate.clone();
}
/**
* Set the value of modificationDate
*
* @param modificationDate new value of modificationDate
* @deprecated This field is no longer used
*/
public void setModificationDate(Date modificationDate) {
this.modificationDate = modificationDate;
this.modificationDate = (Date) modificationDate.clone();
}
/**
@@ -149,6 +157,7 @@ public class Entry {
*
* @param modificationDate new value of modificationDate
* @throws ParseException is thrown when a parse exception occurs.
* @deprecated This field is no longer used
*/
public void setModificationDate(String modificationDate) throws ParseException {
@@ -162,6 +171,7 @@ public class Entry {
}
/**
* The nvdId.
* @deprecated This field is no longer used
*/
protected String nvdId;
@@ -169,6 +179,7 @@ public class Entry {
* Get the value of nvdId
*
* @return the value of nvdId
* @deprecated This field is no longer used
*/
public String getNvdId() {
return nvdId;
@@ -178,6 +189,7 @@ public class Entry {
* Set the value of nvdId
*
* @param nvdId new value of nvdId
* @deprecated This field is no longer used
*/
public void setNvdId(String nvdId) {
this.nvdId = nvdId;

View File

@@ -34,22 +34,13 @@ public abstract class Fields {
* The key for the vendor field.
*/
public static final String VENDOR = "vendor";
/**
* The key for the version field.
*/
public static final String VERSION = "version";
//public static final String REVISION = "revision";
/**
* The key for the product field.
*/
public static final String PRODUCT = "product";
/**
* The key for the title field. This is a field combining vendor, product,
* and version.
* The key for the version field.
*/
public static final String TITLE = "title";
/**
* The key for the nvdId field.
*/
public static final String NVDID = "nvdid";
public static final String VERSION = "version";
//public static final String REVISION = "revision";
}

View File

@@ -36,8 +36,8 @@ import java.util.logging.Level;
import java.util.logging.Logger;
import javax.xml.parsers.ParserConfigurationException;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.KeywordAnalyzer;
import org.apache.lucene.analysis.PerFieldAnalyzerWrapper;
import org.apache.lucene.analysis.core.KeywordAnalyzer;
import org.apache.lucene.analysis.miscellaneous.PerFieldAnalyzerWrapper;
import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory;
@@ -88,7 +88,7 @@ public class Index extends AbstractIndex implements CachedWebDataSource {
* @return the data directory for this index.
* @throws IOException is thrown if an IOException occurs of course...
*/
protected File getDataDirectory() throws IOException {
public File getDataDirectory() throws IOException {
String fileName = Settings.getString(Settings.KEYS.CPE_INDEX);
String filePath = Index.class.getProtectionDomain().getCodeSource().getLocation().getPath();
String decodedPath = URLDecoder.decode(filePath, "UTF-8");
@@ -121,7 +121,7 @@ public class Index extends AbstractIndex implements CachedWebDataSource {
fieldAnalyzers.put(Fields.NAME, new KeywordAnalyzer());
PerFieldAnalyzerWrapper wrapper = new PerFieldAnalyzerWrapper(
new StandardAnalyzer(Version.LUCENE_35), fieldAnalyzers);
new StandardAnalyzer(Version.LUCENE_40), fieldAnalyzers);
return wrapper;
}
@@ -132,6 +132,8 @@ public class Index extends AbstractIndex implements CachedWebDataSource {
*
* @throws UpdateException is thrown if there is a problem updating the
* index.
*
* @deprecated this should no longer be used as the raw CPE hosted at NIST is not complete enough.
*/
public void update() throws UpdateException {
try {
@@ -180,6 +182,8 @@ public class Index extends AbstractIndex implements CachedWebDataSource {
* directory.
*
* @param timeStamp the timestamp to write.
*
* @deprecated this should no longer be used as the raw CPE hosted at NIST is not complete enough.
*/
private void writeLastUpdatedPropertyFile(long timeStamp) throws UpdateException {
String dir;
@@ -193,24 +197,27 @@ public class Index extends AbstractIndex implements CachedWebDataSource {
Properties prop = new Properties();
prop.put(Index.LAST_UPDATED, String.valueOf(timeStamp));
OutputStream os = null;
OutputStreamWriter out = null;
try {
os = new FileOutputStream(cpeProp);
OutputStreamWriter out = new OutputStreamWriter(os);
out = new OutputStreamWriter(os, "UTF-8");
prop.store(out, dir);
} catch (FileNotFoundException ex) {
Logger.getLogger(Index.class.getName()).log(Level.SEVERE, null, ex);
} catch (IOException ex) {
Logger.getLogger(Index.class.getName()).log(Level.SEVERE, null, ex);
} finally {
try {
os.flush();
} catch (IOException ex) {
Logger.getLogger(Index.class.getName()).log(Level.SEVERE, null, ex);
}
try {
os.close();
} catch (IOException ex) {
Logger.getLogger(Index.class.getName()).log(Level.SEVERE, null, ex);
if (os != null) {
try {
os.flush();
} catch (IOException ex) {
Logger.getLogger(Index.class.getName()).log(Level.SEVERE, null, ex);
}
try {
os.close();
} catch (IOException ex) {
Logger.getLogger(Index.class.getName()).log(Level.SEVERE, null, ex);
}
}
}
}
@@ -229,6 +236,8 @@ public class Index extends AbstractIndex implements CachedWebDataSource {
* downloading the cpe.meta data file.
* @throws UpdateException is thrown if there is an error locating the last updated
* properties file.
*
* @deprecated this should no longer be used as the raw CPE hosted at NIST is not complete enough.
*/
public long updateNeeded() throws MalformedURLException, DownloadFailedException, UpdateException {
long retVal = 0;
@@ -271,6 +280,14 @@ public class Index extends AbstractIndex implements CachedWebDataSource {
Logger.getLogger(Index.class.getName()).log(Level.FINEST, null, ex);
} catch (NumberFormatException ex) {
Logger.getLogger(Index.class.getName()).log(Level.FINEST, null, ex);
} finally {
if (is != null) {
try {
is.close();
} catch (IOException ex) {
Logger.getLogger(Index.class.getName()).log(Level.SEVERE, null, ex);
}
}
}
if (currentlyPublishedDate > lastUpdated) {
retVal = currentlyPublishedDate;

View File

@@ -21,8 +21,9 @@ package org.codesecure.dependencycheck.data.cpe.xml;
import java.io.IOException;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.StoredField;
import org.apache.lucene.document.TextField;
import org.apache.lucene.index.CorruptIndexException;
import org.apache.lucene.index.FieldInfo.IndexOptions;
import org.apache.lucene.index.Term;
import org.codesecure.dependencycheck.data.cpe.Entry;
import org.codesecure.dependencycheck.data.cpe.Fields;
@@ -59,44 +60,30 @@ public class Indexer extends Index implements EntrySaveDelegate {
protected Document convertEntryToDoc(Entry entry) {
Document doc = new Document();
Field name = new Field(Fields.NAME, entry.getName(), Field.Store.YES, Field.Index.ANALYZED);
name.setIndexOptions(IndexOptions.DOCS_ONLY);
Field name = new StoredField(Fields.NAME, entry.getName());
doc.add(name);
Field nvdId = new Field(Fields.NVDID, entry.getNvdId(), Field.Store.NO, Field.Index.ANALYZED);
nvdId.setIndexOptions(IndexOptions.DOCS_ONLY);
doc.add(nvdId);
Field vendor = new Field(Fields.VENDOR, entry.getVendor(), Field.Store.NO, Field.Index.ANALYZED);
vendor.setIndexOptions(IndexOptions.DOCS_ONLY);
Field vendor = new TextField(Fields.VENDOR, entry.getVendor(), Field.Store.NO);
vendor.setBoost(5.0F);
doc.add(vendor);
Field product = new Field(Fields.PRODUCT, entry.getProduct(), Field.Store.NO, Field.Index.ANALYZED);
product.setIndexOptions(IndexOptions.DOCS_ONLY);
Field product = new TextField(Fields.PRODUCT, entry.getProduct(), Field.Store.NO);
product.setBoost(5.0F);
doc.add(product);
Field title = new Field(Fields.TITLE, entry.getTitle(), Field.Store.YES, Field.Index.ANALYZED);
title.setIndexOptions(IndexOptions.DOCS_ONLY);
//title.setBoost(1.0F);
doc.add(title);
//TODO revision should likely be its own field
if (entry.getVersion() != null) {
Field version = null;
if (entry.getRevision() != null) {
version = new Field(Fields.VERSION, entry.getVersion() + " "
+ entry.getRevision(), Field.Store.NO, Field.Index.ANALYZED);
version = new TextField(Fields.VERSION, entry.getVersion() + " "
+ entry.getRevision(), Field.Store.NO);
} else {
version = new Field(Fields.VERSION, entry.getVersion(),
Field.Store.NO, Field.Index.ANALYZED);
version = new TextField(Fields.VERSION, entry.getVersion(),
Field.Store.NO);
}
version.setIndexOptions(IndexOptions.DOCS_ONLY);
version.setBoost(0.8F);
doc.add(version);
}
return doc;
}
}

View File

@@ -23,6 +23,7 @@ import java.util.logging.Level;
import java.util.logging.Logger;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.index.CorruptIndexException;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
@@ -98,13 +99,7 @@ public abstract class AbstractIndex {
}
}
if (indexSearcher != null) {
try {
indexSearcher.close();
} catch (IOException ex) {
Logger.getLogger(AbstractIndex.class.getName()).log(Level.SEVERE, null, ex);
} finally {
indexSearcher = null;
}
indexSearcher = null;
}
if (analyzer != null) {
@@ -140,7 +135,7 @@ public abstract class AbstractIndex {
if (!isOpen()) {
open();
}
IndexWriterConfig conf = new IndexWriterConfig(Version.LUCENE_35, analyzer);
IndexWriterConfig conf = new IndexWriterConfig(Version.LUCENE_40, analyzer);
indexWriter = new IndexWriter(directory, conf);
}
@@ -170,7 +165,8 @@ public abstract class AbstractIndex {
if (!isOpen()) {
open();
}
indexReader = IndexReader.open(directory, true);
//indexReader = IndexReader.open(directory, true);
indexReader = DirectoryReader.open(directory);
}
/**

View File

@@ -18,7 +18,7 @@ package org.codesecure.dependencycheck.data.lucene;
* Copyright (c) 2012 Jeremy Long. All Rights Reserved.
*/
import org.apache.lucene.search.DefaultSimilarity;
import org.apache.lucene.search.similarities.DefaultSimilarity;
/**
*
@@ -41,7 +41,7 @@ public class DependencySimilarity extends DefaultSimilarity {
* @return 1
*/
@Override
public float idf(int docFreq, int numDocs) {
public float idf(long docFreq, long numDocs) {
return 1;
}
}

View File

@@ -28,8 +28,8 @@ import java.util.logging.Logger;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.KeywordAnalyzer;
import org.apache.lucene.analysis.PerFieldAnalyzerWrapper;
import org.apache.lucene.analysis.core.KeywordAnalyzer;
import org.apache.lucene.analysis.miscellaneous.PerFieldAnalyzerWrapper;
import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory;
@@ -40,6 +40,7 @@ import org.codesecure.dependencycheck.data.lucene.AbstractIndex;
import org.codesecure.dependencycheck.data.nvdcve.xml.Importer;
import org.codesecure.dependencycheck.utils.DownloadFailedException;
import org.codesecure.dependencycheck.utils.Downloader;
import org.codesecure.dependencycheck.utils.FileUtils;
import org.codesecure.dependencycheck.utils.Settings;
/**
@@ -49,6 +50,10 @@ import org.codesecure.dependencycheck.utils.Settings;
*/
public class Index extends AbstractIndex implements CachedWebDataSource {
/**
* The current version of the index
*/
public static final String INDEX_VERSION = "1.0";
/**
* The name of the properties file containing the timestamp of the last
* update.
@@ -119,7 +124,7 @@ public class Index extends AbstractIndex implements CachedWebDataSource {
fieldAnalyzers.put(Fields.VULNERABLE_CPE, new KeywordAnalyzer());
PerFieldAnalyzerWrapper wrapper = new PerFieldAnalyzerWrapper(
new StandardAnalyzer(Version.LUCENE_35), fieldAnalyzers);
new StandardAnalyzer(Version.LUCENE_40), fieldAnalyzers);
return wrapper;
}
@@ -203,7 +208,7 @@ public class Index extends AbstractIndex implements CachedWebDataSource {
}
File cveProp = new File(dir + File.separatorChar + UPDATE_PROPERTIES_FILE);
Properties prop = new Properties();
prop.put("version", INDEX_VERSION);
for (NvdCveUrl cve : updated.values()) {
prop.put(LAST_UPDATED_BASE + cve.id, String.valueOf(cve.getTimestamp()));
}
@@ -211,7 +216,7 @@ public class Index extends AbstractIndex implements CachedWebDataSource {
OutputStream os = null;
try {
os = new FileOutputStream(cveProp);
OutputStreamWriter out = new OutputStreamWriter(os);
OutputStreamWriter out = new OutputStreamWriter(os, "UTF-8");
prop.store(out, dir);
} catch (FileNotFoundException ex) {
Logger.getLogger(Index.class.getName()).log(Level.SEVERE, null, ex);
@@ -220,15 +225,17 @@ public class Index extends AbstractIndex implements CachedWebDataSource {
Logger.getLogger(Index.class.getName()).log(Level.SEVERE, null, ex);
throw new UpdateException("Unable to update last updated properties file.", ex);
} finally {
try {
os.flush();
} catch (IOException ex) {
Logger.getLogger(Index.class.getName()).log(Level.SEVERE, null, ex);
}
try {
os.close();
} catch (IOException ex) {
Logger.getLogger(Index.class.getName()).log(Level.SEVERE, null, ex);
if (os != null) {
try {
os.flush();
} catch (IOException ex) {
Logger.getLogger(Index.class.getName()).log(Level.SEVERE, null, ex);
}
try {
os.close();
} catch (IOException ex) {
Logger.getLogger(Index.class.getName()).log(Level.SEVERE, null, ex);
}
}
}
}
@@ -271,10 +278,23 @@ public class Index extends AbstractIndex implements CachedWebDataSource {
File cveProp = new File(dir + File.separatorChar + UPDATE_PROPERTIES_FILE);
if (cveProp.exists()) {
Properties prop = new Properties();
InputStream is;
InputStream is = null;
try {
is = new FileInputStream(cveProp);
prop.load(is);
if (prop.getProperty("version") == null) {
is.close();
//this is an old version of the lucene index - just delete it
FileUtils.delete(f);
//this importer also updates the CPE index and it is also using an old version
org.codesecure.dependencycheck.data.cpe.Index cpeidx = new org.codesecure.dependencycheck.data.cpe.Index();
File cpeDir = cpeidx.getDataDirectory();
FileUtils.delete(cpeDir);
return currentlyPublished;
}
long lastUpdated = Long.parseLong(prop.getProperty(Index.LAST_UPDATED_MODIFIED));
Date now = new Date();
int days = Settings.getInt(Settings.KEYS.CVE_MODIFIED_VALID_FOR_DAYS);
@@ -308,6 +328,14 @@ public class Index extends AbstractIndex implements CachedWebDataSource {
Logger.getLogger(Index.class.getName()).log(Level.FINEST, null, ex);
} catch (NumberFormatException ex) {
Logger.getLogger(Index.class.getName()).log(Level.FINEST, null, ex);
} finally {
if (is != null) {
try {
is.close();
} catch (IOException ex) {
Logger.getLogger(Index.class.getName()).log(Level.SEVERE, null, ex);
}
}
}
}
}
@@ -447,7 +475,7 @@ public class Index extends AbstractIndex implements CachedWebDataSource {
* @throws IOException is thrown if an IOExcpetion occurs.
*/
private String readFile(File file) throws IOException {
FileReader stream = new FileReader(file);
InputStreamReader stream = new InputStreamReader(new FileInputStream(file), "UTF-8");
StringBuilder str = new StringBuilder((int) file.length());
try {
char[] buf = new char[8096];

View File

@@ -20,6 +20,7 @@ package org.codesecure.dependencycheck.data.nvdcve;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.util.Set;
import java.util.logging.Level;
import java.util.logging.Logger;
@@ -142,6 +143,9 @@ public class NvdCveAnalyzer implements org.codesecure.dependencycheck.analyzer.A
} catch (JAXBException ex) {
Logger.getLogger(NvdCveAnalyzer.class.getName()).log(Level.SEVERE, null, ex);
dependency.addAnalysisException(new AnalysisException("Unable to retrieve vulnerability data", ex));
} catch (UnsupportedEncodingException ex) {
Logger.getLogger(NvdCveAnalyzer.class.getName()).log(Level.SEVERE, null, ex);
dependency.addAnalysisException(new AnalysisException("Unable to retrieve vulnerability data - utf-8", ex));
}
}
} catch (IOException ex) {
@@ -198,11 +202,11 @@ public class NvdCveAnalyzer implements org.codesecure.dependencycheck.analyzer.A
this.open();
}
private Vulnerability parseVulnerability(String xml) throws JAXBException {
private Vulnerability parseVulnerability(String xml) throws JAXBException, UnsupportedEncodingException {
JAXBContext jaxbContext = JAXBContext.newInstance(VulnerabilityType.class);
Unmarshaller unmarshaller = jaxbContext.createUnmarshaller();
ByteArrayInputStream input = new ByteArrayInputStream(xml.getBytes());
ByteArrayInputStream input = new ByteArrayInputStream(xml.getBytes("UTF-8"));
VulnerabilityType cvedata = (VulnerabilityType) unmarshaller.unmarshal(input);
if (cvedata == null) {
return null;

View File

@@ -20,6 +20,7 @@ package org.codesecure.dependencycheck.data.nvdcve.xml;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.util.logging.Level;
import java.util.logging.Logger;
import javax.xml.bind.JAXBContext;
@@ -27,8 +28,9 @@ import javax.xml.bind.JAXBException;
import javax.xml.bind.Marshaller;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.StoredField;
import org.apache.lucene.document.StringField;
import org.apache.lucene.index.CorruptIndexException;
import org.apache.lucene.index.FieldInfo.IndexOptions;
import org.apache.lucene.index.Term;
import org.codesecure.dependencycheck.data.lucene.LuceneUtils;
import org.codesecure.dependencycheck.data.nvdcve.generated.VulnerabilityType;
@@ -55,7 +57,12 @@ public class Indexer extends Index implements EntrySaveDelegate {
*/
public void saveEntry(VulnerabilityType vulnerability) throws CorruptIndexException, IOException {
try {
Document doc = convertEntryToDoc(vulnerability);
Document doc = null;
try {
doc = convertEntryToDoc(vulnerability);
} catch (UnsupportedEncodingException ex) {
Logger.getLogger(Indexer.class.getName()).log(Level.SEVERE, null, ex);
}
if (doc == null) {
return;
@@ -74,8 +81,9 @@ public class Indexer extends Index implements EntrySaveDelegate {
* @param vulnerability a VULNERABLE_CPE Entry.
* @return a Lucene Document containing a VULNERABLE_CPE Entry.
* @throws JAXBException is thrown when there is a JAXBException.
* @throws UnsupportedEncodingException if the system doesn't support utf-8
*/
protected Document convertEntryToDoc(VulnerabilityType vulnerability) throws JAXBException {
protected Document convertEntryToDoc(VulnerabilityType vulnerability) throws JAXBException, UnsupportedEncodingException {
boolean hasApplication = false;
Document doc = new Document();
@@ -101,14 +109,11 @@ public class Indexer extends Index implements EntrySaveDelegate {
return null;
}
Field name = new Field(Fields.CVE_ID, vulnerability.getId(), Field.Store.NO, Field.Index.ANALYZED);
name.setIndexOptions(IndexOptions.DOCS_ONLY);
Field name = new StringField(Fields.CVE_ID, vulnerability.getId(), Field.Store.NO);
doc.add(name);
Field description = new Field(Fields.DESCRIPTION, vulnerability.getSummary(), Field.Store.NO, Field.Index.ANALYZED);
description.setIndexOptions(IndexOptions.DOCS_ONLY);
doc.add(description);
// Field description = new Field(Fields.DESCRIPTION, vulnerability.getSummary(), Field.Store.NO, Field.Index.ANALYZED);
// doc.add(description);
JAXBContext context = JAXBContext.newInstance("org.codesecure.dependencycheck.data.nvdcve.generated");
@@ -119,7 +124,7 @@ public class Indexer extends Index implements EntrySaveDelegate {
m.marshal(vulnerability, out);
Field xml = new Field(Fields.XML, out.toString(), Field.Store.YES, Field.Index.NO);
Field xml = new StoredField(Fields.XML, out.toString("UTF-8"));
doc.add(xml);
return doc;
@@ -141,8 +146,7 @@ public class Indexer extends Index implements EntrySaveDelegate {
}
private void addVulnerableCpe(String cpe, Document doc) {
Field vulnerable = new Field(Fields.VULNERABLE_CPE, cpe, Field.Store.NO, Field.Index.ANALYZED);
vulnerable.setIndexOptions(IndexOptions.DOCS_ONLY);
Field vulnerable = new StringField(Fields.VULNERABLE_CPE, cpe, Field.Store.NO);
doc.add(vulnerable);
}
}

View File

@@ -20,9 +20,10 @@ package org.codesecure.dependencycheck.data.nvdcve.xml;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.UnsupportedEncodingException;
import java.util.logging.Level;
import java.util.logging.Logger;
@@ -30,8 +31,9 @@ import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.StoredField;
import org.apache.lucene.document.StringField;
import org.apache.lucene.index.CorruptIndexException;
import org.apache.lucene.index.FieldInfo.IndexOptions;
import org.apache.lucene.index.Term;
import org.codesecure.dependencycheck.data.cpe.Entry;
import org.codesecure.dependencycheck.data.nvdcve.Fields;
@@ -65,14 +67,15 @@ public class NvdCveParser extends Index {
* @param file the reference to the NVD CVE file
*/
public void parse(File file) {
FileReader fr = null;
InputStreamReader fr = null;
BufferedReader br = null;
Pattern rxEntry = Pattern.compile("^\\s*<entry\\s*id\\=\\\"([^\\\"]+)\\\".*$");
Pattern rxEntryEnd = Pattern.compile("^\\s*</entry>.*$");
Pattern rxFact = Pattern.compile("^\\s*<cpe\\-lang\\:fact\\-ref name=\\\"([^\\\"]+).*$");
Pattern rxSummary = Pattern.compile("^\\s*<vuln:summary>([^\\<]+).*$");
//Pattern rxSummary = Pattern.compile("^\\s*<vuln:summary>([^\\<]+).*$");
try {
fr = new FileReader(file);
fr = new InputStreamReader(new FileInputStream(file), "UTF-8");
br = new BufferedReader(fr);
StringBuilder sb = new StringBuilder(7000);
String str = null;
@@ -121,23 +124,21 @@ public class NvdCveParser extends Index {
sb.append("id=\"").append(id).append("\">");
//sb.append(str); //need to do the above to get the correct schema generated from files.
Field name = new Field(Fields.CVE_ID, id, Field.Store.NO, Field.Index.ANALYZED);
name.setIndexOptions(IndexOptions.DOCS_ONLY);
Field name = new StringField(Fields.CVE_ID, id, Field.Store.NO);
doc.add(name);
continue;
}
Matcher matcherSummary = rxSummary.matcher(str);
if (matcherSummary.matches()) {
String summary = matcherSummary.group(1);
Field description = new Field(Fields.DESCRIPTION, summary, Field.Store.NO, Field.Index.ANALYZED);
description.setIndexOptions(IndexOptions.DOCS_ONLY);
doc.add(description);
continue;
}
// Matcher matcherSummary = rxSummary.matcher(str);
// if (matcherSummary.matches()) {
// String summary = matcherSummary.group(1);
// Field description = new Field(Fields.DESCRIPTION, summary, Field.Store.NO);
// doc.add(description);
// continue;
// }
if (matcherEntryEnd.matches()) {
sb.append("</vulnerabilityType>");
Field xml = new Field(Fields.XML, sb.toString(), Field.Store.YES, Field.Index.NO);
Field xml = new StoredField(Fields.XML, sb.toString());
doc.add(xml);
if (!skipEntry) {
@@ -162,7 +163,9 @@ public class NvdCveParser extends Index {
Logger.getLogger(NvdCveParser.class.getName()).log(Level.SEVERE, null, ex);
} finally {
try {
fr.close();
if (fr != null) {
fr.close();
}
} catch (IOException ex) {
Logger.getLogger(NvdCveParser.class.getName()).log(Level.SEVERE, null, ex);
}
@@ -184,8 +187,7 @@ public class NvdCveParser extends Index {
* @throws IOException is thrown if there is an IO Exception while writting to the CPE Index
*/
private void addVulnerableCpe(String cpe, Document doc) throws CorruptIndexException, IOException {
Field vulnerable = new Field(Fields.VULNERABLE_CPE, cpe, Field.Store.NO, Field.Index.ANALYZED);
vulnerable.setIndexOptions(IndexOptions.DOCS_ONLY);
Field vulnerable = new StringField(Fields.VULNERABLE_CPE, cpe, Field.Store.NO);
doc.add(vulnerable);
//HACK - this has initially been placed here as a hack because not all
@@ -194,8 +196,6 @@ public class NvdCveParser extends Index {
Entry cpeEntry = new Entry();
try {
cpeEntry.parseName(cpe);
cpeEntry.setNvdId("0");
cpeEntry.setTitle(cpe);
} catch (UnsupportedEncodingException ex) {
Logger.getLogger(NvdCveParser.class.getName()).log(Level.SEVERE, null, ex);
}

View File

@@ -19,7 +19,7 @@ package org.codesecure.dependencycheck.dependency;
*/
import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.security.NoSuchAlgorithmException;
import java.util.ArrayList;
import java.util.List;
@@ -238,11 +238,10 @@ public class Dependency {
*
* @param type the type of identifier (such as CPE).
* @param value the value of the identifier.
* @param title the title of the identifier.
* @param url the URL of the identifier.
*/
public void addIdentifier(String type, String value, String title, String url) {
Identifier i = new Identifier(type, value, title, url);
public void addIdentifier(String type, String value, String url) {
Identifier i = new Identifier(type, value, url);
this.identifiers.add(i);
}
@@ -419,7 +418,7 @@ public class Dependency {
try {
md5 = Checksum.getMD5Checksum(file);
sha1 = Checksum.getSHA1Checksum(file);
} catch (FileNotFoundException ex) {
} catch (IOException ex) {
Logger.getLogger(Dependency.class.getName()).log(Level.SEVERE, null, ex);
} catch (NoSuchAlgorithmException ex) {
Logger.getLogger(Dependency.class.getName()).log(Level.SEVERE, null, ex);

View File

@@ -29,13 +29,11 @@ public class Identifier {
*
* @param type the identifier type.
* @param value the identifier value.
* @param title the identifier title.
* @param url the identifier url.
*/
Identifier(String type, String value, String title, String url) {
Identifier(String type, String value, String url) {
this.type = type;
this.value = value;
this.title = title;
this.url = url;
}
@@ -44,12 +42,11 @@ public class Identifier {
*
* @param type the identifier type.
* @param value the identifier value.
* @param title the identifier title.
* @param url the identifier url.
* @param description the description of the identifier.
*/
Identifier(String type, String value, String title, String url, String description) {
this(type, value, title, url);
Identifier(String type, String value, String url, String description) {
this(type, value, url);
this.description = description;
}
/**
@@ -74,28 +71,7 @@ public class Identifier {
public void setValue(String value) {
this.value = value;
}
/**
* The title of the identifeir
*/
protected String title;
/**
* Get the value of title
*
* @return the value of title
*/
public String getTitle() {
return title;
}
/**
* Set the value of title
*
* @param title new value of title
*/
public void setTitle(String title) {
this.title = title;
}
/**
* The url for the identifeir
*/

View File

@@ -19,13 +19,14 @@ package org.codesecure.dependencycheck.reporting;
*/
import java.io.FileInputStream;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileWriter;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.OutputStream;
import java.io.OutputStreamWriter;
import java.util.List;
import java.util.logging.Level;
import java.util.logging.Logger;
@@ -94,9 +95,7 @@ public class ReportGenerator {
Context c = manager.createContext();
EasyFactoryConfiguration config = new EasyFactoryConfiguration();
config.addDefaultTools();
config.toolbox("application")
.tool("esc", "org.apache.velocity.tools.generic.EscapeTool")
.tool("org.apache.velocity.tools.generic.DateTool");
config.toolbox("application").tool("esc", "org.apache.velocity.tools.generic.EscapeTool").tool("org.apache.velocity.tools.generic.DateTool");
manager.configure(config);
return c;
}
@@ -143,21 +142,33 @@ public class ReportGenerator {
throw new IOException("Template file doesn't exist");
}
InputStreamReader reader = new InputStreamReader(input);
BufferedWriter writer = null;
InputStreamReader reader = new InputStreamReader(input, "UTF-8");
OutputStreamWriter writer = null;
OutputStream outputStream = null;
try {
writer = new BufferedWriter(new FileWriter(new File(outFileName)));
outputStream = new FileOutputStream(outFileName);
writer = new OutputStreamWriter(outputStream, "UTF-8");
//writer = new BufferedWriter(oswriter);
if (!engine.evaluate(context, writer, templatePath, reader)) {
throw new Exception("Failed to convert the template into html.");
}
writer.flush();
} finally {
try {
writer.close();
} catch (Exception ex) {
Logger.getLogger(ReportGenerator.class.getName()).log(Level.FINEST, null, ex);
if (writer != null) {
try {
writer.close();
} catch (Exception ex) {
Logger.getLogger(ReportGenerator.class.getName()).log(Level.FINEST, null, ex);
}
}
if (outputStream != null) {
try {
outputStream.close();
} catch (Exception ex) {
Logger.getLogger(ReportGenerator.class.getName()).log(Level.FINEST, null, ex);
}
}
try {
reader.close();

View File

@@ -2,7 +2,6 @@ package org.codesecure.dependencycheck.utils;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
import java.security.MessageDigest;
@@ -30,29 +29,30 @@ public class Checksum {
* @param algorithm the algorithm to use to calculate the checksum
* @param file the file to calculate the checksum for
* @return the checksum
* @throws FileNotFoundException when the file does not exist
* @throws IOException when the file does not exist
* @throws NoSuchAlgorithmException when an algorithm is specified that does
* not exist
*/
public static byte[] getChecksum(String algorithm, File file) throws FileNotFoundException, NoSuchAlgorithmException {
InputStream fis = new FileInputStream(file);
public static byte[] getChecksum(String algorithm, File file) throws NoSuchAlgorithmException, IOException {
InputStream fis = null;
byte[] buffer = new byte[1024];
MessageDigest complete = MessageDigest.getInstance(algorithm);
int numRead;
try {
fis = new FileInputStream(file);
do {
numRead = fis.read(buffer);
if (numRead > 0) {
complete.update(buffer, 0, numRead);
}
} while (numRead != -1);
} catch (IOException ex) {
Logger.getLogger(Checksum.class.getName()).log(Level.SEVERE, null, ex);
} finally {
try {
fis.close();
} catch (IOException ex) {
Logger.getLogger(Checksum.class.getName()).log(Level.SEVERE, null, ex);
if (fis != null) {
try {
fis.close();
} catch (IOException ex) {
Logger.getLogger(Checksum.class.getName()).log(Level.SEVERE, null, ex);
}
}
}
return complete.digest();
@@ -63,10 +63,10 @@ public class Checksum {
*
* @param file the file to generate the MD5 checksum
* @return the hex representation of the MD5 hash
* @throws FileNotFoundException when the file passed in does not exist
* @throws IOException when the file passed in does not exist
* @throws NoSuchAlgorithmException when the MD5 algorithm is not available
*/
public static String getMD5Checksum(File file) throws FileNotFoundException, NoSuchAlgorithmException {
public static String getMD5Checksum(File file) throws IOException, NoSuchAlgorithmException {
byte[] b = getChecksum("MD5", file);
return getHex(b);
}
@@ -76,10 +76,10 @@ public class Checksum {
*
* @param file the file to generate the MD5 checksum
* @return the hex representation of the SHA1 hash
* @throws FileNotFoundException when the file passed in does not exist
* @throws IOException when the file passed in does not exist
* @throws NoSuchAlgorithmException when the SHA1 algorithm is not available
*/
public static String getSHA1Checksum(File file) throws FileNotFoundException, NoSuchAlgorithmException {
public static String getSHA1Checksum(File file) throws IOException, NoSuchAlgorithmException {
byte[] b = getChecksum("SHA1", file);
return getHex(b);
}

View File

@@ -128,8 +128,8 @@ public class Downloader {
String encoding = conn.getContentEncoding();
BufferedOutputStream writer = null;
InputStream reader = null;
try {
InputStream reader;
if (unzip || (encoding != null && "gzip".equalsIgnoreCase(encoding))) {
reader = new GZIPInputStream(conn.getInputStream());
} else if (encoding != null && "deflate".equalsIgnoreCase(encoding)) {
@@ -147,6 +147,7 @@ public class Downloader {
} catch (Exception ex) {
throw new DownloadFailedException("Error saving downloaded file.", ex);
} finally {
if (writer != null) {
try {
writer.close();
writer = null;
@@ -154,6 +155,17 @@ public class Downloader {
Logger.getLogger(Downloader.class.getName()).log(Level.FINEST,
"Error closing the writter in Downloader.", ex);
}
}
if (reader != null) {
try {
reader.close();
reader = null;
} catch (Exception ex) {
Logger.getLogger(Downloader.class.getName()).log(Level.FINEST,
"Error closing the reader in Downloader.", ex);
}
}
try {
conn.disconnect();
} finally {

View File

@@ -18,6 +18,10 @@ package org.codesecure.dependencycheck.utils;
* Copyright (c) 2012 Jeremy Long. All Rights Reserved.
*/
import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
/**
* A collection of utilities for processing information about files.
*
@@ -45,4 +49,22 @@ public class FileUtils {
}
return ret;
}
/**
* Deletes a file. If the File is a directory it will recursively delete
* the contents.
*
* @param file the File to delete
* @throws IOException is thrown if the file could not be deleted
*/
public static void delete(File file) throws IOException {
if (file.isDirectory()) {
for (File c : file.listFiles()) {
delete(c);
}
}
if (!file.delete()) {
throw new FileNotFoundException("Failed to delete file: " + file);
}
}
}

View File

@@ -1,2 +1 @@
org.codesecure.dependencycheck.data.nvdcve.Index
org.codesecure.dependencycheck.data.cpe.Index
org.codesecure.dependencycheck.data.nvdcve.Index

View File

@@ -370,7 +370,7 @@ Copyright (c) 2012 Jeremy Long. All Rights Reserved.
<ul>
#foreach($id in $dependency.getIdentifiers())
##yes, we are HTML Encoding the href. this is okay. We can't URL encode as we have to trust the analyzer here...
<li><b>$esc.html($id.type):</b>&nbsp;$esc.html($id.title)&nbsp;:&nbsp;<a href="$esc.html($id.url)" target="_blank">$esc.html($id.value)</a>
<li><b>$esc.html($id.type):</b>&nbsp;<a href="$esc.html($id.url)" target="_blank">$esc.html($id.value)</a>
#if( $id.descrription )
<br/>$esc.html($id.description)
#end

View File

@@ -10,7 +10,7 @@ import java.util.HashSet;
import java.util.List;
import java.util.Set;
import org.apache.lucene.index.CorruptIndexException;
import org.apache.lucene.queryParser.ParseException;
import org.apache.lucene.queryparser.classic.ParseException;
import org.codesecure.dependencycheck.dependency.Dependency;
import org.codesecure.dependencycheck.analyzer.JarAnalyzer;
import org.junit.Test;

View File

@@ -46,8 +46,9 @@ public class IndexIntegrationTest extends BaseIndexTestCase {
@Test
public void testUpdate() throws Exception {
System.out.println("update");
Index instance = new Index();
instance.update();
//deprecated
//Index instance = new Index();
//instance.update();
}
/**
@@ -56,8 +57,9 @@ public class IndexIntegrationTest extends BaseIndexTestCase {
@Test
public void testUpdateNeeded() throws Exception {
System.out.println("updateNeeded");
Index instance = new Index();
instance.updateNeeded();
//deprecated
//Index instance = new Index();
//instance.updateNeeded();
//if an exception is thrown this test fails. However, because it depends on the
// order of the tests what this will return I am just testing for the exception.
//assertTrue(expResult < result);

View File

@@ -249,15 +249,13 @@ public class DependencyTest {
System.out.println("addIdentifier");
String type = "cpe";
String value = "cpe:/a:apache:struts:2.1.2";
String title = "Apache Struts 2.1.2";
String url = "http://somewhere";
Dependency instance = new Dependency();
instance.addIdentifier(type, value, title, url);
instance.addIdentifier(type, value, url);
assertEquals(1,instance.getIdentifiers().size());
Identifier i = instance.getIdentifiers().get(0);
assertEquals(type,i.getType());
assertEquals(value, i.getValue());
assertEquals(title, i.getTitle());
assertEquals(url, i.getUrl());
}

View File

@@ -5,7 +5,7 @@
package org.codesecure.dependencycheck.utils;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.security.NoSuchAlgorithmException;
import junit.framework.TestCase;
import org.junit.Test;
@@ -66,7 +66,7 @@ public class ChecksumTest extends TestCase {
boolean exceptionThrown = false;
try {
byte[] result = Checksum.getChecksum(algorithm, file);
} catch (FileNotFoundException ex) {
} catch (IOException ex) {
exceptionThrown = true;
}
assertTrue(exceptionThrown);