github-mirrors/DependencyCheck
1
0
Fork 0
mirror of https://github.com/ysoftdevs/DependencyCheck.git synced 2026-01-15 08:13:43 +01:00
Code Issues Packages Projects Releases Wiki Activity

bug fixes/enhancements

Browse Source
This commit is contained in:
Jeremy Long
2012-09-10 12:25:15 -04:00
parent dbc10e53e4
commit 4147801074
10 changed files with 216 additions and 82 deletions
Download Patch File Download Diff File Expand all files Collapse all files

64
src/main/java/org/codesecure/dependencycheck/data/cpe/CPEQuery.java
View File

@@ -21,6 +21,8 @@ package org.codesecure.dependencycheck.data.cpe;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.Set;
import java.util.StringTokenizer;
import java.util.logging.Level;
import java.util.logging.Logger;
import org.apache.lucene.analysis.Analyzer;
@@ -37,7 +39,9 @@ import org.apache.lucene.store.Directory;
import org.apache.lucene.util.Version;
import org.codesecure.dependencycheck.data.LuceneUtils;
import org.codesecure.dependencycheck.scanner.Dependency;
import org.codesecure.dependencycheck.scanner.Evidence;
import org.codesecure.dependencycheck.scanner.Evidence.Confidence;
import org.codesecure.dependencycheck.scanner.EvidenceCollection;
/**
* CPEQuery is a utility class that takes a project dependency and attempts
@@ -60,7 +64,7 @@ public class CPEQuery {
* A string representation of a regular expression defining characters
* utilized within the CPE Names.
*/
static final String CLEANSE_CHARACTER_RX = "[^A-Za-z0-9 _-]";
static final String CLEANSE_CHARACTER_RX = "[^A-Za-z0-9 ._-]";
/* A string representation of a regular expression used to remove all but
* alpha characters.
*/
@@ -166,22 +170,25 @@ public class CPEQuery {
* @param dependency the dependency to search for CPE entries on.
* @throws CorruptIndexException is thrown when the Lucene index is corrupt.
* @throws IOException is thrown when an IOException occurs.
* @throws ParseException is thrown when the Lucene query cannot be parsed.
* @throws ParseException is thrown when the Lucene query cannot be parsed.
*/
public void determineCPE(Dependency dependency) throws CorruptIndexException, IOException, ParseException {
Confidence vendorConf = Confidence.HIGH;
Confidence titleConf = Confidence.HIGH;
Confidence versionConf = Confidence.HIGH;
String vendors = dependency.getVendorEvidence().toString(vendorConf);
String vendors = addEvidenceWithoutDuplicateTerms("", dependency.getVendorEvidence(), vendorConf);
//dependency.getVendorEvidence().toString(vendorConf);
// if ("".equals(vendors)) {
// vendors = STRING_THAT_WILL_NEVER_BE_IN_THE_INDEX;
// }
String titles = dependency.getTitleEvidence().toString(titleConf);
String titles = addEvidenceWithoutDuplicateTerms("", dependency.getTitleEvidence(), titleConf);
///dependency.getTitleEvidence().toString(titleConf);
// if ("".equals(titles)) {
// titles = STRING_THAT_WILL_NEVER_BE_IN_THE_INDEX;
// }
String versions = dependency.getVersionEvidence().toString(versionConf);
String versions = addEvidenceWithoutDuplicateTerms("", dependency.getVersionEvidence(), versionConf);
//dependency.getVersionEvidence().toString(versionConf);
// if ("".equals(versions)) {
// versions = STRING_THAT_WILL_NEVER_BE_IN_THE_INDEX;
// }
@@ -205,7 +212,8 @@ public class CPEQuery {
if (round == 0) {
vendorConf = reduceConfidence(vendorConf);
if (dependency.getVendorEvidence().contains(vendorConf)) {
vendors += " " + dependency.getVendorEvidence().toString(vendorConf);
//vendors += " " + dependency.getVendorEvidence().toString(vendorConf);
vendors = addEvidenceWithoutDuplicateTerms(vendors, dependency.getVendorEvidence(), vendorConf);
} else {
cnt += 1;
round += 1;
@@ -214,7 +222,8 @@ public class CPEQuery {
if (round == 1) {
titleConf = reduceConfidence(titleConf);
if (dependency.getTitleEvidence().contains(titleConf)) {
titles += " " + dependency.getTitleEvidence().toString(titleConf);
//titles += " " + dependency.getTitleEvidence().toString(titleConf);
titles = addEvidenceWithoutDuplicateTerms(titles, dependency.getTitleEvidence(), titleConf);
} else {
cnt += 1;
round += 1;
@@ -223,7 +232,8 @@ public class CPEQuery {
if (round == 2) {
versionConf = reduceConfidence(versionConf);
if (dependency.getVersionEvidence().contains(versionConf)) {
versions += " " + dependency.getVersionEvidence().toString(versionConf);
//versions += " " + dependency.getVersionEvidence().toString(versionConf);
versions = addEvidenceWithoutDuplicateTerms(versions, dependency.getVersionEvidence(), versionConf);
}
}
@@ -232,6 +242,33 @@ public class CPEQuery {
} while (!found && (++cnt) < 9);
}
/**
* Returns the text created by concatonating the text and the values from the
* EvidenceCollection (filtered for a specific confidence). This attempts to
* prevent duplicate terms from being added.<br/<br/>
* Note, if the evidence is longer then 200 characters it will be truncated.
*
* @param text the base text.
* @param ec an EvidenceCollection
* @param confidenceFilter a Confidence level to filter the evidence by.
* @return
*/
private String addEvidenceWithoutDuplicateTerms(final String text, final EvidenceCollection ec, Confidence confidenceFilter) {
String txt = (text == null) ? "" : text;
StringBuilder sb = new StringBuilder(txt.length() + (20 * ec.size()));
for (Evidence e : ec.iterator(confidenceFilter)) {
String value = e.getValue();
if (sb.indexOf(value)<0) {
if (value.length()>200) {
sb.append(value.substring(0,200));
} else {
sb.append(value).append(' ');
}
}
}
return sb.toString();
}
/**
* Reduces the given confidence by one level. This returns LOW if the confidence
* passed in is not HIGH.
@@ -282,7 +319,7 @@ public class CPEQuery {
* @throws ParseException when the generated query is not valid.
*/
protected List<Entry> searchCPE(String vendor, String product, String version,
List<String> vendorWeightings, List<String> productWeightings)
Set<String> vendorWeightings, Set<String> productWeightings)
throws CorruptIndexException, IOException, ParseException {
ArrayList<Entry> ret = new ArrayList<Entry>(MAX_QUERY_RESULTS);
@@ -319,7 +356,7 @@ public class CPEQuery {
* @return the Lucene query.
*/
protected String buildSearch(String vendor, String product, String version,
List<String> vendorWeighting, List<String> produdctWeightings) {
Set<String> vendorWeighting, Set<String> produdctWeightings) {
StringBuilder sb = new StringBuilder(vendor.length() + product.length()
+ version.length() + Fields.PRODUCT.length() + Fields.VERSION.length()
@@ -364,7 +401,7 @@ public class CPEQuery {
* importance when searching.
* @return if the append was successful.
*/
private boolean appendWeightedSearch(StringBuilder sb, String field, String searchText, List<String> weightedText) {
private boolean appendWeightedSearch(StringBuilder sb, String field, String searchText, Set<String> weightedText) {
//TODO add a mutator or special analyzer that combines words next to each other and adds them as a key.
sb.append(" ").append(field).append(":( ");
@@ -377,8 +414,9 @@ public class CPEQuery {
if (weightedText == null || weightedText.isEmpty()) {
LuceneUtils.appendEscapedLuceneQuery(sb, cleanText);
} else {
String[] text = cleanText.split("\\s");
for (String word : text) {
StringTokenizer tokens = new StringTokenizer(cleanText);
while (tokens.hasMoreElements()) {
String word = tokens.nextToken();
String temp = null;
for (String weighted : weightedText) {
String weightedStr = cleanseText(weighted);

18
src/main/java/org/codesecure/dependencycheck/data/cpe/Index.java
View File

@@ -25,12 +25,14 @@ import java.io.FileOutputStream;
import java.io.IOException;
import java.io.OutputStream;
import java.io.OutputStreamWriter;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.HashMap;
import java.util.Map;
import java.util.Properties;
import java.util.logging.Level;
import java.util.logging.Logger;
import javax.xml.parsers.ParserConfigurationException;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.KeywordAnalyzer;
import org.apache.lucene.analysis.PerFieldAnalyzerWrapper;
@@ -44,8 +46,10 @@ import org.apache.lucene.util.Version;
import org.codesecure.dependencycheck.utils.Downloader;
import org.codesecure.dependencycheck.utils.Settings;
import org.codesecure.dependencycheck.data.cpe.xml.Importer;
import org.codesecure.dependencycheck.utils.DownloadFailedException;
import org.joda.time.DateTime;
import org.joda.time.Days;
import org.xml.sax.SAXException;
/**
* The Index class is used to utilize and maintain the CPE Index.
@@ -130,9 +134,12 @@ public class Index {
* Downloads the latest CPE XML file from the web and imports it into
* the current CPE Index.
*
* @throws Exception is thrown if an exception occurs.
* @throws MalformedURLException is thrown if the URL for the CPE is malformed.
* @throws ParserConfigurationException is thrown if the parser is misconfigured.
* @throws SAXException is thrown if there is an error parsing the CPE XML.
* @throws IOException is thrown if a temporary file could not be created.
*/
public void updateIndexFromWeb() throws Exception {
public void updateIndexFromWeb() throws MalformedURLException, ParserConfigurationException, SAXException, IOException {
if (updateNeeded()) {
URL url = new URL(Settings.getString(Settings.KEYS.CPE_URL));
File outputPath = null;
@@ -141,7 +148,8 @@ public class Index {
Downloader.fetchFile(url, outputPath);
Importer.importXML(outputPath.toString());
writeLastUpdatedPropertyFile();
} catch (Exception ex) {
} catch (DownloadFailedException ex) {
Logger.getLogger(Index.class.getName()).log(Level.SEVERE, null, ex);
} finally {
boolean deleted = false;
@@ -209,9 +217,9 @@ public class Index {
prop.load(is);
lastUpdated = prop.getProperty(this.LAST_UPDATED);
} catch (FileNotFoundException ex) {
Logger.getLogger(Index.class.getName()).log(Level.SEVERE, null, ex);
Logger.getLogger(Index.class.getName()).log(Level.FINEST, null, ex);
} catch (IOException ex) {
Logger.getLogger(Index.class.getName()).log(Level.SEVERE, null, ex);
Logger.getLogger(Index.class.getName()).log(Level.FINEST, null, ex);
}
try {
long lastupdate = Long.parseLong(lastUpdated);

3
src/main/java/org/codesecure/dependencycheck/data/cpe/xml/Importer.java
View File

@@ -68,6 +68,9 @@ public class Importer {
*/
public static void importXML(String path) throws ParserConfigurationException, SAXException, IOException {
File f = new File(path);
if (!f.exists()) {
f.mkdirs();
}
Importer.importXML(f);
}
}

44
src/main/java/org/codesecure/dependencycheck/scanner/Evidence.java
View File

@@ -29,6 +29,7 @@ public class Evidence {
* The confidence that the evidence is "high" quality.
*/
public enum Confidence {
/**
* High confidence evidence.
*/
@@ -174,4 +175,47 @@ public class Evidence {
public void setConfidence(Confidence confidence) {
this.confidence = confidence;
}
/**
* Implements the hashCode for Evidence.
* @return hash code.
*/
@Override
public int hashCode() {
int hash = 3;
hash = 67 * hash + (this.name != null ? this.name.hashCode() : 0);
hash = 67 * hash + (this.source != null ? this.source.hashCode() : 0);
hash = 67 * hash + (this.value != null ? this.value.hashCode() : 0);
hash = 67 * hash + (this.confidence != null ? this.confidence.hashCode() : 0);
return hash;
}
/**
* Implements equals for Evidence.
* @param that an object to check the equality of.
* @return whether the two objects are equal.
*/
@Override
public boolean equals(Object that) {
if (this == that) {
return true;
}
if (!(that instanceof Evidence)) {
return false;
}
Evidence e = (Evidence) that;
return testEquality(name, e.name) && testEquality(source, e.source) && testEquality(value, e.value)
&& (confidence == null ? e.confidence == null : confidence == e.confidence);
}
/**
* Simple equality test for use within the equals method. This does a case insensitive compare.
* @param l a string to compare.
* @param r another string to compare.
* @return whether the two strings are the same.
*/
private boolean testEquality(String l, String r) {
return l == null ? r == null : l.equalsIgnoreCase(r);
}
}

63
src/main/java/org/codesecure/dependencycheck/scanner/EvidenceCollection.java
View File

@@ -18,9 +18,9 @@ package org.codesecure.dependencycheck.scanner;
* Copyright (c) 2012 Jeremy Long. All Rights Reserved.
*/
import java.util.ArrayList;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Set;
import org.codesecure.dependencycheck.utils.Filter;
/**
@@ -85,15 +85,15 @@ public class EvidenceCollection implements Iterable<Evidence> {
return EvidenceCollection.LOW_CONFIDENCE.filter(this.list);
}
}
private List<Evidence> list = null;
private List<String> weightedStrings = null;
private Set<Evidence> list = null;
private Set<String> weightedStrings = null;
/**
* Creates a new EvidenceCollection.
*/
public EvidenceCollection() {
list = new ArrayList<Evidence>();
weightedStrings = new ArrayList<String>();
list = new HashSet<Evidence>();
weightedStrings = new HashSet<String>();
}
/**
@@ -133,18 +133,16 @@ public class EvidenceCollection implements Iterable<Evidence> {
* @param str to add to the weighting collection.
*/
public void addWeighting(String str) {
if (!weightedStrings.contains(str)) {
weightedStrings.add(str);
}
weightedStrings.add(str);
}
/**
* Returns a list of Weightings - a list of terms that are believed to be of
* Returns a set of Weightings - a list of terms that are believed to be of
* higher confidence when also found in another location.
*
* @return List<String>
* @return Set<String>
*/
public List<String> getWeighting() {
public Set<String> getWeighting() {
return weightedStrings;
}
@@ -208,19 +206,27 @@ public class EvidenceCollection implements Iterable<Evidence> {
}
return ret;
}
/**
* Returns a string of evidence 'values' for a given confidence.
* @param confidence the confidence filter applied to the toString method.
* @return a string containing the evidence.
*/
public String toString(Evidence.Confidence confidence) {
StringBuilder sb = new StringBuilder();
for (Evidence e : this.iterator(confidence)) {
sb.append(e.getValue()).append(' ');
}
return sb.toString();
}
// Removed because this wasn't working right (the function returned the right data, but
// the use of the results was flawed.
// /**
// * Returns a string of evidence 'values' for a given confidence.
// * @param confidence the confidence filter applied to the toString method.
// * @return a string containing the evidence.
// */
// public String toString(Evidence.Confidence confidence) {
// StringBuilder sb = new StringBuilder();
// for (Evidence e : this.iterator(confidence)) {
// String str = e.getValue();
// //TODO this is a cheap hack, need to prevent the same string from hitting multiple times...
// // consider changing the evidencecollection.add to prevent the same "value" for a lower
// // confidence from being added? Might not work due to minor differences in the items in the manifest.
// // might need to actually use a StringTokenizer here and only add single words no in the list.
// if (sb.indexOf(str)<0) {
// sb.append(str).append(' ');
// }
// }
// return sb.toString();
// }
/**
* Returns a string of evidence 'values'.
@@ -235,4 +241,11 @@ public class EvidenceCollection implements Iterable<Evidence> {
return sb.toString();
}
/**
* Returns the number of elements in the EvidenceCollection.
* @return the number of elements in the collection.
*/
public int size() {
return list.size();
}
}

48
src/main/java/org/codesecure/dependencycheck/scanner/JarAnalyzer.java
View File

@@ -22,7 +22,9 @@ import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.security.NoSuchAlgorithmException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map.Entry;
import java.util.jar.Attributes;
import java.util.jar.JarFile;
@@ -47,6 +49,20 @@ import org.codesecure.dependencycheck.utils.Checksum;
*/
public class JarAnalyzer implements Analyzer {
private static List<String> IGNORE_LIST;
public JarAnalyzer() {
IGNORE_LIST = new ArrayList<String>();
IGNORE_LIST.add("built-by");
IGNORE_LIST.add("created-by");
IGNORE_LIST.add("license");
IGNORE_LIST.add("build-jdk");
IGNORE_LIST.add("ant-version");
IGNORE_LIST.add("import-package");
IGNORE_LIST.add("export-package");
IGNORE_LIST.add("sealed");
IGNORE_LIST.add("manifest-version");
}
/**
* item in some manifest, should be considered medium confidence.
*/
@@ -69,6 +85,7 @@ public class JarAnalyzer implements Analyzer {
* read in one character at a time.
*/
private enum STRING_STATE {
ALPHA,
NUMBER,
PERIOD,
@@ -364,20 +381,23 @@ public class JarAnalyzer implements Analyzer {
vendorEvidence.addEvidence(source, key, value, Evidence.Confidence.MEDIUM);
} else {
key = key.toLowerCase();
if (key.contains("version")) {
versionEvidence.addEvidence(source, key, value, Evidence.Confidence.MEDIUM);
} else if (key.contains("title")) {
titleEvidence.addEvidence(source, key, value, Evidence.Confidence.MEDIUM);
} else if (key.contains("vendor")) {
vendorEvidence.addEvidence(source, key, value, Evidence.Confidence.MEDIUM);
} else if (key.contains("name")) {
titleEvidence.addEvidence(source, key, value, Evidence.Confidence.MEDIUM);
vendorEvidence.addEvidence(source, key, value, Evidence.Confidence.MEDIUM);
} else {
titleEvidence.addEvidence(source, key, value, Evidence.Confidence.LOW);
vendorEvidence.addEvidence(source, key, value, Evidence.Confidence.LOW);
if (value.matches(".*\\d.*")) {
versionEvidence.addEvidence(source, key, value, Evidence.Confidence.LOW);
if (!IGNORE_LIST.contains(key)) {
if (key.contains("version")) {
versionEvidence.addEvidence(source, key, value, Evidence.Confidence.MEDIUM);
} else if (key.contains("title")) {
titleEvidence.addEvidence(source, key, value, Evidence.Confidence.MEDIUM);
} else if (key.contains("vendor")) {
vendorEvidence.addEvidence(source, key, value, Evidence.Confidence.MEDIUM);
} else if (key.contains("name")) {
titleEvidence.addEvidence(source, key, value, Evidence.Confidence.MEDIUM);
vendorEvidence.addEvidence(source, key, value, Evidence.Confidence.MEDIUM);
} else {
titleEvidence.addEvidence(source, key, value, Evidence.Confidence.LOW);
vendorEvidence.addEvidence(source, key, value, Evidence.Confidence.LOW);
if (value.matches(".*\\d.*")) {
versionEvidence.addEvidence(source, key, value, Evidence.Confidence.LOW);
}
}
}
}

18
src/main/java/org/codesecure/dependencycheck/utils/Downloader.java
View File

@@ -45,9 +45,9 @@ public class Downloader {
* Retrieves a file from a given URL and saves it to the outputPath.
* @param url the URL of the file to download.
* @param outputPath the path to the save the file to.
* @throws IOException is thrown if an IOException occurs.
* @throws DownloadFailedException is thrown if there is an error downloading the file.
*/
public static void fetchFile(URL url, String outputPath) throws IOException {
public static void fetchFile(URL url, String outputPath) throws DownloadFailedException {
File f = new File(outputPath);
fetchFile(url, f);
}
@@ -56,10 +56,14 @@ public class Downloader {
* Retrieves a file from a given URL and saves it to the outputPath.
* @param url the URL of the file to download.
* @param outputPath the path to the save the file to.
* @throws IOException is thrown if an IOException occurs.
* @throws DownloadFailedException is thrown if there is an error downloading the file.
*/
public static void fetchFile(URL url, File outputPath) throws IOException {
url.openConnection();
public static void fetchFile(URL url, File outputPath) throws DownloadFailedException {
try {
url.openConnection();
} catch (IOException ex) {
throw new DownloadFailedException("Error downloading file.", ex);
}
BufferedOutputStream writer = null;
try {
InputStream reader = url.openStream();
@@ -70,13 +74,13 @@ public class Downloader {
writer.write(buffer, 0, bytesRead);
}
} catch (Exception ex) {
Logger.getLogger(Downloader.class.getName()).log(Level.SEVERE, null, ex);
throw new DownloadFailedException("Error saving downloaded file.", ex);
} finally {
try {
writer.close();
writer = null;
} catch (Exception ex) {
Logger.getLogger(Downloader.class.getName()).log(Level.WARNING,
Logger.getLogger(Downloader.class.getName()).log(Level.FINEST,
"Error closing the writter in Downloader.", ex);
}
}