mirror of
https://github.com/ysoftdevs/DependencyCheck.git
synced 2026-01-14 07:43:40 +01:00
modified CPE Index to seperate writer/reader in prep for adding locking
Former-commit-id: baa48c8e70b36b9b14591d581bc1180b6d30d686
This commit is contained in:
Jeremy Long
@@ -0,0 +1,790 @@
|
||||
/*
|
||||
* This file is part of dependency-check-core.
|
||||
*
|
||||
* Dependency-check-core is free software: you can redistribute it and/or modify it
|
||||
* under the terms of the GNU General Public License as published by the Free
|
||||
* Software Foundation, either version 3 of the License, or (at your option) any
|
||||
* later version.
|
||||
*
|
||||
* Dependency-check-core is distributed in the hope that it will be useful, but
|
||||
* WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
|
||||
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
|
||||
* details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License along with
|
||||
* dependency-check-core. If not, see http://www.gnu.org/licenses/.
|
||||
*
|
||||
* Copyright (c) 2012 Jeremy Long. All Rights Reserved.
|
||||
*/
|
||||
package org.owasp.dependencycheck.analyzer;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.UnsupportedEncodingException;
|
||||
import java.net.URLEncoder;
|
||||
import java.sql.SQLException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
import java.util.StringTokenizer;
|
||||
import java.util.logging.Level;
|
||||
import java.util.logging.Logger;
|
||||
import org.apache.lucene.document.Document;
|
||||
import org.apache.lucene.index.CorruptIndexException;
|
||||
import org.apache.lucene.queryparser.classic.ParseException;
|
||||
import org.apache.lucene.search.ScoreDoc;
|
||||
import org.apache.lucene.search.TopDocs;
|
||||
import org.owasp.dependencycheck.Engine;
|
||||
import org.owasp.dependencycheck.data.lucene.LuceneUtils;
|
||||
import org.owasp.dependencycheck.dependency.Dependency;
|
||||
import org.owasp.dependencycheck.dependency.Evidence;
|
||||
import org.owasp.dependencycheck.dependency.Evidence.Confidence;
|
||||
import org.owasp.dependencycheck.dependency.EvidenceCollection;
|
||||
import org.owasp.dependencycheck.data.cpe.CpeIndexReader;
|
||||
import org.owasp.dependencycheck.data.cpe.Fields;
|
||||
import org.owasp.dependencycheck.data.cpe.IndexEntry;
|
||||
import org.owasp.dependencycheck.data.nvdcve.CveDB;
|
||||
import org.owasp.dependencycheck.data.nvdcve.DatabaseException;
|
||||
import org.owasp.dependencycheck.dependency.Identifier;
|
||||
import org.owasp.dependencycheck.dependency.VulnerableSoftware;
|
||||
import org.owasp.dependencycheck.utils.DependencyVersion;
|
||||
import org.owasp.dependencycheck.utils.DependencyVersionUtil;
|
||||
|
||||
/**
|
||||
* CPEAnalyzer is a utility class that takes a project dependency and attempts
|
||||
* to discern if there is an associated CPE. It uses the evidence contained
|
||||
* within the dependency to search the Lucene index.
|
||||
*
|
||||
* @author Jeremy Long (jeremy.long@owasp.org)
|
||||
*/
|
||||
public class CPEAnalyzer implements Analyzer {
|
||||
|
||||
/**
|
||||
* The maximum number of query results to return.
|
||||
*/
|
||||
static final int MAX_QUERY_RESULTS = 25;
|
||||
/**
|
||||
* The weighting boost to give terms when constructing the Lucene query.
|
||||
*/
|
||||
static final String WEIGHTING_BOOST = "^5";
|
||||
/**
|
||||
* A string representation of a regular expression defining characters
|
||||
* utilized within the CPE Names.
|
||||
*/
|
||||
static final String CLEANSE_CHARACTER_RX = "[^A-Za-z0-9 ._-]";
|
||||
/**
|
||||
* A string representation of a regular expression used to remove all but
|
||||
* alpha characters.
|
||||
*/
|
||||
static final String CLEANSE_NONALPHA_RX = "[^A-Za-z]*";
|
||||
/**
|
||||
* The additional size to add to a new StringBuilder to account for extra
|
||||
* data that will be written into the string.
|
||||
*/
|
||||
static final int STRING_BUILDER_BUFFER = 20;
|
||||
/**
|
||||
* The CPE Index Reader.
|
||||
*/
|
||||
private CpeIndexReader cpe;
|
||||
/**
|
||||
* The CVE Database.
|
||||
*/
|
||||
private CveDB cve;
|
||||
|
||||
/**
|
||||
* Opens the data source.
|
||||
*
|
||||
* @throws IOException when the Lucene directory to be queried does not
|
||||
* exist or is corrupt.
|
||||
* @throws DatabaseException when the database throws an exception. This
|
||||
* usually occurs when the database is in use by another process.
|
||||
*/
|
||||
public void open() throws IOException, DatabaseException {
|
||||
cpe = new CpeIndexReader();
|
||||
cpe.open();
|
||||
cve = new CveDB();
|
||||
try {
|
||||
cve.open();
|
||||
} catch (SQLException ex) {
|
||||
Logger.getLogger(CPEAnalyzer.class.getName()).log(Level.FINE, null, ex);
|
||||
throw new DatabaseException("Unable to open the cve db", ex);
|
||||
} catch (ClassNotFoundException ex) {
|
||||
Logger.getLogger(CPEAnalyzer.class.getName()).log(Level.FINE, null, ex);
|
||||
throw new DatabaseException("Unable to open the cve db", ex);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Closes the data source.
|
||||
*/
|
||||
@Override
|
||||
public void close() {
|
||||
if (cpe != null) {
|
||||
cpe.close();
|
||||
}
|
||||
if (cve != null) {
|
||||
cve.close();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the status of the data source - is the index open.
|
||||
*
|
||||
* @return true or false.
|
||||
*/
|
||||
public boolean isOpen() {
|
||||
return (cpe != null) && cpe.isOpen();
|
||||
}
|
||||
|
||||
/**
|
||||
* Ensures that the Lucene index is closed.
|
||||
*
|
||||
* @throws Throwable when a throwable is thrown.
|
||||
*/
|
||||
@Override
|
||||
protected void finalize() throws Throwable {
|
||||
super.finalize();
|
||||
if (isOpen()) {
|
||||
close();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Searches the data store of CPE entries, trying to identify the CPE for
|
||||
* the given dependency based on the evidence contained within. The
|
||||
* dependency passed in is updated with any identified CPE values.
|
||||
*
|
||||
* @param dependency the dependency to search for CPE entries on.
|
||||
* @throws CorruptIndexException is thrown when the Lucene index is corrupt.
|
||||
* @throws IOException is thrown when an IOException occurs.
|
||||
* @throws ParseException is thrown when the Lucene query cannot be parsed.
|
||||
*/
|
||||
protected void determineCPE(Dependency dependency) throws CorruptIndexException, IOException, ParseException {
|
||||
Confidence vendorConf = Confidence.HIGHEST;
|
||||
Confidence productConf = Confidence.HIGHEST;
|
||||
|
||||
String vendors = addEvidenceWithoutDuplicateTerms("", dependency.getVendorEvidence(), vendorConf);
|
||||
String products = addEvidenceWithoutDuplicateTerms("", dependency.getProductEvidence(), productConf);
|
||||
|
||||
int ctr = 0;
|
||||
do {
|
||||
if (!vendors.isEmpty() && !products.isEmpty()) {
|
||||
final List<IndexEntry> entries = searchCPE(vendors, products, dependency.getProductEvidence().getWeighting(),
|
||||
dependency.getVendorEvidence().getWeighting());
|
||||
|
||||
for (IndexEntry e : entries) {
|
||||
if (verifyEntry(e, dependency)) {
|
||||
final String vendor = e.getVendor();
|
||||
final String product = e.getProduct();
|
||||
determineIdentifiers(dependency, vendor, product);
|
||||
}
|
||||
}
|
||||
}
|
||||
vendorConf = reduceConfidence(vendorConf);
|
||||
if (dependency.getVendorEvidence().contains(vendorConf)) {
|
||||
vendors = addEvidenceWithoutDuplicateTerms(vendors, dependency.getVendorEvidence(), vendorConf);
|
||||
}
|
||||
productConf = reduceConfidence(productConf);
|
||||
if (dependency.getProductEvidence().contains(productConf)) {
|
||||
products = addEvidenceWithoutDuplicateTerms(products, dependency.getProductEvidence(), productConf);
|
||||
}
|
||||
} while ((++ctr) < 4);
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the text created by concatenating the text and the values from
|
||||
* the EvidenceCollection (filtered for a specific confidence). This
|
||||
* attempts to prevent duplicate terms from being added.<br/<br/> Note, if
|
||||
* the evidence is longer then 200 characters it will be truncated.
|
||||
*
|
||||
* @param text the base text.
|
||||
* @param ec an EvidenceCollection
|
||||
* @param confidenceFilter a Confidence level to filter the evidence by.
|
||||
* @return the new evidence text
|
||||
*/
|
||||
private String addEvidenceWithoutDuplicateTerms(final String text, final EvidenceCollection ec, Confidence confidenceFilter) {
|
||||
final String txt = (text == null) ? "" : text;
|
||||
final StringBuilder sb = new StringBuilder(txt.length() + (20 * ec.size()));
|
||||
sb.append(' ').append(txt).append(' ');
|
||||
for (Evidence e : ec.iterator(confidenceFilter)) {
|
||||
String value = e.getValue();
|
||||
|
||||
//hack to get around the fact that lucene does a really good job of recognizing domains and not
|
||||
// splitting them. TODO - put together a better lucene analyzer specific to the domain.
|
||||
if (value.startsWith("http://")) {
|
||||
value = value.substring(7).replaceAll("\\.", " ");
|
||||
}
|
||||
if (value.startsWith("https://")) {
|
||||
value = value.substring(8).replaceAll("\\.", " ");
|
||||
}
|
||||
if (sb.indexOf(" " + value + " ") < 0) {
|
||||
sb.append(value).append(' ');
|
||||
}
|
||||
}
|
||||
return sb.toString().trim();
|
||||
}
|
||||
|
||||
/**
|
||||
* Reduces the given confidence by one level. This returns LOW if the
|
||||
* confidence passed in is not HIGH.
|
||||
*
|
||||
* @param c the confidence to reduce.
|
||||
* @return One less then the confidence passed in.
|
||||
*/
|
||||
private Confidence reduceConfidence(final Confidence c) {
|
||||
if (c == Confidence.HIGHEST) {
|
||||
return Confidence.HIGH;
|
||||
} else if (c == Confidence.HIGH) {
|
||||
return Confidence.MEDIUM;
|
||||
} else {
|
||||
return Confidence.LOW;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* <p>Searches the Lucene CPE index to identify possible CPE entries
|
||||
* associated with the supplied vendor, product, and version.</p>
|
||||
*
|
||||
* <p>If either the vendorWeightings or productWeightings lists have been
|
||||
* populated this data is used to add weighting factors to the search.</p>
|
||||
*
|
||||
* @param vendor the text used to search the vendor field
|
||||
* @param product the text used to search the product field
|
||||
* @param vendorWeightings a list of strings to use to add weighting factors
|
||||
* to the vendor field
|
||||
* @param productWeightings Adds a list of strings that will be used to add
|
||||
* weighting factors to the product search
|
||||
* @return a list of possible CPE values
|
||||
* @throws CorruptIndexException when the Lucene index is corrupt
|
||||
* @throws IOException when the Lucene index is not found
|
||||
* @throws ParseException when the generated query is not valid
|
||||
*/
|
||||
protected List<IndexEntry> searchCPE(String vendor, String product,
|
||||
Set<String> vendorWeightings, Set<String> productWeightings)
|
||||
throws CorruptIndexException, IOException, ParseException {
|
||||
final ArrayList<IndexEntry> ret = new ArrayList<IndexEntry>(MAX_QUERY_RESULTS);
|
||||
|
||||
final String searchString = buildSearch(vendor, product, vendorWeightings, productWeightings);
|
||||
if (searchString == null) {
|
||||
return ret;
|
||||
}
|
||||
|
||||
final TopDocs docs = cpe.search(searchString, MAX_QUERY_RESULTS);
|
||||
for (ScoreDoc d : docs.scoreDocs) {
|
||||
final Document doc = cpe.getDocument(d.doc);
|
||||
final IndexEntry entry = new IndexEntry();
|
||||
entry.setVendor(doc.get(Fields.VENDOR));
|
||||
entry.setProduct(doc.get(Fields.PRODUCT));
|
||||
entry.setSearchScore(d.score);
|
||||
if (!ret.contains(entry)) {
|
||||
ret.add(entry);
|
||||
}
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
|
||||
/**
|
||||
* <p>Builds a Lucene search string by properly escaping data and
|
||||
* constructing a valid search query.</p>
|
||||
*
|
||||
* <p>If either the possibleVendor or possibleProducts lists have been
|
||||
* populated this data is used to add weighting factors to the search string
|
||||
* generated.</p>
|
||||
*
|
||||
* @param vendor text to search the vendor field
|
||||
* @param product text to search the product field
|
||||
* @param vendorWeighting a list of strings to apply to the vendor to boost
|
||||
* the terms weight
|
||||
* @param productWeightings a list of strings to apply to the product to
|
||||
* boost the terms weight
|
||||
* @return the Lucene query
|
||||
*/
|
||||
protected String buildSearch(String vendor, String product,
|
||||
Set<String> vendorWeighting, Set<String> productWeightings) {
|
||||
final String v = vendor; //.replaceAll("[^\\w\\d]", " ");
|
||||
final String p = product; //.replaceAll("[^\\w\\d]", " ");
|
||||
final StringBuilder sb = new StringBuilder(v.length() + p.length()
|
||||
+ Fields.PRODUCT.length() + Fields.VENDOR.length() + STRING_BUILDER_BUFFER);
|
||||
|
||||
if (!appendWeightedSearch(sb, Fields.PRODUCT, p, productWeightings)) {
|
||||
return null;
|
||||
}
|
||||
sb.append(" AND ");
|
||||
if (!appendWeightedSearch(sb, Fields.VENDOR, v, vendorWeighting)) {
|
||||
return null;
|
||||
}
|
||||
return sb.toString();
|
||||
}
|
||||
|
||||
/**
|
||||
* This method constructs a Lucene query for a given field. The searchText
|
||||
* is split into separate words and if the word is within the list of
|
||||
* weighted words then an additional weighting is applied to the term as it
|
||||
* is appended into the query.
|
||||
*
|
||||
* @param sb a StringBuilder that the query text will be appended to.
|
||||
* @param field the field within the Lucene index that the query is
|
||||
* searching.
|
||||
* @param searchText text used to construct the query.
|
||||
* @param weightedText a list of terms that will be considered higher
|
||||
* importance when searching.
|
||||
* @return if the append was successful.
|
||||
*/
|
||||
private boolean appendWeightedSearch(StringBuilder sb, String field, String searchText, Set<String> weightedText) {
|
||||
sb.append(" ").append(field).append(":( ");
|
||||
|
||||
final String cleanText = cleanseText(searchText);
|
||||
|
||||
if ("".equals(cleanText)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
if (weightedText == null || weightedText.isEmpty()) {
|
||||
LuceneUtils.appendEscapedLuceneQuery(sb, cleanText);
|
||||
} else {
|
||||
final StringTokenizer tokens = new StringTokenizer(cleanText);
|
||||
while (tokens.hasMoreElements()) {
|
||||
final String word = tokens.nextToken();
|
||||
String temp = null;
|
||||
for (String weighted : weightedText) {
|
||||
final String weightedStr = cleanseText(weighted);
|
||||
if (equalsIgnoreCaseAndNonAlpha(word, weightedStr)) {
|
||||
temp = LuceneUtils.escapeLuceneQuery(word) + WEIGHTING_BOOST;
|
||||
if (!word.equalsIgnoreCase(weightedStr)) {
|
||||
temp += " " + LuceneUtils.escapeLuceneQuery(weightedStr) + WEIGHTING_BOOST;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (temp == null) {
|
||||
temp = LuceneUtils.escapeLuceneQuery(word);
|
||||
}
|
||||
sb.append(" ").append(temp);
|
||||
}
|
||||
}
|
||||
sb.append(" ) ");
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Removes characters from the input text that are not used within the CPE
|
||||
* index.
|
||||
*
|
||||
* @param text is the text to remove the characters from.
|
||||
* @return the text having removed some characters.
|
||||
*/
|
||||
private String cleanseText(String text) {
|
||||
return text.replaceAll(CLEANSE_CHARACTER_RX, " ");
|
||||
}
|
||||
|
||||
/**
|
||||
* Compares two strings after lower casing them and removing the non-alpha
|
||||
* characters.
|
||||
*
|
||||
* @param l string one to compare.
|
||||
* @param r string two to compare.
|
||||
* @return whether or not the two strings are similar.
|
||||
*/
|
||||
private boolean equalsIgnoreCaseAndNonAlpha(String l, String r) {
|
||||
if (l == null || r == null) {
|
||||
return false;
|
||||
}
|
||||
|
||||
final String left = l.replaceAll(CLEANSE_NONALPHA_RX, "");
|
||||
final String right = r.replaceAll(CLEANSE_NONALPHA_RX, "");
|
||||
return left.equalsIgnoreCase(right);
|
||||
}
|
||||
|
||||
/**
|
||||
* Ensures that the CPE Identified matches the dependency. This validates
|
||||
* that the product, vendor, and version information for the CPE are
|
||||
* contained within the dependencies evidence.
|
||||
*
|
||||
* @param entry a CPE entry.
|
||||
* @param dependency the dependency that the CPE entries could be for.
|
||||
* @return whether or not the entry is valid.
|
||||
*/
|
||||
private boolean verifyEntry(final IndexEntry entry, final Dependency dependency) {
|
||||
boolean isValid = false;
|
||||
|
||||
if (collectionContainsString(dependency.getProductEvidence(), entry.getProduct())
|
||||
&& collectionContainsString(dependency.getVendorEvidence(), entry.getVendor())) {
|
||||
//&& collectionContainsVersion(dependency.getVersionEvidence(), entry.getVersion())
|
||||
isValid = true;
|
||||
}
|
||||
return isValid;
|
||||
}
|
||||
|
||||
/**
|
||||
* Used to determine if the EvidenceCollection contains a specific string.
|
||||
*
|
||||
* @param ec an EvidenceCollection
|
||||
* @param text the text to search for
|
||||
* @return whether or not the EvidenceCollection contains the string
|
||||
*/
|
||||
private boolean collectionContainsString(EvidenceCollection ec, String text) {
|
||||
|
||||
//<editor-fold defaultstate="collapsed" desc="This code fold contains an old version of the code, delete once more testing is done">
|
||||
// String[] splitText = text.split("[\\s_-]");
|
||||
//
|
||||
// for (String search : splitText) {
|
||||
// //final String search = text.replaceAll("[\\s_-]", "").toLowerCase();
|
||||
// if (ec.containsUsedString(search)) {
|
||||
// return true;
|
||||
// }
|
||||
// }
|
||||
//</editor-fold>
|
||||
|
||||
//TODO - likely need to change the split... not sure if this will work for CPE with special chars
|
||||
final String[] words = text.split("[\\s_-]");
|
||||
final List<String> list = new ArrayList<String>();
|
||||
String tempWord = null;
|
||||
for (String word : words) {
|
||||
//single letter words should be concatonated with the next word.
|
||||
// so { "m", "core", "sample" } -> { "mcore", "sample" }
|
||||
if (tempWord != null) {
|
||||
list.add(tempWord + word);
|
||||
tempWord = null;
|
||||
} else if (word.length() <= 2) {
|
||||
tempWord = word;
|
||||
} else {
|
||||
list.add(word);
|
||||
}
|
||||
}
|
||||
// if (tempWord != null) {
|
||||
// //for now ignore any last single letter words...
|
||||
// }
|
||||
boolean contains = true;
|
||||
for (String word : list) {
|
||||
contains &= ec.containsUsedString(word);
|
||||
}
|
||||
return contains;
|
||||
}
|
||||
|
||||
/**
|
||||
* Analyzes a dependency and attempts to determine if there are any CPE
|
||||
* identifiers for this dependency.
|
||||
*
|
||||
* @param dependency The Dependency to analyze.
|
||||
* @param engine The analysis engine
|
||||
* @throws AnalysisException is thrown if there is an issue analyzing the
|
||||
* dependency.
|
||||
*/
|
||||
@Override
|
||||
public void analyze(Dependency dependency, Engine engine) throws AnalysisException {
|
||||
try {
|
||||
determineCPE(dependency);
|
||||
} catch (CorruptIndexException ex) {
|
||||
throw new AnalysisException("CPE Index is corrupt.", ex);
|
||||
} catch (IOException ex) {
|
||||
throw new AnalysisException("Failure opening the CPE Index.", ex);
|
||||
} catch (ParseException ex) {
|
||||
throw new AnalysisException("Unable to parse the generated Lucene query for this dependency.", ex);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns true because this analyzer supports all dependency types.
|
||||
*
|
||||
* @return true.
|
||||
*/
|
||||
@Override
|
||||
public Set<String> getSupportedExtensions() {
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the name of this analyzer.
|
||||
*
|
||||
* @return the name of this analyzer.
|
||||
*/
|
||||
@Override
|
||||
public String getName() {
|
||||
return "CPE Analyzer";
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns true because this analyzer supports all dependency types.
|
||||
*
|
||||
* @param extension the file extension of the dependency being analyzed.
|
||||
* @return true.
|
||||
*/
|
||||
@Override
|
||||
public boolean supportsExtension(String extension) {
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the analysis phase that this analyzer should run in.
|
||||
*
|
||||
* @return the analysis phase that this analyzer should run in.
|
||||
*/
|
||||
@Override
|
||||
public AnalysisPhase getAnalysisPhase() {
|
||||
return AnalysisPhase.IDENTIFIER_ANALYSIS;
|
||||
}
|
||||
|
||||
/**
|
||||
* Opens the CPE Lucene Index.
|
||||
*
|
||||
* @throws Exception is thrown if there is an issue opening the index.
|
||||
*/
|
||||
@Override
|
||||
public void initialize() throws Exception {
|
||||
this.open();
|
||||
}
|
||||
|
||||
/**
|
||||
* Retrieves a list of CPE values from the CveDB based on the vendor and
|
||||
* product passed in. The list is then validated to find only CPEs that are
|
||||
* valid for the given dependency. It is possible that the CPE identified is
|
||||
* a best effort "guess" based on the vendor, product, and version
|
||||
* information.
|
||||
*
|
||||
* @param dependency the Dependency being analyzed
|
||||
* @param vendor the vendor for the CPE being analyzed
|
||||
* @param product the product for the CPE being analyzed
|
||||
* @throws UnsupportedEncodingException is thrown if UTF-8 is not supported
|
||||
*/
|
||||
private void determineIdentifiers(Dependency dependency, String vendor, String product) throws UnsupportedEncodingException {
|
||||
final Set<VulnerableSoftware> cpes = cve.getCPEs(vendor, product);
|
||||
DependencyVersion bestGuess = new DependencyVersion("-");
|
||||
Confidence bestGuessConf = null;
|
||||
final List<IdentifierMatch> collected = new ArrayList<IdentifierMatch>();
|
||||
for (Confidence conf : Confidence.values()) {
|
||||
for (Evidence evidence : dependency.getVersionEvidence().iterator(conf)) {
|
||||
final DependencyVersion evVer = DependencyVersionUtil.parseVersion(evidence.getValue());
|
||||
if (evVer == null) {
|
||||
continue;
|
||||
}
|
||||
for (VulnerableSoftware vs : cpes) {
|
||||
DependencyVersion dbVer;
|
||||
if (vs.getRevision() != null && !vs.getRevision().isEmpty()) {
|
||||
dbVer = DependencyVersionUtil.parseVersion(vs.getVersion() + "." + vs.getRevision());
|
||||
} else {
|
||||
dbVer = DependencyVersionUtil.parseVersion(vs.getVersion());
|
||||
}
|
||||
if (dbVer == null //special case, no version specified - everything is vulnerable
|
||||
|| evVer.equals(dbVer)) { //woot exect match
|
||||
final String url = String.format("http://web.nvd.nist.gov/view/vuln/search?cpe=%s", URLEncoder.encode(vs.getName(), "UTF-8"));
|
||||
final IdentifierMatch match = new IdentifierMatch("cpe", vs.getName(), url, IdentifierConfidence.EXACT_MATCH, conf);
|
||||
collected.add(match);
|
||||
} else {
|
||||
//TODO the following isn't quite right is it? need to think about this guessing game a bit more.
|
||||
if (evVer.getVersionParts().size() <= dbVer.getVersionParts().size()
|
||||
&& evVer.matchesAtLeastThreeLevels(dbVer)) {
|
||||
if (bestGuessConf == null || bestGuessConf.compareTo(conf) > 0) {
|
||||
if (bestGuess.getVersionParts().size() < dbVer.getVersionParts().size()) {
|
||||
bestGuess = dbVer;
|
||||
bestGuessConf = conf;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if (bestGuessConf == null || bestGuessConf.compareTo(conf) > 0) {
|
||||
if (bestGuess.getVersionParts().size() < evVer.getVersionParts().size()) {
|
||||
bestGuess = evVer;
|
||||
bestGuessConf = conf;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
final String cpeName = String.format("cpe:/a:%s:%s:%s", vendor, product, bestGuess.toString());
|
||||
final String url = null; //String.format("http://web.nvd.nist.gov/view/vuln/search?cpe=%s", URLEncoder.encode(cpeName, "UTF-8"));
|
||||
if (bestGuessConf == null) {
|
||||
bestGuessConf = Confidence.LOW;
|
||||
}
|
||||
final IdentifierMatch match = new IdentifierMatch("cpe", cpeName, url, IdentifierConfidence.BEST_GUESS, bestGuessConf);
|
||||
collected.add(match);
|
||||
|
||||
Collections.sort(collected);
|
||||
final IdentifierConfidence bestIdentifierQuality = collected.get(0).getConfidence();
|
||||
final Confidence bestEvidenceQuality = collected.get(0).getEvidenceConfidence();
|
||||
for (IdentifierMatch m : collected) {
|
||||
if (bestIdentifierQuality.equals(m.getConfidence())
|
||||
&& bestEvidenceQuality.equals(m.getEvidenceConfidence())) {
|
||||
dependency.addIdentifier(m.getIdentifier());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* The confidence whether the identifier is an exact match, or a best guess.
|
||||
*/
|
||||
private enum IdentifierConfidence {
|
||||
|
||||
/**
|
||||
* An exact match for the CPE.
|
||||
*/
|
||||
EXACT_MATCH,
|
||||
/**
|
||||
* A best guess for the CPE.
|
||||
*/
|
||||
BEST_GUESS
|
||||
}
|
||||
|
||||
/**
|
||||
* A simple object to hold an identifier and carry information about the
|
||||
* confidence in the identifier.
|
||||
*/
|
||||
private static class IdentifierMatch implements Comparable<IdentifierMatch> {
|
||||
|
||||
/**
|
||||
* Constructs an IdentiferMatch.
|
||||
*
|
||||
* @param type the type of identifier (such as CPE)
|
||||
* @param value the value of the identifier
|
||||
* @param url the URL of the identifier
|
||||
* @param identifierConfidence the confidence in the identifier: best
|
||||
* guess or exact match
|
||||
* @param evidenceConfidence the confidence of the evidence used to find
|
||||
* the identifier
|
||||
*/
|
||||
IdentifierMatch(String type, String value, String url, IdentifierConfidence identifierConfidence, Confidence evidenceConfidence) {
|
||||
this.identifier = new Identifier(type, value, url);
|
||||
this.confidence = identifierConfidence;
|
||||
this.evidenceConfidence = evidenceConfidence;
|
||||
}
|
||||
//<editor-fold defaultstate="collapsed" desc="Property implementations: evidenceConfidence, confidence, identifier">
|
||||
/**
|
||||
* The confidence in the evidence used to identify this match.
|
||||
*/
|
||||
private Confidence evidenceConfidence;
|
||||
|
||||
/**
|
||||
* Get the value of evidenceConfidence
|
||||
*
|
||||
* @return the value of evidenceConfidence
|
||||
*/
|
||||
public Confidence getEvidenceConfidence() {
|
||||
return evidenceConfidence;
|
||||
}
|
||||
|
||||
/**
|
||||
* Set the value of evidenceConfidence
|
||||
*
|
||||
* @param evidenceConfidence new value of evidenceConfidence
|
||||
*/
|
||||
public void setEvidenceConfidence(Confidence evidenceConfidence) {
|
||||
this.evidenceConfidence = evidenceConfidence;
|
||||
}
|
||||
/**
|
||||
* The confidence whether this is an exact match, or a best guess.
|
||||
*/
|
||||
private IdentifierConfidence confidence;
|
||||
|
||||
/**
|
||||
* Get the value of confidence.
|
||||
*
|
||||
* @return the value of confidence
|
||||
*/
|
||||
public IdentifierConfidence getConfidence() {
|
||||
return confidence;
|
||||
}
|
||||
|
||||
/**
|
||||
* Set the value of confidence.
|
||||
*
|
||||
* @param confidence new value of confidence
|
||||
*/
|
||||
public void setConfidence(IdentifierConfidence confidence) {
|
||||
this.confidence = confidence;
|
||||
}
|
||||
/**
|
||||
* The CPE identifier.
|
||||
*/
|
||||
private Identifier identifier;
|
||||
|
||||
/**
|
||||
* Get the value of identifier.
|
||||
*
|
||||
* @return the value of identifier
|
||||
*/
|
||||
public Identifier getIdentifier() {
|
||||
return identifier;
|
||||
}
|
||||
|
||||
/**
|
||||
* Set the value of identifier.
|
||||
*
|
||||
* @param identifier new value of identifier
|
||||
*/
|
||||
public void setIdentifier(Identifier identifier) {
|
||||
this.identifier = identifier;
|
||||
}
|
||||
//</editor-fold>
|
||||
//<editor-fold defaultstate="collapsed" desc="Standard implementations of toString, hashCode, and equals">
|
||||
|
||||
/**
|
||||
* Standard toString() implementation.
|
||||
*
|
||||
* @return the string representation of the object
|
||||
*/
|
||||
@Override
|
||||
public String toString() {
|
||||
return "IdentifierMatch{" + "evidenceConfidence=" + evidenceConfidence
|
||||
+ ", confidence=" + confidence + ", identifier=" + identifier + '}';
|
||||
}
|
||||
|
||||
/**
|
||||
* Standard hashCode() implementation.
|
||||
*
|
||||
* @return the hashCode
|
||||
*/
|
||||
@Override
|
||||
public int hashCode() {
|
||||
int hash = 5;
|
||||
hash = 97 * hash + (this.evidenceConfidence != null ? this.evidenceConfidence.hashCode() : 0);
|
||||
hash = 97 * hash + (this.confidence != null ? this.confidence.hashCode() : 0);
|
||||
hash = 97 * hash + (this.identifier != null ? this.identifier.hashCode() : 0);
|
||||
return hash;
|
||||
}
|
||||
|
||||
/**
|
||||
* Standard equals implementation.
|
||||
*
|
||||
* @param obj the object to compare
|
||||
* @return true if the objects are equal, otherwise false
|
||||
*/
|
||||
@Override
|
||||
public boolean equals(Object obj) {
|
||||
if (obj == null) {
|
||||
return false;
|
||||
}
|
||||
if (getClass() != obj.getClass()) {
|
||||
return false;
|
||||
}
|
||||
final IdentifierMatch other = (IdentifierMatch) obj;
|
||||
if (this.evidenceConfidence != other.evidenceConfidence) {
|
||||
return false;
|
||||
}
|
||||
if (this.confidence != other.confidence) {
|
||||
return false;
|
||||
}
|
||||
if (this.identifier != other.identifier && (this.identifier == null || !this.identifier.equals(other.identifier))) {
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
//</editor-fold>
|
||||
|
||||
/**
|
||||
* Standard implementation of compareTo that compares identifier
|
||||
* confidence, evidence confidence, and then the identifier.
|
||||
*
|
||||
* @param o the IdentifierMatch to compare to
|
||||
* @return the natural ordering of IdentifierMatch
|
||||
*/
|
||||
@Override
|
||||
public int compareTo(IdentifierMatch o) {
|
||||
int conf = this.confidence.compareTo(o.confidence);
|
||||
if (conf == 0) {
|
||||
conf = this.evidenceConfidence.compareTo(o.evidenceConfidence);
|
||||
if (conf == 0) {
|
||||
conf = identifier.compareTo(o.identifier);
|
||||
}
|
||||
}
|
||||
return conf;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,108 @@
|
||||
/*
|
||||
* This file is part of dependency-check-core.
|
||||
*
|
||||
* Dependency-check-core is free software: you can redistribute it and/or modify it
|
||||
* under the terms of the GNU General Public License as published by the Free
|
||||
* Software Foundation, either version 3 of the License, or (at your option) any
|
||||
* later version.
|
||||
*
|
||||
* Dependency-check-core is distributed in the hope that it will be useful, but
|
||||
* WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
|
||||
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
|
||||
* details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License along with
|
||||
* dependency-check-core. If not, see http://www.gnu.org/licenses/.
|
||||
*
|
||||
* Copyright (c) 2013 Jeremy Long. All Rights Reserved.
|
||||
*/
|
||||
package org.owasp.dependencycheck.data.cpe;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.util.logging.Level;
|
||||
import java.util.logging.Logger;
|
||||
import org.apache.lucene.store.Directory;
|
||||
import org.apache.lucene.store.FSDirectory;
|
||||
import org.owasp.dependencycheck.utils.Settings;
|
||||
|
||||
/**
|
||||
* The Base Index class used to access the CPE Index.
|
||||
*
|
||||
* @author Jeremy Long (jeremy.long@owasp.org)
|
||||
*/
|
||||
public abstract class BaseIndex {
|
||||
|
||||
/**
|
||||
* The Lucene directory containing the index.
|
||||
*/
|
||||
protected Directory directory;
|
||||
/**
|
||||
* Indicates whether or not the Lucene Index is open.
|
||||
*/
|
||||
protected boolean indexOpen = false;
|
||||
|
||||
/**
|
||||
* Opens the CPE Index.
|
||||
*
|
||||
* @throws IOException is thrown if an IOException occurs opening the index.
|
||||
*/
|
||||
public void open() throws IOException {
|
||||
directory = this.openDirectory();
|
||||
indexOpen = true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Closes the CPE Index.
|
||||
*/
|
||||
public void close() {
|
||||
try {
|
||||
directory.close();
|
||||
} catch (IOException ex) {
|
||||
final String msg = "Unable to update database due to an IO error.";
|
||||
Logger.getLogger(BaseIndex.class.getName()).log(Level.SEVERE, msg);
|
||||
Logger.getLogger(BaseIndex.class.getName()).log(Level.FINE, null, ex);
|
||||
} finally {
|
||||
directory = null;
|
||||
}
|
||||
indexOpen = false;
|
||||
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the status of the data source - is the index open.
|
||||
*
|
||||
* @return true or false.
|
||||
*/
|
||||
public boolean isOpen() {
|
||||
return indexOpen;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the Lucene directory object for the CPE Index.
|
||||
*
|
||||
* @return the Lucene Directory object for the CPE Index.
|
||||
* @throws IOException is thrown if an IOException occurs.
|
||||
*/
|
||||
protected Directory openDirectory() throws IOException {
|
||||
final File path = getDataDirectory();
|
||||
return FSDirectory.open(path);
|
||||
}
|
||||
|
||||
/**
|
||||
* Retrieves the directory that the JAR file exists in so that we can ensure
|
||||
* we always use a common data directory.
|
||||
*
|
||||
* @return the data directory for this index.
|
||||
* @throws IOException is thrown if an IOException occurs of course...
|
||||
*/
|
||||
public static File getDataDirectory() throws IOException {
|
||||
final File path = Settings.getFile(Settings.KEYS.CPE_DATA_DIRECTORY);
|
||||
if (!path.exists()) {
|
||||
if (!path.mkdirs()) {
|
||||
throw new IOException("Unable to create CPE Data directory");
|
||||
}
|
||||
}
|
||||
return path;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,179 @@
|
||||
/*
|
||||
* This file is part of dependency-check-core.
|
||||
*
|
||||
* Dependency-check-core is free software: you can redistribute it and/or modify it
|
||||
* under the terms of the GNU General Public License as published by the Free
|
||||
* Software Foundation, either version 3 of the License, or (at your option) any
|
||||
* later version.
|
||||
*
|
||||
* Dependency-check-core is distributed in the hope that it will be useful, but
|
||||
* WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
|
||||
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
|
||||
* details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License along with
|
||||
* dependency-check-core. If not, see http://www.gnu.org/licenses/.
|
||||
*
|
||||
* Copyright (c) 2013 Jeremy Long. All Rights Reserved.
|
||||
*/
|
||||
package org.owasp.dependencycheck.data.cpe;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
import java.util.logging.Level;
|
||||
import java.util.logging.Logger;
|
||||
import org.apache.lucene.analysis.Analyzer;
|
||||
import org.apache.lucene.analysis.core.KeywordAnalyzer;
|
||||
import org.apache.lucene.analysis.miscellaneous.PerFieldAnalyzerWrapper;
|
||||
import org.apache.lucene.document.Document;
|
||||
import org.apache.lucene.index.CorruptIndexException;
|
||||
import org.apache.lucene.index.DirectoryReader;
|
||||
import org.apache.lucene.index.IndexReader;
|
||||
import org.apache.lucene.queryparser.classic.ParseException;
|
||||
import org.apache.lucene.queryparser.classic.QueryParser;
|
||||
import org.apache.lucene.search.IndexSearcher;
|
||||
import org.apache.lucene.search.Query;
|
||||
import org.apache.lucene.search.TopDocs;
|
||||
import org.apache.lucene.util.Version;
|
||||
import org.owasp.dependencycheck.data.lucene.FieldAnalyzer;
|
||||
import org.owasp.dependencycheck.data.lucene.SearchFieldAnalyzer;
|
||||
|
||||
/**
|
||||
*
|
||||
* @author Jeremy Long (jeremy.long@owasp.org)
|
||||
*/
|
||||
public class CpeIndexReader extends BaseIndex {
|
||||
|
||||
/**
|
||||
* The Lucene IndexReader.
|
||||
*/
|
||||
private IndexReader indexReader;
|
||||
/**
|
||||
* The Lucene IndexSearcher.
|
||||
*/
|
||||
private IndexSearcher indexSearcher;
|
||||
/**
|
||||
* The Lucene Analyzer used for Searching.
|
||||
*/
|
||||
private Analyzer searchingAnalyzer;
|
||||
/**
|
||||
* The Lucene QueryParser used for Searching.
|
||||
*/
|
||||
private QueryParser queryParser;
|
||||
/**
|
||||
* The search field analyzer for the product field.
|
||||
*/
|
||||
private SearchFieldAnalyzer productSearchFieldAnalyzer;
|
||||
/**
|
||||
* The search field analyzer for the vendor field.
|
||||
*/
|
||||
private SearchFieldAnalyzer vendorSearchFieldAnalyzer;
|
||||
|
||||
/**
|
||||
* Opens the CPE Index.
|
||||
*
|
||||
* @throws IOException is thrown if an IOException occurs opening the index.
|
||||
*/
|
||||
@Override
|
||||
public void open() throws IOException {
|
||||
//TODO add spinlock (shared)
|
||||
super.open();
|
||||
indexReader = DirectoryReader.open(directory);
|
||||
indexSearcher = new IndexSearcher(indexReader);
|
||||
searchingAnalyzer = createSearchingAnalyzer();
|
||||
queryParser = new QueryParser(Version.LUCENE_43, Fields.DOCUMENT_KEY, searchingAnalyzer);
|
||||
}
|
||||
|
||||
/**
|
||||
* Closes the CPE Index.
|
||||
*/
|
||||
@Override
|
||||
public void close() {
|
||||
//TODO remove spinlock (shared)
|
||||
if (searchingAnalyzer != null) {
|
||||
searchingAnalyzer.close();
|
||||
searchingAnalyzer = null;
|
||||
}
|
||||
if (indexReader != null) {
|
||||
try {
|
||||
indexReader.close();
|
||||
} catch (IOException ex) {
|
||||
Logger.getLogger(CpeIndexReader.class.getName()).log(Level.FINEST, null, ex);
|
||||
}
|
||||
indexReader = null;
|
||||
}
|
||||
queryParser = null;
|
||||
indexSearcher = null;
|
||||
super.close();
|
||||
}
|
||||
|
||||
/**
|
||||
* Searches the index using the given search string.
|
||||
*
|
||||
* @param searchString the query text
|
||||
* @param maxQueryResults the maximum number of documents to return
|
||||
* @return the TopDocs found by the search
|
||||
* @throws ParseException thrown when the searchString is invalid
|
||||
* @throws IOException is thrown if there is an issue with the underlying
|
||||
* Index
|
||||
*/
|
||||
public TopDocs search(String searchString, int maxQueryResults) throws ParseException, IOException {
|
||||
final Query query = queryParser.parse(searchString);
|
||||
return indexSearcher.search(query, maxQueryResults);
|
||||
}
|
||||
|
||||
/**
|
||||
* Searches the index using the given query.
|
||||
*
|
||||
* @param query the query used to search the index
|
||||
* @param maxQueryResults the max number of results to return
|
||||
* @return the TopDocs found be the query
|
||||
* @throws CorruptIndexException thrown if the Index is corrupt
|
||||
* @throws IOException thrown if there is an IOException
|
||||
*/
|
||||
public TopDocs search(Query query, int maxQueryResults) throws CorruptIndexException, IOException {
|
||||
resetSearchingAnalyzer();
|
||||
return indexSearcher.search(query, maxQueryResults);
|
||||
}
|
||||
|
||||
/**
|
||||
* Retrieves a document from the Index.
|
||||
*
|
||||
* @param documentId the id of the document to retrieve
|
||||
* @return the Document
|
||||
* @throws IOException thrown if there is an IOException
|
||||
*/
|
||||
public Document getDocument(int documentId) throws IOException {
|
||||
return indexSearcher.doc(documentId);
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates an Analyzer for searching the CPE Index.
|
||||
*
|
||||
* @return the CPE Analyzer.
|
||||
*/
|
||||
@SuppressWarnings("unchecked")
|
||||
private Analyzer createSearchingAnalyzer() {
|
||||
final Map fieldAnalyzers = new HashMap();
|
||||
fieldAnalyzers.put(Fields.DOCUMENT_KEY, new KeywordAnalyzer());
|
||||
productSearchFieldAnalyzer = new SearchFieldAnalyzer(Version.LUCENE_43);
|
||||
vendorSearchFieldAnalyzer = new SearchFieldAnalyzer(Version.LUCENE_43);
|
||||
fieldAnalyzers.put(Fields.PRODUCT, productSearchFieldAnalyzer);
|
||||
fieldAnalyzers.put(Fields.VENDOR, vendorSearchFieldAnalyzer);
|
||||
|
||||
return new PerFieldAnalyzerWrapper(new FieldAnalyzer(Version.LUCENE_43), fieldAnalyzers);
|
||||
}
|
||||
|
||||
/**
|
||||
* Resets the searching analyzers
|
||||
*/
|
||||
private void resetSearchingAnalyzer() {
|
||||
if (productSearchFieldAnalyzer != null) {
|
||||
productSearchFieldAnalyzer.clear();
|
||||
}
|
||||
if (vendorSearchFieldAnalyzer != null) {
|
||||
vendorSearchFieldAnalyzer.clear();
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,149 @@
|
||||
/*
|
||||
* This file is part of dependency-check-core.
|
||||
*
|
||||
* Dependency-check-core is free software: you can redistribute it and/or modify it
|
||||
* under the terms of the GNU General Public License as published by the Free
|
||||
* Software Foundation, either version 3 of the License, or (at your option) any
|
||||
* later version.
|
||||
*
|
||||
* Dependency-check-core is distributed in the hope that it will be useful, but
|
||||
* WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
|
||||
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
|
||||
* details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License along with
|
||||
* dependency-check-core. If not, see http://www.gnu.org/licenses/.
|
||||
*
|
||||
* Copyright (c) 2013 Jeremy Long. All Rights Reserved.
|
||||
*/
|
||||
package org.owasp.dependencycheck.data.cpe;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
import java.util.logging.Level;
|
||||
import java.util.logging.Logger;
|
||||
import org.apache.lucene.analysis.Analyzer;
|
||||
import org.apache.lucene.analysis.core.KeywordAnalyzer;
|
||||
import org.apache.lucene.analysis.miscellaneous.PerFieldAnalyzerWrapper;
|
||||
import org.apache.lucene.document.Document;
|
||||
import org.apache.lucene.document.Field;
|
||||
import org.apache.lucene.document.StringField;
|
||||
import org.apache.lucene.document.TextField;
|
||||
import org.apache.lucene.index.CorruptIndexException;
|
||||
import org.apache.lucene.index.IndexWriter;
|
||||
import org.apache.lucene.index.IndexWriterConfig;
|
||||
import org.apache.lucene.index.Term;
|
||||
import org.apache.lucene.util.Version;
|
||||
import org.owasp.dependencycheck.data.lucene.FieldAnalyzer;
|
||||
|
||||
/**
|
||||
*
|
||||
* @author Jeremy Long (jeremy.long@owasp.org)
|
||||
*/
|
||||
public class CpeIndexWriter extends BaseIndex {
|
||||
|
||||
/**
|
||||
* The IndexWriter for the Lucene index.
|
||||
*/
|
||||
private IndexWriter indexWriter;
|
||||
/**
|
||||
* The Lucene Analyzer used for Indexing.
|
||||
*/
|
||||
private Analyzer indexingAnalyzer;
|
||||
|
||||
/**
|
||||
* Opens the CPE Index.
|
||||
*
|
||||
* @throws IOException is thrown if an IOException occurs opening the index.
|
||||
*/
|
||||
@Override
|
||||
public void open() throws IOException {
|
||||
//TODO add spinlock
|
||||
super.open();
|
||||
indexingAnalyzer = createIndexingAnalyzer();
|
||||
final IndexWriterConfig conf = new IndexWriterConfig(Version.LUCENE_43, indexingAnalyzer);
|
||||
indexWriter = new IndexWriter(directory, conf);
|
||||
}
|
||||
|
||||
/**
|
||||
* Closes the CPE Index.
|
||||
*/
|
||||
@Override
|
||||
public void close() {
|
||||
//TODO remove spinlock
|
||||
if (indexWriter != null) {
|
||||
commit();
|
||||
try {
|
||||
indexWriter.close(true);
|
||||
} catch (CorruptIndexException ex) {
|
||||
final String msg = "Unable to update database, there is a corrupt index.";
|
||||
Logger.getLogger(CpeIndexWriter.class.getName()).log(Level.SEVERE, msg);
|
||||
Logger.getLogger(CpeIndexWriter.class.getName()).log(Level.FINE, null, ex);
|
||||
} catch (IOException ex) {
|
||||
final String msg = "Unable to update database due to an IO error.";
|
||||
Logger.getLogger(CpeIndexWriter.class.getName()).log(Level.SEVERE, msg);
|
||||
Logger.getLogger(CpeIndexWriter.class.getName()).log(Level.FINE, null, ex);
|
||||
} finally {
|
||||
indexWriter = null;
|
||||
}
|
||||
}
|
||||
if (indexingAnalyzer != null) {
|
||||
indexingAnalyzer.close();
|
||||
indexingAnalyzer = null;
|
||||
}
|
||||
super.close();
|
||||
}
|
||||
|
||||
/**
|
||||
* Commits any pending changes.
|
||||
*/
|
||||
public void commit() {
|
||||
if (indexWriter != null) {
|
||||
try {
|
||||
indexWriter.forceMerge(1);
|
||||
indexWriter.commit();
|
||||
} catch (CorruptIndexException ex) {
|
||||
final String msg = "Unable to update database, there is a corrupt index.";
|
||||
Logger.getLogger(CpeIndexWriter.class.getName()).log(Level.SEVERE, msg);
|
||||
Logger.getLogger(CpeIndexWriter.class.getName()).log(Level.FINE, null, ex);
|
||||
} catch (IOException ex) {
|
||||
final String msg = "Unable to update database due to an IO error.";
|
||||
Logger.getLogger(CpeIndexWriter.class.getName()).log(Level.SEVERE, msg);
|
||||
Logger.getLogger(CpeIndexWriter.class.getName()).log(Level.FINE, null, ex);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates the indexing analyzer for the CPE Index.
|
||||
*
|
||||
* @return the CPE Analyzer.
|
||||
*/
|
||||
@SuppressWarnings("unchecked")
|
||||
private Analyzer createIndexingAnalyzer() {
|
||||
final Map fieldAnalyzers = new HashMap();
|
||||
fieldAnalyzers.put(Fields.DOCUMENT_KEY, new KeywordAnalyzer());
|
||||
return new PerFieldAnalyzerWrapper(new FieldAnalyzer(Version.LUCENE_43), fieldAnalyzers);
|
||||
}
|
||||
|
||||
/**
|
||||
* Saves a CPE IndexEntry into the Lucene index.
|
||||
*
|
||||
* @param entry a CPE entry.
|
||||
* @throws CorruptIndexException is thrown if the index is corrupt.
|
||||
* @throws IOException is thrown if an IOException occurs.
|
||||
*/
|
||||
public void saveEntry(IndexEntry entry) throws CorruptIndexException, IOException {
|
||||
final Document doc = new Document();
|
||||
final Field documentKey = new StringField(Fields.DOCUMENT_KEY, entry.getDocumentId(), Field.Store.NO);
|
||||
final Field vendor = new TextField(Fields.VENDOR, entry.getVendor(), Field.Store.YES);
|
||||
final Field product = new TextField(Fields.PRODUCT, entry.getProduct(), Field.Store.YES);
|
||||
doc.add(documentKey);
|
||||
doc.add(vendor);
|
||||
doc.add(product);
|
||||
|
||||
final Term term = new Term(Fields.DOCUMENT_KEY, entry.getDocumentId());
|
||||
indexWriter.updateDocument(term, doc);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,256 @@
|
||||
/*
|
||||
* This file is part of dependency-check-core.
|
||||
*
|
||||
* Dependency-check-core is free software: you can redistribute it and/or modify it
|
||||
* under the terms of the GNU General Public License as published by the Free
|
||||
* Software Foundation, either version 3 of the License, or (at your option) any
|
||||
* later version.
|
||||
*
|
||||
* Dependency-check-core is distributed in the hope that it will be useful, but
|
||||
* WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
|
||||
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
|
||||
* details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License along with
|
||||
* dependency-check-core. If not, see http://www.gnu.org/licenses/.
|
||||
*
|
||||
* Copyright (c) 2012 Jeremy Long. All Rights Reserved.
|
||||
*/
|
||||
package org.owasp.dependencycheck.analyzer;
|
||||
|
||||
import org.owasp.dependencycheck.data.cpe.IndexEntry;
|
||||
import org.owasp.dependencycheck.analyzer.CPEAnalyzer;
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
import org.apache.lucene.index.CorruptIndexException;
|
||||
import org.apache.lucene.queryparser.classic.ParseException;
|
||||
import org.junit.After;
|
||||
import org.junit.AfterClass;
|
||||
import org.owasp.dependencycheck.dependency.Dependency;
|
||||
import org.owasp.dependencycheck.analyzer.JarAnalyzer;
|
||||
import org.junit.Assert;
|
||||
import org.junit.Before;
|
||||
import org.junit.BeforeClass;
|
||||
import org.junit.Test;
|
||||
import org.owasp.dependencycheck.analyzer.FalsePositiveAnalyzer;
|
||||
import org.owasp.dependencycheck.analyzer.FileNameAnalyzer;
|
||||
import org.owasp.dependencycheck.analyzer.HintAnalyzer;
|
||||
import org.owasp.dependencycheck.data.cpe.BaseIndexTestCase;
|
||||
import org.owasp.dependencycheck.data.cpe.IndexEntry;
|
||||
import static org.owasp.dependencycheck.data.cpe.BaseIndexTestCase.ensureIndexExists;
|
||||
import org.owasp.dependencycheck.dependency.Identifier;
|
||||
|
||||
/**
|
||||
*
|
||||
* @author Jeremy Long (jeremy.long@owasp.org)
|
||||
*/
|
||||
public class CPEAnalyzerTest extends BaseIndexTestCase {
|
||||
|
||||
@BeforeClass
|
||||
public static void setUpClass() throws Exception {
|
||||
}
|
||||
|
||||
@AfterClass
|
||||
public static void tearDownClass() throws Exception {
|
||||
}
|
||||
|
||||
@Before
|
||||
public void setUp() throws Exception {
|
||||
super.setUp();
|
||||
}
|
||||
|
||||
@After
|
||||
public void tearDown() throws Exception {
|
||||
super.tearDown();
|
||||
}
|
||||
|
||||
/**
|
||||
* Tests of buildSearch of class CPEAnalyzer.
|
||||
*
|
||||
* @throws IOException is thrown when an IO Exception occurs.
|
||||
* @throws CorruptIndexException is thrown when the index is corrupt.
|
||||
* @throws ParseException is thrown when a parse exception occurs
|
||||
*/
|
||||
@Test
|
||||
public void testBuildSearch() throws IOException, CorruptIndexException, ParseException {
|
||||
Set<String> productWeightings = new HashSet<String>(1);
|
||||
productWeightings.add("struts2");
|
||||
|
||||
Set<String> vendorWeightings = new HashSet<String>(1);
|
||||
vendorWeightings.add("apache");
|
||||
|
||||
String vendor = "apache software foundation";
|
||||
String product = "struts 2 core";
|
||||
String version = "2.1.2";
|
||||
CPEAnalyzer instance = new CPEAnalyzer();
|
||||
|
||||
String queryText = instance.buildSearch(vendor, product, null, null);
|
||||
String expResult = " product:( struts 2 core ) AND vendor:( apache software foundation ) ";
|
||||
Assert.assertTrue(expResult.equals(queryText));
|
||||
|
||||
queryText = instance.buildSearch(vendor, product, null, productWeightings);
|
||||
expResult = " product:( struts^5 struts2^5 2 core ) AND vendor:( apache software foundation ) ";
|
||||
Assert.assertTrue(expResult.equals(queryText));
|
||||
|
||||
queryText = instance.buildSearch(vendor, product, vendorWeightings, null);
|
||||
expResult = " product:( struts 2 core ) AND vendor:( apache^5 software foundation ) ";
|
||||
Assert.assertTrue(expResult.equals(queryText));
|
||||
|
||||
queryText = instance.buildSearch(vendor, product, vendorWeightings, productWeightings);
|
||||
expResult = " product:( struts^5 struts2^5 2 core ) AND vendor:( apache^5 software foundation ) ";
|
||||
Assert.assertTrue(expResult.equals(queryText));
|
||||
}
|
||||
|
||||
/**
|
||||
* Test of open method, of class CPEAnalyzer.
|
||||
*
|
||||
* @throws Exception is thrown when an exception occurs
|
||||
*/
|
||||
@Test
|
||||
public void testOpen() throws Exception {
|
||||
CPEAnalyzer instance = new CPEAnalyzer();
|
||||
Assert.assertFalse(instance.isOpen());
|
||||
instance.open();
|
||||
Assert.assertTrue(instance.isOpen());
|
||||
instance.close();
|
||||
Assert.assertFalse(instance.isOpen());
|
||||
}
|
||||
|
||||
/**
|
||||
* Test of determineCPE method, of class CPEAnalyzer.
|
||||
*
|
||||
* @throws Exception is thrown when an exception occurs
|
||||
*/
|
||||
@Test
|
||||
public void testDetermineCPE_full() throws Exception {
|
||||
callDetermineCPE_full("spring-context-support-2.5.5.jar", "cpe:/a:vmware:springsource_spring_framework:2.5.5");
|
||||
callDetermineCPE_full("spring-core-3.0.0.RELEASE.jar", "cpe:/a:vmware:springsource_spring_framework:3.0.0");
|
||||
callDetermineCPE_full("org.mortbay.jetty.jar", "cpe:/a:mortbay_jetty:jetty:4.2");
|
||||
callDetermineCPE_full("jaxb-xercesImpl-1.5.jar", null);
|
||||
callDetermineCPE_full("ehcache-core-2.2.0.jar", null);
|
||||
}
|
||||
|
||||
/**
|
||||
* Test of determineCPE method, of class CPEAnalyzer.
|
||||
*
|
||||
* @throws Exception is thrown when an exception occurs
|
||||
*/
|
||||
public void callDetermineCPE_full(String depName, String expResult) throws Exception {
|
||||
|
||||
File file = new File(this.getClass().getClassLoader().getResource(depName).getPath());
|
||||
Dependency dep = new Dependency(file);
|
||||
|
||||
FileNameAnalyzer fnAnalyzer = new FileNameAnalyzer();
|
||||
fnAnalyzer.analyze(dep, null);
|
||||
|
||||
JarAnalyzer jarAnalyzer = new JarAnalyzer();
|
||||
jarAnalyzer.analyze(dep, null);
|
||||
HintAnalyzer hAnalyzer = new HintAnalyzer();
|
||||
hAnalyzer.analyze(dep, null);
|
||||
|
||||
|
||||
CPEAnalyzer instance = new CPEAnalyzer();
|
||||
instance.open();
|
||||
instance.analyze(dep, null);
|
||||
instance.close();
|
||||
FalsePositiveAnalyzer fp = new FalsePositiveAnalyzer();
|
||||
fp.analyze(dep, null);
|
||||
|
||||
// for (Identifier i : dep.getIdentifiers()) {
|
||||
// System.out.println(i.getValue());
|
||||
// }
|
||||
if (expResult != null) {
|
||||
Identifier expIdentifier = new Identifier("cpe", expResult, expResult);
|
||||
Assert.assertTrue("Incorrect match: { dep:'" + dep.getFileName() + "' }", dep.getIdentifiers().contains(expIdentifier));
|
||||
} else {
|
||||
Assert.assertTrue("Match found when an Identifier should not have been found: { dep:'" + dep.getFileName() + "' }", dep.getIdentifiers().isEmpty());
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Test of determineCPE method, of class CPEAnalyzer.
|
||||
*
|
||||
* @throws Exception is thrown when an exception occurs
|
||||
*/
|
||||
@Test
|
||||
public void testDetermineCPE() throws Exception {
|
||||
File file = new File(this.getClass().getClassLoader().getResource("struts2-core-2.1.2.jar").getPath());
|
||||
//File file = new File(this.getClass().getClassLoader().getResource("axis2-adb-1.4.1.jar").getPath());
|
||||
Dependency struts = new Dependency(file);
|
||||
|
||||
FileNameAnalyzer fnAnalyzer = new FileNameAnalyzer();
|
||||
fnAnalyzer.analyze(struts, null);
|
||||
|
||||
JarAnalyzer jarAnalyzer = new JarAnalyzer();
|
||||
jarAnalyzer.analyze(struts, null);
|
||||
|
||||
|
||||
File fileCommonValidator = new File(this.getClass().getClassLoader().getResource("commons-validator-1.4.0.jar").getPath());
|
||||
Dependency commonValidator = new Dependency(fileCommonValidator);
|
||||
jarAnalyzer.analyze(commonValidator, null);
|
||||
|
||||
File fileSpring = new File(this.getClass().getClassLoader().getResource("spring-core-2.5.5.jar").getPath());
|
||||
Dependency spring = new Dependency(fileSpring);
|
||||
jarAnalyzer.analyze(spring, null);
|
||||
|
||||
File fileSpring3 = new File(this.getClass().getClassLoader().getResource("spring-core-3.0.0.RELEASE.jar").getPath());
|
||||
Dependency spring3 = new Dependency(fileSpring3);
|
||||
jarAnalyzer.analyze(spring3, null);
|
||||
|
||||
CPEAnalyzer instance = new CPEAnalyzer();
|
||||
instance.open();
|
||||
instance.determineCPE(commonValidator);
|
||||
instance.determineCPE(struts);
|
||||
instance.determineCPE(spring);
|
||||
instance.determineCPE(spring3);
|
||||
instance.close();
|
||||
|
||||
String expResult = "cpe:/a:apache:struts:2.1.2";
|
||||
Identifier expIdentifier = new Identifier("cpe", expResult, expResult);
|
||||
String expResultSpring = "cpe:/a:springsource:spring_framework:2.5.5";
|
||||
String expResultSpring3 = "cpe:/a:vmware:springsource_spring_framework:3.0.0";
|
||||
|
||||
Assert.assertTrue("Apache Common Validator - found an identifier?", commonValidator.getIdentifiers().isEmpty());
|
||||
Assert.assertTrue("Incorrect match size - struts", struts.getIdentifiers().size() >= 1);
|
||||
Assert.assertTrue("Incorrect match - struts", struts.getIdentifiers().contains(expIdentifier));
|
||||
Assert.assertTrue("Incorrect match size - spring3 - " + spring3.getIdentifiers().size(), spring3.getIdentifiers().size() >= 1);
|
||||
|
||||
//the following two only work if the HintAnalyzer is used.
|
||||
//Assert.assertTrue("Incorrect match size - spring", spring.getIdentifiers().size() == 1);
|
||||
//Assert.assertTrue("Incorrect match - spring", spring.getIdentifiers().get(0).getValue().equals(expResultSpring));
|
||||
|
||||
}
|
||||
|
||||
/**
|
||||
* Test of searchCPE method, of class CPEAnalyzer.
|
||||
*
|
||||
* @throws Exception is thrown when an exception occurs
|
||||
*/
|
||||
@Test
|
||||
public void testSearchCPE() throws Exception {
|
||||
String vendor = "apache software foundation";
|
||||
String product = "struts 2 core";
|
||||
String version = "2.1.2";
|
||||
String expResult = "cpe:/a:apache:struts:2.1.2";
|
||||
|
||||
CPEAnalyzer instance = new CPEAnalyzer();
|
||||
instance.open();
|
||||
|
||||
//TODO - yeah, not a very good test as the results are the same with or without weighting...
|
||||
Set<String> productWeightings = new HashSet<String>(1);
|
||||
productWeightings.add("struts2");
|
||||
|
||||
Set<String> vendorWeightings = new HashSet<String>(1);
|
||||
vendorWeightings.add("apache");
|
||||
|
||||
List<IndexEntry> result = instance.searchCPE(vendor, product, productWeightings, vendorWeightings);
|
||||
//TODO fix this assert
|
||||
//Assert.assertEquals(expResult, result.get(0).getName());
|
||||
|
||||
|
||||
instance.close();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,63 @@
|
||||
/*
|
||||
* This file is part of dependency-check-core.
|
||||
*
|
||||
* Dependency-check-core is free software: you can redistribute it and/or modify it
|
||||
* under the terms of the GNU General Public License as published by the Free
|
||||
* Software Foundation, either version 3 of the License, or (at your option) any
|
||||
* later version.
|
||||
*
|
||||
* Dependency-check-core is distributed in the hope that it will be useful, but
|
||||
* WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
|
||||
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
|
||||
* details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License along with
|
||||
* dependency-check-core. If not, see http://www.gnu.org/licenses/.
|
||||
*
|
||||
* Copyright (c) 2012 Jeremy Long. All Rights Reserved.
|
||||
*/
|
||||
package org.owasp.dependencycheck.data.cpe;
|
||||
|
||||
import org.owasp.dependencycheck.data.cpe.BaseIndex;
|
||||
import java.io.File;
|
||||
import org.junit.After;
|
||||
import org.junit.AfterClass;
|
||||
import org.junit.Before;
|
||||
import org.junit.BeforeClass;
|
||||
import org.junit.Test;
|
||||
import static org.junit.Assert.*;
|
||||
|
||||
/**
|
||||
*
|
||||
* @author Jeremy Long (jeremy.long@owasp.org)
|
||||
*/
|
||||
public class BaseIndexTest {
|
||||
|
||||
@BeforeClass
|
||||
public static void setUpClass() throws Exception {
|
||||
}
|
||||
|
||||
@AfterClass
|
||||
public static void tearDownClass() throws Exception {
|
||||
}
|
||||
|
||||
@Before
|
||||
public void setUp() {
|
||||
}
|
||||
|
||||
@After
|
||||
public void tearDown() {
|
||||
}
|
||||
|
||||
/**
|
||||
* Test of getDataDirectory method, of class BaseIndex.
|
||||
*
|
||||
* @throws Exception
|
||||
*/
|
||||
@Test
|
||||
public void testGetDataDirectory() throws Exception {
|
||||
String file = BaseIndex.getDataDirectory().getPath();
|
||||
String exp = File.separatorChar + "target" + File.separatorChar + "data" + File.separatorChar + "cpe";
|
||||
assertTrue(file.contains(exp));
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user