From e33b5c36ffee9c7f99c11f74736612668e6e9248 Mon Sep 17 00:00:00 2001 From: Jeremy Long Date: Sat, 24 Aug 2013 16:23:57 -0400 Subject: [PATCH] commiting deletions and no-ops Former-commit-id: a83093e257e1c6d9c65db7c77b5b90a403576c12 --- .../dependencycheck/data/cpe/CPEAnalyzer.java | 794 ------------------ .../owasp/dependencycheck/data/cpe/Index.java | 170 ---- .../data/lucene/AbstractIndex.java | 341 -------- .../data/nvdcve/xml/DatabaseUpdater.java | 11 +- .../data/nvdcve/xml/NvdCve20Handler.java | 8 +- .../data/cpe/CPEAnalyzerTest.java | 254 ------ .../dependencycheck/data/cpe/IndexTest.java | 103 --- 7 files changed, 10 insertions(+), 1671 deletions(-) delete mode 100644 dependency-check-core/src/main/java/org/owasp/dependencycheck/data/cpe/CPEAnalyzer.java delete mode 100644 dependency-check-core/src/main/java/org/owasp/dependencycheck/data/cpe/Index.java delete mode 100644 dependency-check-core/src/main/java/org/owasp/dependencycheck/data/lucene/AbstractIndex.java delete mode 100644 dependency-check-core/src/test/java/org/owasp/dependencycheck/data/cpe/CPEAnalyzerTest.java delete mode 100644 dependency-check-core/src/test/java/org/owasp/dependencycheck/data/cpe/IndexTest.java diff --git a/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/cpe/CPEAnalyzer.java b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/cpe/CPEAnalyzer.java deleted file mode 100644 index dbd4b9f0f..000000000 --- a/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/cpe/CPEAnalyzer.java +++ /dev/null @@ -1,794 +0,0 @@ -/* - * This file is part of dependency-check-core. - * - * Dependency-check-core is free software: you can redistribute it and/or modify it - * under the terms of the GNU General Public License as published by the Free - * Software Foundation, either version 3 of the License, or (at your option) any - * later version. - * - * Dependency-check-core is distributed in the hope that it will be useful, but - * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more - * details. - * - * You should have received a copy of the GNU General Public License along with - * dependency-check-core. If not, see http://www.gnu.org/licenses/. - * - * Copyright (c) 2012 Jeremy Long. All Rights Reserved. - */ -package org.owasp.dependencycheck.data.cpe; - -import java.io.IOException; -import java.io.UnsupportedEncodingException; -import java.net.URLEncoder; -import java.sql.SQLException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.Set; -import java.util.StringTokenizer; -import java.util.logging.Level; -import java.util.logging.Logger; -import org.apache.lucene.document.Document; -import org.apache.lucene.index.CorruptIndexException; -import org.apache.lucene.queryparser.classic.ParseException; -import org.apache.lucene.search.ScoreDoc; -import org.apache.lucene.search.TopDocs; -import org.owasp.dependencycheck.Engine; -import org.owasp.dependencycheck.analyzer.AnalysisException; -import org.owasp.dependencycheck.analyzer.AnalysisPhase; -import org.owasp.dependencycheck.data.lucene.LuceneUtils; -import org.owasp.dependencycheck.dependency.Dependency; -import org.owasp.dependencycheck.dependency.Evidence; -import org.owasp.dependencycheck.dependency.Evidence.Confidence; -import org.owasp.dependencycheck.dependency.EvidenceCollection; -import org.owasp.dependencycheck.analyzer.Analyzer; -import org.owasp.dependencycheck.data.nvdcve.CveDB; -import org.owasp.dependencycheck.data.nvdcve.DatabaseException; -import org.owasp.dependencycheck.dependency.Identifier; -import org.owasp.dependencycheck.dependency.VulnerableSoftware; -import org.owasp.dependencycheck.utils.DependencyVersion; -import org.owasp.dependencycheck.utils.DependencyVersionUtil; - -/** - * CPEAnalyzer is a utility class that takes a project dependency and attempts - * to discern if there is an associated CPE. It uses the evidence contained - * within the dependency to search the Lucene index. - * - * @author Jeremy Long (jeremy.long@owasp.org) - */ -public class CPEAnalyzer implements Analyzer { - - /** - * The maximum number of query results to return. - */ - static final int MAX_QUERY_RESULTS = 25; - /** - * The weighting boost to give terms when constructing the Lucene query. - */ - static final String WEIGHTING_BOOST = "^5"; - /** - * A string representation of a regular expression defining characters - * utilized within the CPE Names. - */ - static final String CLEANSE_CHARACTER_RX = "[^A-Za-z0-9 ._-]"; - /** - * A string representation of a regular expression used to remove all but - * alpha characters. - */ - static final String CLEANSE_NONALPHA_RX = "[^A-Za-z]*"; - /** - * The additional size to add to a new StringBuilder to account for extra - * data that will be written into the string. - */ - static final int STRING_BUILDER_BUFFER = 20; - /** - * The CPE Index. - */ - private Index cpe; - /** - * The CVE Database. - */ - private CveDB cve; - - /** - * Opens the data source. - * - * @throws IOException when the Lucene directory to be queried does not - * exist or is corrupt. - * @throws DatabaseException when the database throws an exception. This - * usually occurs when the database is in use by another process. - */ - public void open() throws IOException, DatabaseException { - cpe = new Index(); - cpe.open(); - cve = new CveDB(); - try { - cve.open(); - } catch (SQLException ex) { - Logger.getLogger(CPEAnalyzer.class.getName()).log(Level.FINE, null, ex); - throw new DatabaseException("Unable to open the cve db", ex); - } catch (ClassNotFoundException ex) { - Logger.getLogger(CPEAnalyzer.class.getName()).log(Level.FINE, null, ex); - throw new DatabaseException("Unable to open the cve db", ex); - } - } - - /** - * Closes the data source. - */ - @Override - public void close() { - cpe.close(); - cve.close(); - } - - /** - * Returns the status of the data source - is the index open. - * - * @return true or false. - */ - public boolean isOpen() { - return (cpe != null) && cpe.isOpen(); - } - - /** - * Ensures that the Lucene index is closed. - * - * @throws Throwable when a throwable is thrown. - */ - @Override - protected void finalize() throws Throwable { - super.finalize(); - if (isOpen()) { - close(); - } - } - - /** - * Searches the data store of CPE entries, trying to identify the CPE for - * the given dependency based on the evidence contained within. The - * dependency passed in is updated with any identified CPE values. - * - * @param dependency the dependency to search for CPE entries on. - * @throws CorruptIndexException is thrown when the Lucene index is corrupt. - * @throws IOException is thrown when an IOException occurs. - * @throws ParseException is thrown when the Lucene query cannot be parsed. - */ - protected void determineCPE(Dependency dependency) throws CorruptIndexException, IOException, ParseException { - Confidence vendorConf = Confidence.HIGHEST; - Confidence productConf = Confidence.HIGHEST; - - String vendors = addEvidenceWithoutDuplicateTerms("", dependency.getVendorEvidence(), vendorConf); - String products = addEvidenceWithoutDuplicateTerms("", dependency.getProductEvidence(), productConf); - - //boolean found = false; - int ctr = 0; - do { - if (!vendors.isEmpty() && !products.isEmpty()) { - final List entries = searchCPE(vendors, products, dependency.getProductEvidence().getWeighting(), - dependency.getVendorEvidence().getWeighting()); - - for (IndexEntry e : entries) { - if (verifyEntry(e, dependency)) { - //found = true; // we found a vendor/product pair. Now find version from the cve db. - final String vendor = e.getVendor(); - final String product = e.getProduct(); - // cve.getVersions(vendor, product); - determineIdentifiers(dependency, vendor, product); - } - } - } - //if (!found) { - vendorConf = reduceConfidence(vendorConf); - if (dependency.getVendorEvidence().contains(vendorConf)) { - //vendors += " " + dependency.getVendorEvidence().toString(vendorConf); - vendors = addEvidenceWithoutDuplicateTerms(vendors, dependency.getVendorEvidence(), vendorConf); - } - productConf = reduceConfidence(productConf); - if (dependency.getProductEvidence().contains(productConf)) { - //products += " " + dependency.getProductEvidence().toString(productConf); - products = addEvidenceWithoutDuplicateTerms(products, dependency.getProductEvidence(), productConf); - } - //} - //} while (!found && (++ctr) < 4); - } while ((++ctr) < 4); - } - - /** - * Returns the text created by concatenating the text and the values from - * the EvidenceCollection (filtered for a specific confidence). This - * attempts to prevent duplicate terms from being added.
Note, if - * the evidence is longer then 200 characters it will be truncated. - * - * @param text the base text. - * @param ec an EvidenceCollection - * @param confidenceFilter a Confidence level to filter the evidence by. - * @return the new evidence text - */ - private String addEvidenceWithoutDuplicateTerms(final String text, final EvidenceCollection ec, Confidence confidenceFilter) { - final String txt = (text == null) ? "" : text; - final StringBuilder sb = new StringBuilder(txt.length() + (20 * ec.size())); - sb.append(' ').append(txt).append(' '); - for (Evidence e : ec.iterator(confidenceFilter)) { - String value = e.getValue(); - - //hack to get around the fact that lucene does a really good job of recognizing domains and not - // splitting them. TODO - put together a better lucene analyzer specific to the domain. - if (value.startsWith("http://")) { - value = value.substring(7).replaceAll("\\.", " "); - } - if (value.startsWith("https://")) { - value = value.substring(8).replaceAll("\\.", " "); - } - if (sb.indexOf(" " + value + " ") < 0) { - sb.append(value).append(' '); - } - } - return sb.toString().trim(); - } - - /** - * Reduces the given confidence by one level. This returns LOW if the - * confidence passed in is not HIGH. - * - * @param c the confidence to reduce. - * @return One less then the confidence passed in. - */ - private Confidence reduceConfidence(final Confidence c) { - if (c == Confidence.HIGHEST) { - return Confidence.HIGH; - } else if (c == Confidence.HIGH) { - return Confidence.MEDIUM; - } else { - return Confidence.LOW; - } - } - - /** - *

Searches the Lucene CPE index to identify possible CPE entries - * associated with the supplied vendor, product, and version.

- * - *

If either the vendorWeightings or productWeightings lists have been - * populated this data is used to add weighting factors to the search.

- * - * @param vendor the text used to search the vendor field - * @param product the text used to search the product field - * @param vendorWeightings a list of strings to use to add weighting factors - * to the vendor field - * @param productWeightings Adds a list of strings that will be used to add - * weighting factors to the product search - * @return a list of possible CPE values - * @throws CorruptIndexException when the Lucene index is corrupt - * @throws IOException when the Lucene index is not found - * @throws ParseException when the generated query is not valid - */ - protected List searchCPE(String vendor, String product, - Set vendorWeightings, Set productWeightings) - throws CorruptIndexException, IOException, ParseException { - final ArrayList ret = new ArrayList(MAX_QUERY_RESULTS); - - final String searchString = buildSearch(vendor, product, vendorWeightings, productWeightings); - if (searchString == null) { - return ret; - } - - final TopDocs docs = cpe.search(searchString, MAX_QUERY_RESULTS); - for (ScoreDoc d : docs.scoreDocs) { - final Document doc = cpe.getDocument(d.doc); - final IndexEntry entry = new IndexEntry(); - entry.setVendor(doc.get(Fields.VENDOR)); - entry.setProduct(doc.get(Fields.PRODUCT)); - entry.setSearchScore(d.score); - if (!ret.contains(entry)) { - ret.add(entry); - } - } - return ret; - } - - /** - *

Builds a Lucene search string by properly escaping data and - * constructing a valid search query.

- * - *

If either the possibleVendor or possibleProducts lists have been - * populated this data is used to add weighting factors to the search string - * generated.

- * - * @param vendor text to search the vendor field - * @param product text to search the product field - * @param vendorWeighting a list of strings to apply to the vendor to boost - * the terms weight - * @param productWeightings a list of strings to apply to the product to - * boost the terms weight - * @return the Lucene query - */ - protected String buildSearch(String vendor, String product, - Set vendorWeighting, Set productWeightings) { - final String v = vendor; //.replaceAll("[^\\w\\d]", " "); - final String p = product; //.replaceAll("[^\\w\\d]", " "); - final StringBuilder sb = new StringBuilder(v.length() + p.length() - + Fields.PRODUCT.length() + Fields.VENDOR.length() + STRING_BUILDER_BUFFER); - - if (!appendWeightedSearch(sb, Fields.PRODUCT, p, productWeightings)) { - return null; - } - sb.append(" AND "); - if (!appendWeightedSearch(sb, Fields.VENDOR, v, vendorWeighting)) { - return null; - } - return sb.toString(); - } - - /** - * This method constructs a Lucene query for a given field. The searchText - * is split into separate words and if the word is within the list of - * weighted words then an additional weighting is applied to the term as it - * is appended into the query. - * - * @param sb a StringBuilder that the query text will be appended to. - * @param field the field within the Lucene index that the query is - * searching. - * @param searchText text used to construct the query. - * @param weightedText a list of terms that will be considered higher - * importance when searching. - * @return if the append was successful. - */ - private boolean appendWeightedSearch(StringBuilder sb, String field, String searchText, Set weightedText) { - sb.append(" ").append(field).append(":( "); - - final String cleanText = cleanseText(searchText); - - if ("".equals(cleanText)) { - return false; - } - - if (weightedText == null || weightedText.isEmpty()) { - LuceneUtils.appendEscapedLuceneQuery(sb, cleanText); - } else { - final StringTokenizer tokens = new StringTokenizer(cleanText); - while (tokens.hasMoreElements()) { - final String word = tokens.nextToken(); - String temp = null; - for (String weighted : weightedText) { - final String weightedStr = cleanseText(weighted); - if (equalsIgnoreCaseAndNonAlpha(word, weightedStr)) { - temp = LuceneUtils.escapeLuceneQuery(word) + WEIGHTING_BOOST; - if (!word.equalsIgnoreCase(weightedStr)) { - temp += " " + LuceneUtils.escapeLuceneQuery(weightedStr) + WEIGHTING_BOOST; - } - } - } - if (temp == null) { - temp = LuceneUtils.escapeLuceneQuery(word); - } - sb.append(" ").append(temp); - } - } - sb.append(" ) "); - return true; - } - - /** - * Removes characters from the input text that are not used within the CPE - * index. - * - * @param text is the text to remove the characters from. - * @return the text having removed some characters. - */ - private String cleanseText(String text) { - return text.replaceAll(CLEANSE_CHARACTER_RX, " "); - } - - /** - * Compares two strings after lower casing them and removing the non-alpha - * characters. - * - * @param l string one to compare. - * @param r string two to compare. - * @return whether or not the two strings are similar. - */ - private boolean equalsIgnoreCaseAndNonAlpha(String l, String r) { - if (l == null || r == null) { - return false; - } - - final String left = l.replaceAll(CLEANSE_NONALPHA_RX, ""); - final String right = r.replaceAll(CLEANSE_NONALPHA_RX, ""); - return left.equalsIgnoreCase(right); - } - - /** - * Ensures that the CPE Identified matches the dependency. This validates - * that the product, vendor, and version information for the CPE are - * contained within the dependencies evidence. - * - * @param entry a CPE entry. - * @param dependency the dependency that the CPE entries could be for. - * @return whether or not the entry is valid. - */ - private boolean verifyEntry(final IndexEntry entry, final Dependency dependency) { - boolean isValid = false; - - if (collectionContainsString(dependency.getProductEvidence(), entry.getProduct()) - && collectionContainsString(dependency.getVendorEvidence(), entry.getVendor())) { - //&& collectionContainsVersion(dependency.getVersionEvidence(), entry.getVersion()) - isValid = true; - } - return isValid; - } - - /** - * Used to determine if the EvidenceCollection contains a specific string. - * - * @param ec an EvidenceCollection - * @param text the text to search for - * @return whether or not the EvidenceCollection contains the string - */ - private boolean collectionContainsString(EvidenceCollection ec, String text) { - - // - // String[] splitText = text.split("[\\s_-]"); - // - // for (String search : splitText) { - // //final String search = text.replaceAll("[\\s_-]", "").toLowerCase(); - // if (ec.containsUsedString(search)) { - // return true; - // } - // } - // - - //TODO - likely need to change the split... not sure if this will work for CPE with special chars - final String[] words = text.split("[\\s_-]"); - final List list = new ArrayList(); - String tempWord = null; - for (String word : words) { - //single letter words should be concatonated with the next word. - // so { "m", "core", "sample" } -> { "mcore", "sample" } - if (tempWord != null) { - list.add(tempWord + word); - tempWord = null; - } else if (word.length() <= 2) { - tempWord = word; - } else { - list.add(word); - } - } -// if (tempWord != null) { -// //for now ignore any last single letter words... -// } - boolean contains = true; - for (String word : list) { - contains &= ec.containsUsedString(word); - } - return contains; - } - - /** - * Analyzes a dependency and attempts to determine if there are any CPE - * identifiers for this dependency. - * - * @param dependency The Dependency to analyze. - * @param engine The analysis engine - * @throws AnalysisException is thrown if there is an issue analyzing the - * dependency. - */ - @Override - public void analyze(Dependency dependency, Engine engine) throws AnalysisException { - try { - determineCPE(dependency); - } catch (CorruptIndexException ex) { - throw new AnalysisException("CPE Index is corrupt.", ex); - } catch (IOException ex) { - throw new AnalysisException("Failure opening the CPE Index.", ex); - } catch (ParseException ex) { - throw new AnalysisException("Unable to parse the generated Lucene query for this dependency.", ex); - } - } - - /** - * Returns true because this analyzer supports all dependency types. - * - * @return true. - */ - @Override - public Set getSupportedExtensions() { - return null; - } - - /** - * Returns the name of this analyzer. - * - * @return the name of this analyzer. - */ - @Override - public String getName() { - return "CPE Analyzer"; - } - - /** - * Returns true because this analyzer supports all dependency types. - * - * @param extension the file extension of the dependency being analyzed. - * @return true. - */ - @Override - public boolean supportsExtension(String extension) { - return true; - } - - /** - * Returns the analysis phase that this analyzer should run in. - * - * @return the analysis phase that this analyzer should run in. - */ - @Override - public AnalysisPhase getAnalysisPhase() { - return AnalysisPhase.IDENTIFIER_ANALYSIS; - } - - /** - * Opens the CPE Lucene Index. - * - * @throws Exception is thrown if there is an issue opening the index. - */ - @Override - public void initialize() throws Exception { - this.open(); - } - - /** - * Retrieves a list of CPE values from the CveDB based on the vendor and - * product passed in. The list is then validated to find only CPEs that are - * valid for the given dependency. It is possible that the CPE identified is - * a best effort "guess" based on the vendor, product, and version - * information. - * - * @param dependency the Dependency being analyzed - * @param vendor the vendor for the CPE being analyzed - * @param product the product for the CPE being analyzed - * @throws UnsupportedEncodingException is thrown if UTF-8 is not supported - */ - private void determineIdentifiers(Dependency dependency, String vendor, String product) throws UnsupportedEncodingException { - final Set cpes = cve.getCPEs(vendor, product); - DependencyVersion bestGuess = new DependencyVersion("-"); - Confidence bestGuessConf = null; - final List collected = new ArrayList(); - for (Confidence conf : Confidence.values()) { - for (Evidence evidence : dependency.getVersionEvidence().iterator(conf)) { - final DependencyVersion evVer = DependencyVersionUtil.parseVersion(evidence.getValue()); - if (evVer == null) { - continue; - } - for (VulnerableSoftware vs : cpes) { - DependencyVersion dbVer; - if (vs.getRevision() != null && !vs.getRevision().isEmpty()) { - dbVer = DependencyVersionUtil.parseVersion(vs.getVersion() + "." + vs.getRevision()); - } else { - dbVer = DependencyVersionUtil.parseVersion(vs.getVersion()); - } - if (dbVer == null //special case, no version specified - everything is vulnerable - || evVer.equals(dbVer)) { //woot exect match - final String url = String.format("http://web.nvd.nist.gov/view/vuln/search?cpe=%s", URLEncoder.encode(vs.getName(), "UTF-8")); - final IdentifierMatch match = new IdentifierMatch("cpe", vs.getName(), url, IdentifierConfidence.EXACT_MATCH, conf); - collected.add(match); - } else { - //TODO the following isn't quite right is it? need to think about this guessing game a bit more. - if (evVer.getVersionParts().size() <= dbVer.getVersionParts().size() - && evVer.matchesAtLeastThreeLevels(dbVer)) { - if (bestGuessConf == null || bestGuessConf.compareTo(conf) > 0) { - if (bestGuess.getVersionParts().size() < dbVer.getVersionParts().size()) { - bestGuess = dbVer; - bestGuessConf = conf; - } - } - } - } - } - if (bestGuessConf == null || bestGuessConf.compareTo(conf) > 0) { - if (bestGuess.getVersionParts().size() < evVer.getVersionParts().size()) { - bestGuess = evVer; - bestGuessConf = conf; - } - } - } - } - final String cpeName = String.format("cpe:/a:%s:%s:%s", vendor, product, bestGuess.toString()); - final String url = null; //String.format("http://web.nvd.nist.gov/view/vuln/search?cpe=%s", URLEncoder.encode(cpeName, "UTF-8")); - if (bestGuessConf == null) { - bestGuessConf = Confidence.LOW; - } - final IdentifierMatch match = new IdentifierMatch("cpe", cpeName, url, IdentifierConfidence.BEST_GUESS, bestGuessConf); - collected.add(match); - - Collections.sort(collected); - final IdentifierConfidence bestIdentifierQuality = collected.get(0).getConfidence(); - final Confidence bestEvidenceQuality = collected.get(0).getEvidenceConfidence(); - for (IdentifierMatch m : collected) { - if (bestIdentifierQuality.equals(m.getConfidence()) - && bestEvidenceQuality.equals(m.getEvidenceConfidence())) { - dependency.addIdentifier(m.getIdentifier()); - } - } - } - - /** - * The confidence whether the identifier is an exact match, or a best guess. - */ - private enum IdentifierConfidence { - - /** - * An exact match for the CPE. - */ - EXACT_MATCH, - /** - * A best guess for the CPE. - */ - BEST_GUESS - } - - /** - * A simple object to hold an identifier and carry information about the - * confidence in the identifier. - */ - private static class IdentifierMatch implements Comparable { - - /** - * Constructs an IdentiferMatch. - * - * @param type the type of identifier (such as CPE) - * @param value the value of the identifier - * @param url the URL of the identifier - * @param identifierConfidence the confidence in the identifier: best - * guess or exact match - * @param evidenceConfidence the confidence of the evidence used to find - * the identifier - */ - IdentifierMatch(String type, String value, String url, IdentifierConfidence identifierConfidence, Confidence evidenceConfidence) { - this.identifier = new Identifier(type, value, url); - this.confidence = identifierConfidence; - this.evidenceConfidence = evidenceConfidence; - } - // - /** - * The confidence in the evidence used to identify this match. - */ - private Confidence evidenceConfidence; - - /** - * Get the value of evidenceConfidence - * - * @return the value of evidenceConfidence - */ - public Confidence getEvidenceConfidence() { - return evidenceConfidence; - } - - /** - * Set the value of evidenceConfidence - * - * @param evidenceConfidence new value of evidenceConfidence - */ - public void setEvidenceConfidence(Confidence evidenceConfidence) { - this.evidenceConfidence = evidenceConfidence; - } - /** - * The confidence whether this is an exact match, or a best guess. - */ - private IdentifierConfidence confidence; - - /** - * Get the value of confidence. - * - * @return the value of confidence - */ - public IdentifierConfidence getConfidence() { - return confidence; - } - - /** - * Set the value of confidence. - * - * @param confidence new value of confidence - */ - public void setConfidence(IdentifierConfidence confidence) { - this.confidence = confidence; - } - /** - * The CPE identifier. - */ - private Identifier identifier; - - /** - * Get the value of identifier. - * - * @return the value of identifier - */ - public Identifier getIdentifier() { - return identifier; - } - - /** - * Set the value of identifier. - * - * @param identifier new value of identifier - */ - public void setIdentifier(Identifier identifier) { - this.identifier = identifier; - } - // - // - - /** - * Standard toString() implementation. - * - * @return the string representation of the object - */ - @Override - public String toString() { - return "IdentifierMatch{" + "evidenceConfidence=" + evidenceConfidence - + ", confidence=" + confidence + ", identifier=" + identifier + '}'; - } - - /** - * Standard hashCode() implementation. - * - * @return the hashCode - */ - @Override - public int hashCode() { - int hash = 5; - hash = 97 * hash + (this.evidenceConfidence != null ? this.evidenceConfidence.hashCode() : 0); - hash = 97 * hash + (this.confidence != null ? this.confidence.hashCode() : 0); - hash = 97 * hash + (this.identifier != null ? this.identifier.hashCode() : 0); - return hash; - } - - /** - * Standard equals implementation. - * - * @param obj the object to compare - * @return true if the objects are equal, otherwise false - */ - @Override - public boolean equals(Object obj) { - if (obj == null) { - return false; - } - if (getClass() != obj.getClass()) { - return false; - } - final IdentifierMatch other = (IdentifierMatch) obj; - if (this.evidenceConfidence != other.evidenceConfidence) { - return false; - } - if (this.confidence != other.confidence) { - return false; - } - if (this.identifier != other.identifier && (this.identifier == null || !this.identifier.equals(other.identifier))) { - return false; - } - return true; - } - // - - /** - * Standard implementation of compareTo that compares identifier - * confidence, evidence confidence, and then the identifier. - * - * @param o the IdentifierMatch to compare to - * @return the natural ordering of IdentifierMatch - */ - @Override - public int compareTo(IdentifierMatch o) { - int conf = this.confidence.compareTo(o.confidence); - if (conf == 0) { - conf = this.evidenceConfidence.compareTo(o.evidenceConfidence); - if (conf == 0) { - conf = identifier.compareTo(o.identifier); - } - } - return conf; - } - } -} diff --git a/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/cpe/Index.java b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/cpe/Index.java deleted file mode 100644 index d63a7a189..000000000 --- a/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/cpe/Index.java +++ /dev/null @@ -1,170 +0,0 @@ -/* - * This file is part of dependency-check-core. - * - * Dependency-check-core is free software: you can redistribute it and/or modify it - * under the terms of the GNU General Public License as published by the Free - * Software Foundation, either version 3 of the License, or (at your option) any - * later version. - * - * Dependency-check-core is distributed in the hope that it will be useful, but - * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more - * details. - * - * You should have received a copy of the GNU General Public License along with - * dependency-check-core. If not, see http://www.gnu.org/licenses/. - * - * Copyright (c) 2012 Jeremy Long. All Rights Reserved. - */ -package org.owasp.dependencycheck.data.cpe; - -import java.io.File; -import java.io.IOException; -import java.util.HashMap; -import java.util.Map; -import org.apache.lucene.analysis.Analyzer; -import org.apache.lucene.analysis.core.KeywordAnalyzer; -import org.apache.lucene.analysis.miscellaneous.PerFieldAnalyzerWrapper; -import org.apache.lucene.document.Document; -import org.apache.lucene.document.Field; -import org.apache.lucene.document.TextField; -import org.apache.lucene.index.CorruptIndexException; -import org.apache.lucene.index.Term; -import org.apache.lucene.queryparser.classic.QueryParser; -import org.apache.lucene.store.Directory; -import org.apache.lucene.store.FSDirectory; -import org.apache.lucene.util.Version; -import org.owasp.dependencycheck.data.lucene.AbstractIndex; -import org.owasp.dependencycheck.utils.Settings; -import org.owasp.dependencycheck.data.lucene.FieldAnalyzer; -import org.owasp.dependencycheck.data.lucene.SearchFieldAnalyzer; - -/** - * The Index class is used to utilize and maintain the CPE Index. - * - * @author Jeremy Long (jeremy.long@owasp.org) - */ -public class Index extends AbstractIndex { - - /** - * Returns the directory that holds the CPE Index. - * - * @return the Directory containing the CPE Index. - * @throws IOException is thrown if an IOException occurs. - */ - @Override - public Directory getDirectory() throws IOException { - final File path = getDataDirectory(); - return FSDirectory.open(path); - } - - /** - * Retrieves the directory that the JAR file exists in so that we can ensure - * we always use a common data directory. - * - * @return the data directory for this index. - * @throws IOException is thrown if an IOException occurs of course... - */ - public static File getDataDirectory() throws IOException { - final File path = Settings.getFile(Settings.KEYS.CPE_DATA_DIRECTORY); - if (!path.exists()) { - if (!path.mkdirs()) { - throw new IOException("Unable to create CPE Data directory"); - } - } - return path; - } - - /** - * Creates an Analyzer for the CPE Index. - * - * @return the CPE Analyzer. - */ - @SuppressWarnings("unchecked") - @Override - public Analyzer createIndexingAnalyzer() { - final Map fieldAnalyzers = new HashMap(); - fieldAnalyzers.put(Fields.DOCUMENT_KEY, new KeywordAnalyzer()); - return new PerFieldAnalyzerWrapper(new FieldAnalyzer(Version.LUCENE_43), fieldAnalyzers); - } - /** - * The search field analyzer for the product field. - */ - private SearchFieldAnalyzer productSearchFieldAnalyzer; - /** - * The search field analyzer for the vendor field. - */ - private SearchFieldAnalyzer vendorSearchFieldAnalyzer; - - /** - * Creates an Analyzer for searching the CPE Index. - * - * @return the CPE Analyzer. - */ - @SuppressWarnings("unchecked") - @Override - public Analyzer createSearchingAnalyzer() { - final Map fieldAnalyzers = new HashMap(); - - fieldAnalyzers.put(Fields.DOCUMENT_KEY, new KeywordAnalyzer()); - productSearchFieldAnalyzer = new SearchFieldAnalyzer(Version.LUCENE_43); - vendorSearchFieldAnalyzer = new SearchFieldAnalyzer(Version.LUCENE_43); - fieldAnalyzers.put(Fields.PRODUCT, productSearchFieldAnalyzer); - fieldAnalyzers.put(Fields.VENDOR, vendorSearchFieldAnalyzer); - - return new PerFieldAnalyzerWrapper(new FieldAnalyzer(Version.LUCENE_43), fieldAnalyzers); - } - - /** - * Creates the Lucene QueryParser used when querying the index. - * - * @return a QueryParser. - */ - @Override - public QueryParser createQueryParser() { - return new QueryParser(Version.LUCENE_43, Fields.DOCUMENT_KEY, getSearchingAnalyzer()); - } - - /** - * Resets the searching analyzers - */ - @Override - protected void resetSearchingAnalyzer() { - if (productSearchFieldAnalyzer != null) { - productSearchFieldAnalyzer.clear(); - } - if (vendorSearchFieldAnalyzer != null) { - vendorSearchFieldAnalyzer.clear(); - } - } - - /** - * Saves a CPE IndexEntry into the Lucene index. - * - * @param entry a CPE entry. - * @throws CorruptIndexException is thrown if the index is corrupt. - * @throws IOException is thrown if an IOException occurs. - */ - public void saveEntry(IndexEntry entry) throws CorruptIndexException, IOException { - final Document doc = convertEntryToDoc(entry); - final Term term = new Term(Fields.DOCUMENT_KEY, entry.getDocumentId()); - getIndexWriter().updateDocument(term, doc); - } - - /** - * Converts a CPE entry into a Lucene Document. - * - * @param entry a CPE IndexEntry. - * @return a Lucene Document containing a CPE IndexEntry. - */ - protected Document convertEntryToDoc(IndexEntry entry) { - final Document doc = new Document(); - - final Field vendor = new TextField(Fields.VENDOR, entry.getVendor(), Field.Store.YES); - doc.add(vendor); - - final Field product = new TextField(Fields.PRODUCT, entry.getProduct(), Field.Store.YES); - doc.add(product); - return doc; - } -} diff --git a/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/lucene/AbstractIndex.java b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/lucene/AbstractIndex.java deleted file mode 100644 index 2fb2ad66f..000000000 --- a/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/lucene/AbstractIndex.java +++ /dev/null @@ -1,341 +0,0 @@ -/* - * This file is part of dependency-check-core. - * - * Dependency-check-core is free software: you can redistribute it and/or modify it - * under the terms of the GNU General Public License as published by the Free - * Software Foundation, either version 3 of the License, or (at your option) any - * later version. - * - * Dependency-check-core is distributed in the hope that it will be useful, but - * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more - * details. - * - * You should have received a copy of the GNU General Public License along with - * dependency-check-core. If not, see http://www.gnu.org/licenses/. - * - * Copyright (c) 2012 Jeremy Long. All Rights Reserved. - */ -package org.owasp.dependencycheck.data.lucene; - -import java.io.IOException; -import java.util.logging.Level; -import java.util.logging.Logger; -import org.apache.lucene.analysis.Analyzer; -import org.apache.lucene.document.Document; -import org.apache.lucene.index.CorruptIndexException; -import org.apache.lucene.index.DirectoryReader; -import org.apache.lucene.index.IndexReader; -import org.apache.lucene.index.IndexWriter; -import org.apache.lucene.index.IndexWriterConfig; -import org.apache.lucene.queryparser.classic.ParseException; -import org.apache.lucene.queryparser.classic.QueryParser; -import org.apache.lucene.search.IndexSearcher; -import org.apache.lucene.search.Query; -import org.apache.lucene.search.TopDocs; -import org.apache.lucene.store.Directory; -import org.apache.lucene.store.LockObtainFailedException; -import org.apache.lucene.util.Version; - -/** - * The base Index for other index objects. Implements the open and close - * methods. - * - * @author Jeremy Long (jeremy.long@owasp.org) - */ -public abstract class AbstractIndex { - - /** - * The Lucene directory containing the index. - */ - private Directory directory; - /** - * The IndexWriter for the Lucene index. - */ - private IndexWriter indexWriter; - /** - * The Lucene IndexReader. - */ - private IndexReader indexReader; - /** - * The Lucene IndexSearcher. - */ - private IndexSearcher indexSearcher; - /** - * The Lucene Analyzer used for Indexing. - */ - private Analyzer indexingAnalyzer; - /** - * The Lucene Analyzer used for Searching. - */ - private Analyzer searchingAnalyzer; - /** - * The Lucene QueryParser used for Searching. - */ - private QueryParser queryParser; - /** - * Indicates whether or not the Lucene Index is open. - */ - private boolean indexOpen = false; - - /** - * Opens the CPE Index. - * - * @throws IOException is thrown if an IOException occurs opening the index. - */ - public void open() throws IOException { - directory = this.getDirectory(); - indexingAnalyzer = this.getIndexingAnalyzer(); - searchingAnalyzer = this.getSearchingAnalyzer(); - indexOpen = true; - } - - /** - * Commits any pending changes. - */ - public void commit() { - if (indexWriter != null) { - try { - indexWriter.commit(); - } catch (CorruptIndexException ex) { - final String msg = "Unable to update database, there is a corrupt index."; - Logger.getLogger(AbstractIndex.class.getName()).log(Level.SEVERE, msg); - Logger.getLogger(AbstractIndex.class.getName()).log(Level.FINE, null, ex); - } catch (IOException ex) { - final String msg = "Unable to update database due to an IO error."; - Logger.getLogger(AbstractIndex.class.getName()).log(Level.SEVERE, msg); - Logger.getLogger(AbstractIndex.class.getName()).log(Level.FINE, null, ex); - } - } - } - - /** - * Closes the CPE Index. - */ - public void close() { - if (indexWriter != null) { - commit(); - try { - indexWriter.close(true); - } catch (CorruptIndexException ex) { - final String msg = "Unable to update database, there is a corrupt index."; - Logger.getLogger(AbstractIndex.class.getName()).log(Level.SEVERE, msg); - Logger.getLogger(AbstractIndex.class.getName()).log(Level.FINE, null, ex); - } catch (IOException ex) { - final String msg = "Unable to update database due to an IO error."; - Logger.getLogger(AbstractIndex.class.getName()).log(Level.SEVERE, msg); - Logger.getLogger(AbstractIndex.class.getName()).log(Level.FINE, null, ex); - } finally { - indexWriter = null; - } - } - if (indexSearcher != null) { - indexSearcher = null; - } - - if (indexingAnalyzer != null) { - indexingAnalyzer.close(); - indexingAnalyzer = null; - } - - if (searchingAnalyzer != null) { - searchingAnalyzer.close(); - searchingAnalyzer = null; - } - - try { - directory.close(); - } catch (IOException ex) { - final String msg = "Unable to update database due to an IO error."; - Logger.getLogger(AbstractIndex.class.getName()).log(Level.SEVERE, msg); - Logger.getLogger(AbstractIndex.class.getName()).log(Level.FINE, null, ex); - } finally { - directory = null; - } - indexOpen = false; - } - - /** - * Returns the status of the data source - is the index open. - * - * @return true or false. - */ - public boolean isOpen() { - return indexOpen; - } - - /** - * Opens the Lucene Index Writer. - * - * @throws CorruptIndexException is thrown if the Lucene index is corrupt. - * @throws IOException is thrown if an IOException occurs opening the index. - */ - public void openIndexWriter() throws CorruptIndexException, IOException { - if (!isOpen()) { - open(); - } - final IndexWriterConfig conf = new IndexWriterConfig(Version.LUCENE_43, indexingAnalyzer); - indexWriter = new IndexWriter(directory, conf); - } - - /** - * Retrieves the IndexWriter for the Lucene Index. - * - * @return an IndexWriter. - * @throws CorruptIndexException is thrown if the Lucene Index is corrupt. - * @throws LockObtainFailedException is thrown if there is an exception - * obtaining a lock on the Lucene index. - * @throws IOException is thrown if an IOException occurs opening the index. - */ - public IndexWriter getIndexWriter() throws CorruptIndexException, LockObtainFailedException, IOException { - if (indexWriter == null) { - openIndexWriter(); - } - return indexWriter; - } - - /** - * Opens the Lucene Index for reading. - * - * @throws CorruptIndexException is thrown if the index is corrupt. - * @throws IOException is thrown if there is an exception reading the index. - */ - public void openIndexReader() throws CorruptIndexException, IOException { - if (!isOpen()) { - open(); - } - //indexReader = IndexReader.open(directory, true); - indexReader = DirectoryReader.open(directory); - } - - /** - * Returns an IndexSearcher for the Lucene Index. - * - * @return an IndexSearcher. - * @throws CorruptIndexException is thrown if the index is corrupt. - * @throws IOException is thrown if there is an exception reading the index. - */ - protected IndexSearcher getIndexSearcher() throws CorruptIndexException, IOException { - if (indexReader == null) { - openIndexReader(); - } - if (indexSearcher == null) { - indexSearcher = new IndexSearcher(indexReader); - } - return indexSearcher; - } - - /** - * Returns an Analyzer to be used when indexing. - * - * @return an Analyzer. - */ - public Analyzer getIndexingAnalyzer() { - if (indexingAnalyzer == null) { - indexingAnalyzer = createIndexingAnalyzer(); - } - return indexingAnalyzer; - } - - /** - * Returns an analyzer used for searching the index - * - * @return a lucene analyzer - */ - protected Analyzer getSearchingAnalyzer() { - if (searchingAnalyzer == null) { - searchingAnalyzer = createSearchingAnalyzer(); - } - return searchingAnalyzer; - } - - /** - * Gets a query parser - * - * @return a query parser - */ - protected QueryParser getQueryParser() { - if (queryParser == null) { - queryParser = createQueryParser(); - } - return queryParser; - } - - /** - * Searches the index using the given search string. - * - * @param searchString the query text - * @param maxQueryResults the maximum number of documents to return - * @return the TopDocs found by the search - * @throws ParseException thrown when the searchString is invalid - * @throws IOException is thrown if there is an issue with the underlying - * Index - */ - public TopDocs search(String searchString, int maxQueryResults) throws ParseException, IOException { - final QueryParser parser = getQueryParser(); - final Query query = parser.parse(searchString); - resetSearchingAnalyzer(); - final IndexSearcher is = getIndexSearcher(); - return is.search(query, maxQueryResults); - } - - /** - * Searches the index using the given query. - * - * @param query the query used to search the index - * @param maxQueryResults the max number of results to return - * @return the TopDocs found be the query - * @throws CorruptIndexException thrown if the Index is corrupt - * @throws IOException thrown if there is an IOException - */ - public TopDocs search(Query query, int maxQueryResults) throws CorruptIndexException, IOException { - final IndexSearcher is = getIndexSearcher(); - return is.search(query, maxQueryResults); - } - - /** - * Retrieves a document from the Index. - * - * @param documentId the id of the document to retrieve - * @return the Document - * @throws IOException thrown if there is an IOException - */ - public Document getDocument(int documentId) throws IOException { - final IndexSearcher is = getIndexSearcher(); - return is.doc(documentId); - } - - /** - * Gets the directory that contains the Lucene Index. - * - * @return a Lucene Directory - * @throws IOException is thrown when an IOException occurs - */ - public abstract Directory getDirectory() throws IOException; - - /** - * Creates the Lucene Analyzer used when indexing. - * - * @return a Lucene Analyzer - */ - public abstract Analyzer createIndexingAnalyzer(); - - /** - * Creates the Lucene Analyzer used when querying the index. - * - * @return a Lucene Analyzer - */ - public abstract Analyzer createSearchingAnalyzer(); - - /** - * Creates the Lucene QueryParser used when querying the index. - * - * @return a QueryParser - */ - public abstract QueryParser createQueryParser(); - - /** - * Resets the searching analyzers - */ - protected abstract void resetSearchingAnalyzer(); -} diff --git a/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/nvdcve/xml/DatabaseUpdater.java b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/nvdcve/xml/DatabaseUpdater.java index af160e3ce..d65fad8b4 100644 --- a/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/nvdcve/xml/DatabaseUpdater.java +++ b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/nvdcve/xml/DatabaseUpdater.java @@ -43,7 +43,8 @@ import java.util.logging.Logger; import javax.xml.parsers.SAXParser; import javax.xml.parsers.SAXParserFactory; import org.owasp.dependencycheck.data.UpdateException; -import org.owasp.dependencycheck.data.cpe.Index; +import org.owasp.dependencycheck.data.cpe.BaseIndex; +import org.owasp.dependencycheck.data.cpe.CpeIndexWriter; import org.owasp.dependencycheck.data.nvdcve.CveDB; import org.owasp.dependencycheck.dependency.VulnerableSoftware; import org.owasp.dependencycheck.utils.DownloadFailedException; @@ -87,7 +88,7 @@ public class DatabaseUpdater implements CachedWebDataSource { /** * Reference to the Cpe Index. */ - private Index cpeIndex = null; + private CpeIndexWriter cpeIndex = null; public DatabaseUpdater() { batchUpdateMode = !Settings.getString(Settings.KEYS.BATCH_UPDATE_URL, "").isEmpty(); @@ -273,8 +274,8 @@ public class DatabaseUpdater implements CachedWebDataSource { try { cveDB = new CveDB(); cveDB.open(); - cpeIndex = new Index(); - cpeIndex.openIndexWriter(); + cpeIndex = new CpeIndexWriter(); + cpeIndex.open(); } catch (IOException ex) { closeDataStores(); Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.FINE, "IO Error opening databases", ex); @@ -640,7 +641,7 @@ public class DatabaseUpdater implements CachedWebDataSource { final File cveDir = CveDB.getDataDirectory(); FileUtils.delete(cveDir); - final File cpeDir = Index.getDataDirectory(); + final File cpeDir = BaseIndex.getDataDirectory(); FileUtils.delete(cpeDir); } diff --git a/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/nvdcve/xml/NvdCve20Handler.java b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/nvdcve/xml/NvdCve20Handler.java index b1ee760ba..3f5808466 100644 --- a/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/nvdcve/xml/NvdCve20Handler.java +++ b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/nvdcve/xml/NvdCve20Handler.java @@ -24,7 +24,7 @@ import java.util.Map; import java.util.logging.Level; import java.util.logging.Logger; import org.apache.lucene.index.CorruptIndexException; -import org.owasp.dependencycheck.data.cpe.Index; +import org.owasp.dependencycheck.data.cpe.CpeIndexWriter; import org.owasp.dependencycheck.data.nvdcve.CveDB; import org.owasp.dependencycheck.data.nvdcve.DatabaseException; import org.owasp.dependencycheck.dependency.Reference; @@ -274,14 +274,14 @@ public class NvdCve20Handler extends DefaultHandler { /** * the cpe index. */ - private Index cpeIndex; + private CpeIndexWriter cpeIndex; /** - * Sets the cpe index. + * Sets the cpe index writer. * * @param index the CPE Lucene Index */ - void setCpeIndex(Index index) { + void setCpeIndex(CpeIndexWriter index) { cpeIndex = index; } diff --git a/dependency-check-core/src/test/java/org/owasp/dependencycheck/data/cpe/CPEAnalyzerTest.java b/dependency-check-core/src/test/java/org/owasp/dependencycheck/data/cpe/CPEAnalyzerTest.java deleted file mode 100644 index b3584a068..000000000 --- a/dependency-check-core/src/test/java/org/owasp/dependencycheck/data/cpe/CPEAnalyzerTest.java +++ /dev/null @@ -1,254 +0,0 @@ -/* - * This file is part of dependency-check-core. - * - * Dependency-check-core is free software: you can redistribute it and/or modify it - * under the terms of the GNU General Public License as published by the Free - * Software Foundation, either version 3 of the License, or (at your option) any - * later version. - * - * Dependency-check-core is distributed in the hope that it will be useful, but - * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more - * details. - * - * You should have received a copy of the GNU General Public License along with - * dependency-check-core. If not, see http://www.gnu.org/licenses/. - * - * Copyright (c) 2012 Jeremy Long. All Rights Reserved. - */ -package org.owasp.dependencycheck.data.cpe; - -import org.owasp.dependencycheck.data.cpe.IndexEntry; -import org.owasp.dependencycheck.data.cpe.CPEAnalyzer; -import java.io.File; -import java.io.IOException; -import java.util.HashSet; -import java.util.List; -import java.util.Set; -import org.apache.lucene.index.CorruptIndexException; -import org.apache.lucene.queryparser.classic.ParseException; -import org.junit.After; -import org.junit.AfterClass; -import org.owasp.dependencycheck.dependency.Dependency; -import org.owasp.dependencycheck.analyzer.JarAnalyzer; -import org.junit.Assert; -import org.junit.Before; -import org.junit.BeforeClass; -import org.junit.Test; -import org.owasp.dependencycheck.analyzer.FalsePositiveAnalyzer; -import org.owasp.dependencycheck.analyzer.FileNameAnalyzer; -import org.owasp.dependencycheck.analyzer.HintAnalyzer; -import static org.owasp.dependencycheck.data.cpe.BaseIndexTestCase.ensureIndexExists; -import org.owasp.dependencycheck.dependency.Identifier; - -/** - * - * @author Jeremy Long (jeremy.long@owasp.org) - */ -public class CPEAnalyzerTest extends BaseIndexTestCase { - - @BeforeClass - public static void setUpClass() throws Exception { - } - - @AfterClass - public static void tearDownClass() throws Exception { - } - - @Before - public void setUp() throws Exception { - super.setUp(); - } - - @After - public void tearDown() throws Exception { - super.tearDown(); - } - - /** - * Tests of buildSearch of class CPEAnalyzer. - * - * @throws IOException is thrown when an IO Exception occurs. - * @throws CorruptIndexException is thrown when the index is corrupt. - * @throws ParseException is thrown when a parse exception occurs - */ - @Test - public void testBuildSearch() throws IOException, CorruptIndexException, ParseException { - Set productWeightings = new HashSet(1); - productWeightings.add("struts2"); - - Set vendorWeightings = new HashSet(1); - vendorWeightings.add("apache"); - - String vendor = "apache software foundation"; - String product = "struts 2 core"; - String version = "2.1.2"; - CPEAnalyzer instance = new CPEAnalyzer(); - - String queryText = instance.buildSearch(vendor, product, null, null); - String expResult = " product:( struts 2 core ) AND vendor:( apache software foundation ) "; - Assert.assertTrue(expResult.equals(queryText)); - - queryText = instance.buildSearch(vendor, product, null, productWeightings); - expResult = " product:( struts^5 struts2^5 2 core ) AND vendor:( apache software foundation ) "; - Assert.assertTrue(expResult.equals(queryText)); - - queryText = instance.buildSearch(vendor, product, vendorWeightings, null); - expResult = " product:( struts 2 core ) AND vendor:( apache^5 software foundation ) "; - Assert.assertTrue(expResult.equals(queryText)); - - queryText = instance.buildSearch(vendor, product, vendorWeightings, productWeightings); - expResult = " product:( struts^5 struts2^5 2 core ) AND vendor:( apache^5 software foundation ) "; - Assert.assertTrue(expResult.equals(queryText)); - } - - /** - * Test of open method, of class CPEAnalyzer. - * - * @throws Exception is thrown when an exception occurs - */ - @Test - public void testOpen() throws Exception { - CPEAnalyzer instance = new CPEAnalyzer(); - Assert.assertFalse(instance.isOpen()); - instance.open(); - Assert.assertTrue(instance.isOpen()); - instance.close(); - Assert.assertFalse(instance.isOpen()); - } - - /** - * Test of determineCPE method, of class CPEAnalyzer. - * - * @throws Exception is thrown when an exception occurs - */ - @Test - public void testDetermineCPE_full() throws Exception { - callDetermineCPE_full("spring-context-support-2.5.5.jar", "cpe:/a:vmware:springsource_spring_framework:2.5.5"); - callDetermineCPE_full("spring-core-3.0.0.RELEASE.jar", "cpe:/a:vmware:springsource_spring_framework:3.0.0"); - callDetermineCPE_full("org.mortbay.jetty.jar", "cpe:/a:mortbay_jetty:jetty:4.2"); - callDetermineCPE_full("jaxb-xercesImpl-1.5.jar", null); - callDetermineCPE_full("ehcache-core-2.2.0.jar", null); - } - - /** - * Test of determineCPE method, of class CPEAnalyzer. - * - * @throws Exception is thrown when an exception occurs - */ - public void callDetermineCPE_full(String depName, String expResult) throws Exception { - - File file = new File(this.getClass().getClassLoader().getResource(depName).getPath()); - Dependency dep = new Dependency(file); - - FileNameAnalyzer fnAnalyzer = new FileNameAnalyzer(); - fnAnalyzer.analyze(dep, null); - - JarAnalyzer jarAnalyzer = new JarAnalyzer(); - jarAnalyzer.analyze(dep, null); - HintAnalyzer hAnalyzer = new HintAnalyzer(); - hAnalyzer.analyze(dep, null); - - - CPEAnalyzer instance = new CPEAnalyzer(); - instance.open(); - instance.analyze(dep, null); - instance.close(); - FalsePositiveAnalyzer fp = new FalsePositiveAnalyzer(); - fp.analyze(dep, null); - -// for (Identifier i : dep.getIdentifiers()) { -// System.out.println(i.getValue()); -// } - if (expResult != null) { - Identifier expIdentifier = new Identifier("cpe", expResult, expResult); - Assert.assertTrue("Incorrect match: { dep:'" + dep.getFileName() + "' }", dep.getIdentifiers().contains(expIdentifier)); - } else { - Assert.assertTrue("Match found when an Identifier should not have been found: { dep:'" + dep.getFileName() + "' }", dep.getIdentifiers().isEmpty()); - } - } - - /** - * Test of determineCPE method, of class CPEAnalyzer. - * - * @throws Exception is thrown when an exception occurs - */ - @Test - public void testDetermineCPE() throws Exception { - File file = new File(this.getClass().getClassLoader().getResource("struts2-core-2.1.2.jar").getPath()); - //File file = new File(this.getClass().getClassLoader().getResource("axis2-adb-1.4.1.jar").getPath()); - Dependency struts = new Dependency(file); - - FileNameAnalyzer fnAnalyzer = new FileNameAnalyzer(); - fnAnalyzer.analyze(struts, null); - - JarAnalyzer jarAnalyzer = new JarAnalyzer(); - jarAnalyzer.analyze(struts, null); - - - File fileCommonValidator = new File(this.getClass().getClassLoader().getResource("commons-validator-1.4.0.jar").getPath()); - Dependency commonValidator = new Dependency(fileCommonValidator); - jarAnalyzer.analyze(commonValidator, null); - - File fileSpring = new File(this.getClass().getClassLoader().getResource("spring-core-2.5.5.jar").getPath()); - Dependency spring = new Dependency(fileSpring); - jarAnalyzer.analyze(spring, null); - - File fileSpring3 = new File(this.getClass().getClassLoader().getResource("spring-core-3.0.0.RELEASE.jar").getPath()); - Dependency spring3 = new Dependency(fileSpring3); - jarAnalyzer.analyze(spring3, null); - - CPEAnalyzer instance = new CPEAnalyzer(); - instance.open(); - instance.determineCPE(commonValidator); - instance.determineCPE(struts); - instance.determineCPE(spring); - instance.determineCPE(spring3); - instance.close(); - - String expResult = "cpe:/a:apache:struts:2.1.2"; - Identifier expIdentifier = new Identifier("cpe", expResult, expResult); - String expResultSpring = "cpe:/a:springsource:spring_framework:2.5.5"; - String expResultSpring3 = "cpe:/a:vmware:springsource_spring_framework:3.0.0"; - - Assert.assertTrue("Apache Common Validator - found an identifier?", commonValidator.getIdentifiers().isEmpty()); - Assert.assertTrue("Incorrect match size - struts", struts.getIdentifiers().size() >= 1); - Assert.assertTrue("Incorrect match - struts", struts.getIdentifiers().contains(expIdentifier)); - Assert.assertTrue("Incorrect match size - spring3 - " + spring3.getIdentifiers().size(), spring3.getIdentifiers().size() >= 1); - - //the following two only work if the HintAnalyzer is used. - //Assert.assertTrue("Incorrect match size - spring", spring.getIdentifiers().size() == 1); - //Assert.assertTrue("Incorrect match - spring", spring.getIdentifiers().get(0).getValue().equals(expResultSpring)); - - } - - /** - * Test of searchCPE method, of class CPEAnalyzer. - * - * @throws Exception is thrown when an exception occurs - */ - @Test - public void testSearchCPE() throws Exception { - String vendor = "apache software foundation"; - String product = "struts 2 core"; - String version = "2.1.2"; - String expResult = "cpe:/a:apache:struts:2.1.2"; - - CPEAnalyzer instance = new CPEAnalyzer(); - instance.open(); - - //TODO - yeah, not a very good test as the results are the same with or without weighting... - Set productWeightings = new HashSet(1); - productWeightings.add("struts2"); - - Set vendorWeightings = new HashSet(1); - vendorWeightings.add("apache"); - - List result = instance.searchCPE(vendor, product, productWeightings, vendorWeightings); - //TODO fix this assert - //Assert.assertEquals(expResult, result.get(0).getName()); - - - instance.close(); - } -} diff --git a/dependency-check-core/src/test/java/org/owasp/dependencycheck/data/cpe/IndexTest.java b/dependency-check-core/src/test/java/org/owasp/dependencycheck/data/cpe/IndexTest.java deleted file mode 100644 index 0835469bf..000000000 --- a/dependency-check-core/src/test/java/org/owasp/dependencycheck/data/cpe/IndexTest.java +++ /dev/null @@ -1,103 +0,0 @@ -/* - * This file is part of dependency-check-core. - * - * Dependency-check-core is free software: you can redistribute it and/or modify it - * under the terms of the GNU General Public License as published by the Free - * Software Foundation, either version 3 of the License, or (at your option) any - * later version. - * - * Dependency-check-core is distributed in the hope that it will be useful, but - * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more - * details. - * - * You should have received a copy of the GNU General Public License along with - * dependency-check-core. If not, see http://www.gnu.org/licenses/. - * - * Copyright (c) 2012 Jeremy Long. All Rights Reserved. - */ -package org.owasp.dependencycheck.data.cpe; - -import org.owasp.dependencycheck.data.cpe.Index; -import java.io.File; -import java.io.IOException; -import java.util.logging.Level; -import java.util.logging.Logger; -import org.apache.lucene.document.Document; -import org.apache.lucene.queryparser.classic.ParseException; -import org.apache.lucene.search.ScoreDoc; -import org.apache.lucene.search.TopDocs; -import org.apache.lucene.store.Directory; -import org.junit.After; -import org.junit.AfterClass; -import org.junit.Before; -import org.junit.BeforeClass; -import org.junit.Test; -import static org.junit.Assert.*; - -/** - * - * @author Jeremy Long (jeremy.long@owasp.org) - */ -public class IndexTest { - - @BeforeClass - public static void setUpClass() throws Exception { - } - - @AfterClass - public static void tearDownClass() throws Exception { - } - - @Before - public void setUp() { - } - - @After - public void tearDown() { - } - - /** - * Test of open method, of class Index. - */ - @Test - public void testOpen() { - Index instance = new Index(); - try { - instance.open(); - //TODO research why are we getting multiple documents for the same documentId. is the update method not working? -// try { -// instance.createSearchingAnalyzer(); -// TopDocs docs = instance.search("product:( project\\-open )", 20); -// for (ScoreDoc d : docs.scoreDocs) { -// final Document doc = instance.getDocument(d.doc); -// String vendor = doc.getField(Fields.VENDOR).stringValue(); -// String product = doc.getField(Fields.PRODUCT).stringValue(); -// System.out.print(d.doc); -// System.out.print(" : "); -// System.out.print(vendor + ":"); -// System.out.println(product); -// } -// } catch (ParseException ex) { -// Logger.getLogger(IndexTest.class.getName()).log(Level.SEVERE, null, ex); -// } - } catch (IOException ex) { - assertNull(ex.getMessage(), ex); - } - instance.close(); - } - - /** - * Test of getDirectory method, of class Index. - * - * @throws Exception - */ - @Test - public void testGetDirectory() throws Exception { - Index index = new Index(); - Directory result = index.getDirectory(); - - String exp = File.separatorChar + "target" + File.separatorChar + "data" + File.separatorChar + "cpe"; - assertTrue(result.toString().contains(exp)); - } -}