github-mirrors/DependencyCheck
1
0
Fork 0
mirror of https://github.com/ysoftdevs/DependencyCheck.git synced 2026-01-15 08:13:43 +01:00
Code Issues Packages Projects Releases Wiki Activity

major rework of Analyzers and applicatioin in general.

Browse Source 
Former-commit-id: 3b081380f586686762f8a6fcb102778bfc42b17b
This commit is contained in:
Jeremy Long
2012-09-25 11:36:04 -04:00
parent 0643c68da1
commit 8c4d02c909
41 changed files with 1379 additions and 378 deletions
Download Patch File Download Diff File Expand all files Collapse all files

34
src/main/java/org/codesecure/dependencycheck/App.java
View File

@@ -25,7 +25,6 @@ import java.util.logging.Level;
import java.util.logging.Logger;
import javax.xml.parsers.ParserConfigurationException;
import org.apache.commons.cli.ParseException;
import org.codesecure.dependencycheck.data.cpe.CPEQuery;
import org.codesecure.dependencycheck.data.cpe.Index;
import org.codesecure.dependencycheck.data.cpe.xml.Importer;
import org.codesecure.dependencycheck.reporting.ReportGenerator;
@@ -128,32 +127,21 @@ public class App {
* @param files the files/directories to scan.
*/
private void runScan(String reportDirectory, String applicationName, String[] files) {
try {
Engine scanner = new Engine();
for (String file : files) {
scanner.scan(file);
}
scanner.analyzeDependencies();
List<Dependency> dependencies = scanner.getDependencies();
CPEQuery query = new CPEQuery();
query.open();
for (Dependency d : dependencies) {
query.determineCPE(d);
}
query.close();
ReportGenerator report = new ReportGenerator();
try {
report.generateReports(reportDirectory, applicationName, dependencies);
} catch (IOException ex) {
Logger.getLogger(App.class.getName()).log(Level.SEVERE, null, ex);
} catch (Exception ex) {
Logger.getLogger(App.class.getName()).log(Level.SEVERE, null, ex);
}
Engine scanner = new Engine();
for (String file : files) {
scanner.scan(file);
}
scanner.analyzeDependencies();
List<Dependency> dependencies = scanner.getDependencies();
ReportGenerator report = new ReportGenerator();
try {
report.generateReports(reportDirectory, applicationName, dependencies);
} catch (IOException ex) {
Logger.getLogger(App.class.getName()).log(Level.SEVERE, null, ex);
} catch (org.apache.lucene.queryParser.ParseException ex) {
} catch (Exception ex) {
Logger.getLogger(App.class.getName()).log(Level.SEVERE, null, ex);
}
}
}

27
src/main/java/org/codesecure/dependencycheck/Engine.java
View File

@@ -52,11 +52,12 @@ public class Engine {
/**
* A Map of analyzers grouped by Analysis phase.
*/
protected EnumMap<AnalysisPhase, List<Analyzer>> analyzers = new EnumMap<AnalysisPhase, List<Analyzer>>(AnalysisPhase.class);
protected EnumMap<AnalysisPhase, List<Analyzer>> analyzers =
new EnumMap<AnalysisPhase, List<Analyzer>>(AnalysisPhase.class);
/**
* A set of extensions supported by the analyzers.
*/
protected static final Set<String> extensions = new HashSet<String>();
protected Set<String> extensions = new HashSet<String>();
/**
* Creates a new Engine.
@@ -172,7 +173,17 @@ public class Engine {
List<Analyzer> analyzerList = analyzers.get(phase);
for (Analyzer a : analyzerList) {
a.initialize();
try {
a.initialize();
} catch (Exception ex) {
Logger.getLogger(Engine.class.getName()).log(Level.SEVERE, "Exception occured initializing " + a.getName() + ".", ex);
try {
a.close();
} catch (Exception ex1) {
Logger.getLogger(Engine.class.getName()).log(Level.FINER, null, ex1);
}
continue;
}
for (Dependency d : dependencies) {
if (a.supportsExtension(d.getFileExtension())) {
try {
@@ -183,16 +194,20 @@ public class Engine {
a.analyze(d);
}
} catch (IOException ex) {
String msg = String.format("IOException occured while scanning the file '%s'.",
String msg = String.format("IOException occured while analyzing the file '%s'.",
d.getActualFilePath());
Logger.getLogger(Engine.class.getName()).log(Level.SEVERE, msg, ex);
}
}
}
a.close();
try {
a.close();
} catch (Exception ex) {
Logger.getLogger(Engine.class.getName()).log(Level.SEVERE, null, ex);
}
}
}
//Now cycle through all of the analyzers one last time to call
// cleanup on any archiveanalyzers. These should only exist in the
// initial phase, but we are going to be thourough just in case.

61
src/main/java/org/codesecure/dependencycheck/analyzer/AnalysisException.java Normal file
View File

@@ -0,0 +1,61 @@
package org.codesecure.dependencycheck.analyzer;
/*
* This file is part of DependencyCheck.
*
* DependencyCheck is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* DependencyCheck is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with DependencyCheck. If not, see http://www.gnu.org/licenses/.
*
* Copyright (c) 2012 Jeremy Long. All Rights Reserved.
*/
/**
* An exception thrown when the analysis of a dependency fails.
*
* @author Jeremy Long (jeremy.long@gmail.com)
*/
public class AnalysisException extends RuntimeException {
private static final long serialVersionUID = 1L;
/**
* Creates a new AnalysisException.
*/
public AnalysisException() {
super();
}
/**
* Creates a new AnalysisException.
* @param msg a message for the exception.
*/
public AnalysisException(String msg) {
super(msg);
}
/**
* Creates a new AnalysisException.
* @param ex the cause of the failure.
*/
public AnalysisException(Throwable ex) {
super(ex);
}
/**
* Creates a new DownloadFailedException.
* @param msg a message for the exception.
* @param ex the cause of the failure.
*/
public AnalysisException(String msg, Throwable ex) {
super(msg, ex);
}
}

4
src/main/java/org/codesecure/dependencycheck/analyzer/AnalysisPhase.java
View File

@@ -20,11 +20,11 @@ package org.codesecure.dependencycheck.analyzer;
/**
* An enumeration defining the phases of analysis.
*
*
* @author Jeremy Long (jeremy.long@gmail.com)
*/
public enum AnalysisPhase {
/**
* The first phase of analysis.
*/

24
src/main/java/org/codesecure/dependencycheck/analyzer/Analyzer.java
View File

@@ -19,7 +19,6 @@ package org.codesecure.dependencycheck.analyzer;
*/
import org.codesecure.dependencycheck.dependency.Dependency;
import java.io.IOException;
import java.util.Set;
/**
@@ -35,18 +34,18 @@ public interface Analyzer {
* Analyzes the given dependency.
*
* @param dependency a dependency to analyze.
* @throws IOException is thrown if there is an error reading the dependency file
* @throws AnalysisException is thrown if there is an error analyzing the dependency file
*/
void analyze(Dependency dependency) throws IOException;
void analyze(Dependency dependency) throws AnalysisException;
/**
* <p>Returns a list of supported file extensions. An example would be an analyzer
* that inpected java jar files. The getSupportedExtensions function would return
* a set with a single element "jar".</p>
*
* <p><b>Note:</b> when implementing this the extensions returned MUST be lowercase.</p>
* <p><b>Note:</b> when implementing this the extensions returned MUST be lowercase.</p>
* @return The file extensions supported by this analyzer.
*
*
* <p>If the analyzer returns null it will not cause additional files to be analyzed
* but will be executed against every file loaded</p>
*/
@@ -57,27 +56,32 @@ public interface Analyzer {
* @return the name of the analyzer.
*/
String getName();
/**
* Returns whether or not this analyzer can process the given extension.
* @param extension the file extension to test for support.
* @return whether or not the specified file extension is supported by tihs analyzer.
*/
boolean supportsExtension(String extension);
/**
* Returns the phase that the analyzer is intended to run in.
* @return the phase that the analyzer is intended to run in.
*/
AnalysisPhase getAnalysisPhase();
/**
* The initialize method is called (once) prior to the analyze method being called on
* all of the dependencies.
*
* @throws Exception is thrown if an exception occurs initializing the analyzer.
*/
void initialize();
void initialize() throws Exception;
/**
* The close method is called after all of the dependencies have been analyzed.
*
* @throws Exception is thrown if an exception occurs closing the analyzer.
*/
void close();
void close() throws Exception;
}

1
src/main/java/org/codesecure/dependencycheck/analyzer/ArchiveAnalyzer.java
View File

@@ -35,6 +35,7 @@ public interface ArchiveAnalyzer {
* the exploded contents.
*
* @param dependency a dependency to analyze.
* @param engine the engine that is scanning the dependencies.
* @throws IOException is thrown if there is an error reading the dependency file
*/
void analyze(Dependency dependency, Engine engine) throws IOException;

7
src/main/java/org/codesecure/dependencycheck/analyzer/FileNameAnalyzer.java
View File

@@ -20,7 +20,6 @@ package org.codesecure.dependencycheck.analyzer;
import org.codesecure.dependencycheck.dependency.Dependency;
import org.codesecure.dependencycheck.dependency.Evidence;
import java.io.IOException;
import java.util.Set;
import java.util.regex.Pattern;
@@ -35,7 +34,7 @@ public class FileNameAnalyzer implements Analyzer {
/**
* The name of the analyzer.
*/
private static final String ANALYZER_NAME = "File Analyzer";
private static final String ANALYZER_NAME = "File Name Analyzer";
/**
* The phase that this analyzer is intended to run in.
*/
@@ -111,9 +110,9 @@ public class FileNameAnalyzer implements Analyzer {
* Collects information about the file such as hashsums.
*
* @param dependency the dependency to analyze.
* @throws IOException is thrown if there is an error reading the JAR file.
* @throws AnalysisException is thrown if there is an error reading the JAR file.
*/
public void analyze(Dependency dependency) throws IOException {
public void analyze(Dependency dependency) throws AnalysisException {
analyzeFileName(dependency);

17
src/main/java/org/codesecure/dependencycheck/analyzer/JarAnalyzer.java
View File

@@ -149,12 +149,15 @@ public class JarAnalyzer extends AbstractAnalyzer {
* checksums to identify the correct CPE information.
*
* @param dependency the dependency to analyze.
* @throws IOException is thrown if there is an error reading the JAR file.
* @throws AnalysisException is thrown if there is an error reading the JAR file.
*/
public void analyze(Dependency dependency) throws IOException {
parseManifest(dependency);
analyzePackageNames(dependency);
public void analyze(Dependency dependency) throws AnalysisException {
try {
parseManifest(dependency);
analyzePackageNames(dependency);
} catch (IOException ex) {
throw new AnalysisException("Exception occured reading the JAR file.", ex);
}
}
@@ -362,7 +365,9 @@ public class JarAnalyzer extends AbstractAnalyzer {
} else {
key = key.toLowerCase();
if (!IGNORE_LIST.contains(key) && !key.contains("license") && !key.endsWith("jdk")) {
if (!IGNORE_LIST.contains(key) && !key.contains("license") && !key.endsWith("jdk")
&& !key.contains("lastmodified")) {
if (key.contains("version")) {
versionEvidence.addEvidence(source, key, value, Evidence.Confidence.MEDIUM);
} else if (key.contains("title")) {

5
src/main/java/org/codesecure/dependencycheck/data/CachedWebDataSource.java
View File

@@ -18,7 +18,6 @@ package org.codesecure.dependencycheck.data;
* Copyright (c) 2012 Jeremy Long. All Rights Reserved.
*/
import java.io.IOException;
import java.net.MalformedURLException;
import javax.xml.parsers.ParserConfigurationException;
@@ -27,7 +26,7 @@ import org.xml.sax.SAXException;
/**
* Defines an Index who's data is retrieved from the Internet. This data can
* be downloaded and the index updated.
*
*
* @author Jeremy Long (jeremy.long@gmail.com)
*/
public interface CachedWebDataSource {
@@ -41,5 +40,5 @@ public interface CachedWebDataSource {
* @throws SAXException is thrown if there is an error parsing the CPE XML.
* @throws IOException is thrown if a temporary file could not be created.
*/
public void update() throws MalformedURLException, ParserConfigurationException, SAXException, IOException;
void update() throws MalformedURLException, ParserConfigurationException, SAXException, IOException;
}

137
src/main/java/org/codesecure/dependencycheck/data/cpe/CPEQuery.java → src/main/java/org/codesecure/dependencycheck/data/cpe/CPEAnalyzer.java
View File

@@ -19,38 +19,37 @@ package org.codesecure.dependencycheck.data.cpe;
*/
import java.io.IOException;
import java.net.URLEncoder;
import java.util.ArrayList;
import java.util.List;
import java.util.Set;
import java.util.StringTokenizer;
import java.util.logging.Level;
import java.util.logging.Logger;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.index.CorruptIndexException;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.queryParser.ParseException;
import org.apache.lucene.queryParser.QueryParser;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.ScoreDoc;
import org.apache.lucene.search.TopDocs;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.Version;
import org.codesecure.dependencycheck.data.LuceneUtils;
import org.codesecure.dependencycheck.analyzer.AnalysisException;
import org.codesecure.dependencycheck.analyzer.AnalysisPhase;
import org.codesecure.dependencycheck.data.lucene.LuceneUtils;
import org.codesecure.dependencycheck.dependency.Dependency;
import org.codesecure.dependencycheck.dependency.Evidence;
import org.codesecure.dependencycheck.dependency.Evidence.Confidence;
import org.codesecure.dependencycheck.dependency.EvidenceCollection;
/**
* CPEQuery is a utility class that takes a project dependency and attempts
* CPEAnalyzer is a utility class that takes a project dependency and attempts
* to decern if there is an associated CPE. It uses the evidence contained
* within the dependency to search the Lucene index.
*
* @author Jeremy Long (jeremy.long@gmail.com)
*/
public class CPEQuery {
public class CPEAnalyzer implements org.codesecure.dependencycheck.analyzer.Analyzer {
/**
* The maximum number of query results to return.
@@ -140,7 +139,7 @@ public class CPEQuery {
* @throws IOException is thrown when an IOException occurs.
* @throws ParseException is thrown when the Lucene query cannot be parsed.
*/
public void determineCPE(Dependency dependency) throws CorruptIndexException, IOException, ParseException {
protected void determineCPE(Dependency dependency) throws CorruptIndexException, IOException, ParseException {
Confidence vendorConf = Confidence.HIGH;
Confidence productConf = Confidence.HIGH;
Confidence versionConf = Confidence.HIGH;
@@ -167,16 +166,21 @@ public class CPEQuery {
List<Entry> entries = searchCPE(vendors, products, versions, dependency.getProductEvidence().getWeighting(),
dependency.getVendorEvidence().getWeighting());
if (entries.size() > 0) {
//TODO - after changing the lucene query to use the AND conditions we should no longer need this.
List<String> verified = verifyEntries(entries, dependency);
if (verified.size() > 0) {
for (Entry e : entries) {
if (verifyEntry(e, dependency)) {
found = true;
dependency.setCPEs(verified);
dependency.addIdentifier(
"cpe",
e.getName(),
e.getTitle(),
"http://web.nvd.nist.gov/view/vuln/search?cpe="
+ URLEncoder.encode(e.getName(), "UTF-8"));
}
}
if (!found) {
int round = cnt % 3;
if (round == 0) {
@@ -206,9 +210,7 @@ public class CPEQuery {
versions = addEvidenceWithoutDuplicateTerms(versions, dependency.getVersionEvidence(), versionConf);
}
}
}
} while (!found && (++cnt) < 9);
}
@@ -226,11 +228,18 @@ public class CPEQuery {
private String addEvidenceWithoutDuplicateTerms(final String text, final EvidenceCollection ec, Confidence confidenceFilter) {
String txt = (text == null) ? "" : text;
StringBuilder sb = new StringBuilder(txt.length() + (20 * ec.size()));
sb.append(txt);
for (Evidence e : ec.iterator(confidenceFilter)) {
String value = e.getValue();
if (value.startsWith("http://")) {
value = value.substring(7).replaceAll("\\.", " ");
}
if (value.startsWith("https://")) {
value = value.substring(8).replaceAll("\\.", " ");
}
if (sb.indexOf(value) < 0) {
if (value.length() > 200) {
sb.append(value.substring(0, 200));
sb.append(value.substring(0, 200)).append(' ');
} else {
sb.append(value).append(' ');
}
@@ -325,8 +334,6 @@ public class CPEQuery {
* to boost the terms weight.
* @return the Lucene query.
*/
//TODO change this whole search mechanism into building the query
// using terms and the org.apache.lucene.search.Query API.
protected String buildSearch(String vendor, String product, String version,
Set<String> vendorWeighting, Set<String> produdctWeightings) {
@@ -346,7 +353,7 @@ public class CPEQuery {
return null;
}
sb.append(" AND ");
sb.append(Fields.VERSION).append(":(");
if (sb.indexOf("^") > 0) {
//if we have a weighting on something else, reduce the weighting on the version a lot
@@ -445,27 +452,83 @@ public class CPEQuery {
}
/**
* Takes a list of entries and a dependency. If the entry has terms that were
* used (i.e. this CPE entry wasn't identified because the version matched
* but the product names did not) then the CPE Entry is returned in a list
* of possible CPE Entries.
* Ensures that the CPE Identified matches the dependency. This validates that
* the product, vendor, and version information for the CPE are contained within
* the dependencies evidence.
*
* @param entries a list of CPE entries.
* @param entry a CPE entry.
* @param dependency the dependency that the CPE entries could be for.
* @return a list of matched CPE entries.
* @return whether or not the entry is valid.
*/
private List<String> verifyEntries(final List<Entry> entries, final Dependency dependency) {
List<String> verified = new ArrayList<String>();
for (Entry e : entries) {
if (dependency.getProductEvidence().containsUsedString(e.getProduct())
&& dependency.getVendorEvidence().containsUsedString(e.getVendor())) {
//TODO - determine if this is right? Should we be carrying too much about the
// version at this point? Likely need to implement the versionAnalyzer....
if (dependency.getVersionEvidence().containsUsedString(e.getVersion())) {
verified.add(e.getName());
}
private boolean verifyEntry(final Entry entry, final Dependency dependency) {
boolean isValid = false;
if (dependency.getProductEvidence().containsUsedString(entry.getProduct())
&& dependency.getVendorEvidence().containsUsedString(entry.getVendor())) {
//TODO - determine if this is right? Should we be carrying too much about the
// version at this point? Likely need to implement the versionAnalyzer....
if (dependency.getVersionEvidence().containsUsedString(entry.getVersion())) {
isValid = true;
}
}
return verified;
return isValid;
}
/**
* Analyzes a dependency and attempts to determine if there are any CPE identifiers
* for this dependency.
* @param dependency The Dependency to analyze.
* @throws AnalysisException is thrown if there is an issue analyzing the dependency.
*/
public void analyze(Dependency dependency) throws AnalysisException {
try {
determineCPE(dependency);
} catch (CorruptIndexException ex) {
throw new AnalysisException("CPE Index is corrupt.", ex);
} catch (IOException ex) {
throw new AnalysisException("Failure opening the CPE Index.", ex);
} catch (ParseException ex) {
throw new AnalysisException("Unable to parse the generated Lucene query for this dependency.", ex);
}
}
/**
* Returns true because this analyzer supports all dependency types.
* @return true.
*/
public Set<String> getSupportedExtensions() {
return null;
}
/**
* Returns the name of this analyzer.
* @return the name of this analyzer.
*/
public String getName() {
return "CPE Analyzer";
}
/**
* Returns true because this analyzer supports all dependency types.
* @param extension the file extension of the dependency being analyzed.
* @return true.
*/
public boolean supportsExtension(String extension) {
return true;
}
/**
* Returns the analysis phase that this analyzer should run in.
* @return the analysis phase that this analyzer should run in.
*/
public AnalysisPhase getAnalysisPhase() {
return AnalysisPhase.IDENTIFIER_ANALYSIS;
}
/**
* Opens the CPE Lucene Index.
* @throws Exception is thrown if there is an issue opening the index.
*/
public void initialize() throws Exception {
this.open();
}
}

6
src/main/java/org/codesecure/dependencycheck/data/cpe/Index.java
View File

@@ -38,13 +38,10 @@ import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.KeywordAnalyzer;
import org.apache.lucene.analysis.PerFieldAnalyzerWrapper;
import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.index.CorruptIndexException;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory;
import org.apache.lucene.util.Version;
import org.codesecure.dependencycheck.data.AbstractIndex;
import org.codesecure.dependencycheck.data.lucene.AbstractIndex;
import org.codesecure.dependencycheck.data.CachedWebDataSource;
import org.codesecure.dependencycheck.utils.Downloader;
import org.codesecure.dependencycheck.utils.Settings;
@@ -68,7 +65,6 @@ public class Index extends AbstractIndex implements CachedWebDataSource {
*/
private static final String LAST_UPDATED = "lastupdated";
/**
* Returns the directory that holds the CPE Index.
*

10
src/main/java/org/codesecure/dependencycheck/data/cpe/xml/Indexer.java
View File

@@ -24,15 +24,10 @@ import org.apache.lucene.document.Field;
import org.apache.lucene.index.CorruptIndexException;
import org.apache.lucene.index.FieldInfo.IndexOptions;
import org.apache.lucene.index.Term;
import org.codesecure.dependencycheck.data.LuceneUtils;
import org.codesecure.dependencycheck.data.cpe.Entry;
import org.codesecure.dependencycheck.data.lucene.LuceneUtils;
import org.codesecure.dependencycheck.data.cpe.Entry;
import org.codesecure.dependencycheck.data.cpe.Fields;
import org.codesecure.dependencycheck.data.cpe.Fields;
import org.codesecure.dependencycheck.data.cpe.Index;
import org.codesecure.dependencycheck.data.cpe.Index;
import org.codesecure.dependencycheck.data.cpe.xml.EntrySaveDelegate;
import org.codesecure.dependencycheck.data.cpe.xml.EntrySaveDelegate;
/**
* The Indexer is used to convert a CPE Entry, retrieved from the CPE XML file,
@@ -82,7 +77,7 @@ public class Indexer extends Index implements EntrySaveDelegate {
product.setBoost(5.0F);
doc.add(product);
Field title = new Field(Fields.TITLE, entry.getTitle(), Field.Store.NO, Field.Index.ANALYZED);
Field title = new Field(Fields.TITLE, entry.getTitle(), Field.Store.YES, Field.Index.ANALYZED);
title.setIndexOptions(IndexOptions.DOCS_ONLY);
//title.setBoost(1.0F);
doc.add(title);
@@ -104,5 +99,4 @@ public class Indexer extends Index implements EntrySaveDelegate {
return doc;
}
}

5
src/main/java/org/codesecure/dependencycheck/data/cve/Index.java
View File

@@ -41,7 +41,7 @@ import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory;
import org.apache.lucene.util.Version;
import org.codesecure.dependencycheck.data.AbstractIndex;
import org.codesecure.dependencycheck.data.lucene.AbstractIndex;
import org.codesecure.dependencycheck.data.CachedWebDataSource;
import org.codesecure.dependencycheck.utils.Downloader;
import org.codesecure.dependencycheck.utils.Settings;
@@ -65,7 +65,6 @@ public class Index extends AbstractIndex implements CachedWebDataSource {
*/
private static final String LAST_UPDATED = "lastupdated";
/**
* Returns the directory that holds the CPE Index.
*
@@ -76,7 +75,7 @@ public class Index extends AbstractIndex implements CachedWebDataSource {
String fileName = Settings.getString(Settings.KEYS.CVE_INDEX);
File path = new File(fileName);
Directory dir = FSDirectory.open(path);
return dir;
}

3
src/main/java/org/codesecure/dependencycheck/data/cve/xml/Indexer.java
View File

@@ -24,11 +24,10 @@ import org.apache.lucene.document.Field;
import org.apache.lucene.index.CorruptIndexException;
import org.apache.lucene.index.FieldInfo.IndexOptions;
import org.apache.lucene.index.Term;
import org.codesecure.dependencycheck.data.LuceneUtils;
import org.codesecure.dependencycheck.data.lucene.LuceneUtils;
import org.codesecure.dependencycheck.data.cve.Entry;
import org.codesecure.dependencycheck.data.cve.Fields;
import org.codesecure.dependencycheck.data.cve.Index;
import org.codesecure.dependencycheck.data.cve.xml.EntrySaveDelegate;
/**
* The Indexer is used to convert a CPE Entry, retrieved from the CPE XML file,

46
src/main/java/org/codesecure/dependencycheck/data/AbstractIndex.java → src/main/java/org/codesecure/dependencycheck/data/lucene/AbstractIndex.java
View File

@@ -1,4 +1,4 @@
package org.codesecure.dependencycheck.data;
package org.codesecure.dependencycheck.data.lucene;
/*
* This file is part of DependencyCheck.
*
@@ -33,11 +33,11 @@ import org.apache.lucene.util.Version;
/**
* The base Index for other index objects. Implements the open and close methods.
*
*
* @author Jeremy Long (jeremy.long@gmail.com)
*/
public abstract class AbstractIndex {
/**
* The Lucene directory containing the index.
*/
@@ -46,7 +46,7 @@ public abstract class AbstractIndex {
* The IndexWriter for the Lucene index.
*/
protected IndexWriter indexWriter = null;
/**
/**
* The Lucene IndexReader.
*/
private IndexReader indexReader = null;
@@ -54,12 +54,10 @@ public abstract class AbstractIndex {
* The Lucene IndexSearcher.
*/
private IndexSearcher indexSearcher = null;
/**
* The Lucene Analyzer.
*/
private Analyzer analyzer = null;
/**
* Indicates whether or not the Lucene Index is open.
*/
@@ -106,7 +104,7 @@ public abstract class AbstractIndex {
indexSearcher = null;
}
}
if (analyzer != null) {
analyzer.close();
analyzer = null;
@@ -120,7 +118,7 @@ public abstract class AbstractIndex {
}
indexOpen = false;
}
/**
* Returns the status of the data source - is the index open.
* @return true or false.
@@ -128,9 +126,10 @@ public abstract class AbstractIndex {
public boolean isOpen() {
return indexOpen;
}
/**
* Opens the Lucene Index Writer.
*
*
* @throws CorruptIndexException is thrown if the Lucene index is corrupt.
* @throws IOException is thrown if an IOException occurs opening the index.
*/
@@ -141,10 +140,10 @@ public abstract class AbstractIndex {
IndexWriterConfig conf = new IndexWriterConfig(Version.LUCENE_35, analyzer);
indexWriter = new IndexWriter(directory, conf);
}
/**
* Retrieves the IndexWriter for the Lucene Index.
*
*
* @return an IndexWriter.
* @throws CorruptIndexException is thrown if the Lucene Index is corrupt.
* @throws LockObtainFailedException is thrown if there is an exception obtaining a lock on the Lucene index.
@@ -156,14 +155,25 @@ public abstract class AbstractIndex {
}
return indexWriter;
}
/**
* Opens the Lucene Index for reading.
* @throws CorruptIndexException is thrown if the index is corrupt.
* @throws IOException is thrown if there is an exception reading the index.
*/
public void openIndexReader() throws CorruptIndexException, IOException {
if (!isOpen()) {
open();
}
indexReader = IndexReader.open(directory, true);
}
/**
* Returns an IndexSearcher for the Lucene Index.
* @return an IndexSearcher.
* @throws CorruptIndexException is thrown if the index is corrupt.
* @throws IOException is thrown if there is an exception reading the index.
*/
public IndexSearcher getIndexSearcher() throws CorruptIndexException, IOException {
if (indexReader == null) {
openIndexReader();
@@ -173,24 +183,28 @@ public abstract class AbstractIndex {
}
return indexSearcher;
}
/**
* Returns an Analyzer for the Lucene Index.
* @return an Analyzer.
*/
public Analyzer getAnalyzer() {
if (analyzer == null) {
analyzer = createAnalyzer();
}
return analyzer;
}
/**
* Gets the directory that contains the Lucene Index.
* @return a Lucene Directory.
* @throws IOException is thrown when an IOException occurs.
*/
public abstract Directory getDirectory() throws IOException;
/**
* Creates the Lucene Analyzer used when indexing and searching the index.
* @return a Lucene Analyzer.
*/
public abstract Analyzer createAnalyzer();
}

6
src/main/java/org/codesecure/dependencycheck/data/DependencySimilarity.java → src/main/java/org/codesecure/dependencycheck/data/lucene/DependencySimilarity.java
View File

@@ -1,4 +1,4 @@
package org.codesecure.dependencycheck.data;
package org.codesecure.dependencycheck.data.lucene;
/*
* This file is part of DependencyCheck.
*
@@ -31,9 +31,9 @@ public class DependencySimilarity extends DefaultSimilarity {
/**
* <p>Override the default idf implementation so that frequency within
* all document is ignored.</p>
*
*
* See <a href="http://www.lucenetutorial.com/advanced-topics/scoring.html">this article</a> for more details.
*
*
* @param docFreq - the number of documents which contain the term
* @param numDocs - the total number of documents in the collection
* @return 1

2
src/main/java/org/codesecure/dependencycheck/data/LuceneUtils.java → src/main/java/org/codesecure/dependencycheck/data/lucene/LuceneUtils.java
View File

@@ -1,4 +1,4 @@
package org.codesecure.dependencycheck.data;
package org.codesecure.dependencycheck.data.lucene;
/*
* This file is part of DependencyCheck.
*

6
src/main/java/org/codesecure/dependencycheck/data/VersionAnalyzer.java → src/main/java/org/codesecure/dependencycheck/data/lucene/VersionAnalyzer.java
View File

@@ -1,4 +1,4 @@
package org.codesecure.dependencycheck.data;
package org.codesecure.dependencycheck.data.lucene;
/*
* This file is part of DependencyCheck.
*
@@ -25,11 +25,9 @@ package org.codesecure.dependencycheck.data;
*/
public class VersionAnalyzer {
//TODO Implement this...
// use custom attributes for major, minor, x, x, x, rcx
// these can then be used to weight the score for searches on the version.
// see http://lucene.apache.org/core/3_6_1/api/core/org/apache/lucene/analysis/package-summary.html#package_description
// look at this article to implement
// look at this article to implement
// http://www.codewrecks.com/blog/index.php/2012/08/25/index-your-blog-using-tags-and-lucene-net/
}

12
src/main/java/org/codesecure/dependencycheck/data/lucene/package-info.java Normal file
View File

@@ -0,0 +1,12 @@
/**
* <html>
* <head>
* <title>org.codesecure.dependencycheck.data.lucene</title>
* </head>
* <body>
* Contains classes used to work with the Lucene Indexes.
* </body>
* </html>
*/
package org.codesecure.dependencycheck.data.lucene;

2
src/main/java/org/codesecure/dependencycheck/data/package-info.java
View File

@@ -4,7 +4,7 @@
* <title>org.codesecure.dependencycheck.data</title>
* </head>
* <body>
* Contains utility classes used to work with the Lucene Indexes.
* Contains classes used to work with the data sources.
* </body>
* </html>
*/

14
src/main/java/org/codesecure/dependencycheck/dependency/Dependency.java
View File

@@ -90,6 +90,10 @@ public class Dependency {
identifiers = new ArrayList<Identifier>();
}
/**
* Constructs a new Dependency object.
* @param file the File to create the dependency object from.
*/
public Dependency(File file) {
this();
this.actualFilePath = file.getPath();
@@ -232,10 +236,9 @@ public class Dependency {
* @param value the value of the identifier.
* @param title the title of the identifier.
* @param url the URL of the identifier.
* @param description the description of the identifier.
*/
public void addIdentifier(String type, String value, String title, String url, String description) {
Identifier i = new Identifier(type, value, title, url, description);
public void addIdentifier(String type, String value, String title, String url) {
Identifier i = new Identifier(type, value, title, url);
this.identifiers.add(i);
}
@@ -245,7 +248,7 @@ public class Dependency {
* @return an EvidenceCollection.
*/
public EvidenceCollection getEvidence() {
return EvidenceCollection.mergeUsed(this.productEvidence, this.vendorEvidence, this.versionEvidence);
return EvidenceCollection.merge(this.productEvidence, this.vendorEvidence, this.versionEvidence);
}
/**
@@ -254,8 +257,7 @@ public class Dependency {
* @return an EvidenceCollection.
*/
public EvidenceCollection getEvidenceUsed() {
EvidenceCollection ec = EvidenceCollection.mergeUsed(this.productEvidence, this.vendorEvidence, this.versionEvidence);
return ec;
return EvidenceCollection.mergeUsed(this.productEvidence, this.vendorEvidence, this.versionEvidence);
}
/**

44
src/main/java/org/codesecure/dependencycheck/dependency/EvidenceCollection.java
View File

@@ -146,6 +146,14 @@ public class EvidenceCollection implements Iterable<Evidence> {
return weightedStrings;
}
/**
* Returns the set of evidence.
* @return the set of evidence.
*/
public Set<Evidence> getEvidence() {
return list;
}
/**
* Implements the iterator interface for the Evidence Collection.
* @return an Iterator<Evidence>.
@@ -206,27 +214,21 @@ public class EvidenceCollection implements Iterable<Evidence> {
}
return ret;
}
// Removed because this wasn't working right (the function returned the right data, but
// the use of the results was flawed.
// /**
// * Returns a string of evidence 'values' for a given confidence.
// * @param confidence the confidence filter applied to the toString method.
// * @return a string containing the evidence.
// */
// public String toString(Evidence.Confidence confidence) {
// StringBuilder sb = new StringBuilder();
// for (Evidence e : this.iterator(confidence)) {
// String str = e.getValue();
// //TODO this is a cheap hack, need to prevent the same string from hitting multiple times...
// // consider changing the evidencecollection.add to prevent the same "value" for a lower
// // confidence from being added? Might not work due to minor differences in the items in the manifest.
// // might need to actually use a StringTokenizer here and only add single words no in the list.
// if (sb.indexOf(str)<0) {
// sb.append(str).append(' ');
// }
// }
// return sb.toString();
// }
/**
* Merges multiple EvidenceCollections together.
*
* @param ec One or more EvidenceCollections.
* @return a new EvidenceCollection.
*/
public static EvidenceCollection merge(EvidenceCollection... ec) {
EvidenceCollection ret = new EvidenceCollection();
for (EvidenceCollection col : ec) {
ret.list.addAll(col.list);
ret.weightedStrings.addAll(col.weightedStrings);
}
return ret;
}
/**
* Returns a string of evidence 'values'.

40
src/main/java/org/codesecure/dependencycheck/dependency/Identifier.java
View File

@@ -24,15 +24,23 @@ package org.codesecure.dependencycheck.dependency;
*/
public class Identifier {
protected String value;
Identifier(String type, String value, String title, String url, String description) {
/**
* Constructs a new Identifier with the specified data.
* @param type the identifier type.
* @param value the identifier value.
* @param title the identifier title.
* @param url the identifier url.
*/
Identifier(String type, String value, String title, String url) {
this.type = type;
this.value = value;
this.title = title;
this.url = url;
this.description = description;
}
/**
* The value of the identifeir
*/
protected String value;
/**
* Get the value of value
@@ -51,6 +59,9 @@ public class Identifier {
public void setValue(String value) {
this.value = value;
}
/**
* The title of the identifeir
*/
protected String title;
/**
@@ -70,25 +81,9 @@ public class Identifier {
public void setTitle(String title) {
this.title = title;
}
protected String description;
/**
* Get the value of description
*
* @return the value of description
* The url for the identifeir
*/
public String getDescription() {
return description;
}
/**
* Set the value of description
*
* @param description new value of description
*/
public void setDescription(String description) {
this.description = description;
}
protected String url;
/**
@@ -108,6 +103,9 @@ public class Identifier {
public void setUrl(String url) {
this.url = url;
}
/**
* The type of the identifeir
*/
protected String type;
/**

4
src/main/java/org/codesecure/dependencycheck/utils/DownloadFailedException.java
View File

@@ -46,7 +46,7 @@ public class DownloadFailedException extends IOException {
/**
* Creates a new DownloadFailedException.
* @param ex the cause of te download failure.
* @param ex the cause of the download failure.
*/
public DownloadFailedException(Throwable ex) {
super(ex);
@@ -55,7 +55,7 @@ public class DownloadFailedException extends IOException {
/**
* Creates a new DownloadFailedException.
* @param msg a message for the exception.
* @param ex the cause of te download failure.
* @param ex the cause of the download failure.
*/
public DownloadFailedException(String msg, Throwable ex) {
super(msg, ex);

6
src/main/java/org/codesecure/dependencycheck/utils/FileUtils.java
View File

@@ -25,6 +25,12 @@ package org.codesecure.dependencycheck.utils;
*/
public class FileUtils {
/**
* Private constructor for a utility class.
*/
private FileUtils() {
}
/**
* Returns the (lowercase) file extension for a specified file.
* @param fileName the file name to retrieve the file extension from.