github-mirrors/DependencyCheck
1
0
Fork 0
mirror of https://github.com/ysoftdevs/DependencyCheck.git synced 2026-05-10 17:09:58 +02:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'upmaster' into openssl-source-analyzer

Browse Source 
Conflicts:
	dependency-check-core/src/test/java/org/owasp/dependencycheck/analyzer/PythonPackageAnalyzerTest.java

Former-commit-id: 6d92982227ad0ff7c4381d03eb1bf542dfe7697f
This commit is contained in:
Dale Visser
2015-07-09 16:14:54 -04:00
parent a3ad598004 b227cf890b
commit e5fdda60fc
90 changed files with 2301 additions and 13037 deletions
Download Patch File Download Diff File Expand all files Collapse all files

21
dependency-check-core/src/main/java/org/owasp/dependencycheck/Engine.java
View File

@@ -37,6 +37,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.FileFilter;
import java.util.ArrayList;
import java.util.EnumMap;
import java.util.HashSet;
@@ -50,7 +51,7 @@ import java.util.Set;
*
* @author Jeremy Long
*/
public class Engine {
public class Engine implements FileFilter {
/**
* The list of dependencies.
@@ -317,7 +318,7 @@ public class Engine {
extension = fileName;
}
Dependency dependency = null;
if (supportsExtension(extension)) {
if (accept(file)) {
dependency = new Dependency(file);
if (extension.equals(fileName)) {
dependency.setFileExtension(extension);
@@ -330,8 +331,8 @@ public class Engine {
/**
* Runs the analyzers against all of the dependencies. Since the mutable dependencies list is exposed via
* {@link #getDependencies()}, this method iterates over a copy of the dependencies list. Thus, the potential for
* {@link java.util.ConcurrentModificationException}s is avoided, and analyzers may safely add or remove entries
* from the dependencies list.
* {@link java.util.ConcurrentModificationException}s is avoided, and analyzers may safely add or remove entries from the
* dependencies list.
*/
public void analyzeDependencies() {
boolean autoUpdate = true;
@@ -379,7 +380,7 @@ public class Engine {
boolean shouldAnalyze = true;
if (a instanceof FileTypeAnalyzer) {
final FileTypeAnalyzer fAnalyzer = (FileTypeAnalyzer) a;
shouldAnalyze = fAnalyzer.supportsExtension(d.getFileExtension());
shouldAnalyze = fAnalyzer.accept(d.getActualFile());
}
if (shouldAnalyze) {
LOGGER.debug("Begin Analysis of '{}'", d.getActualFilePath());
@@ -482,18 +483,18 @@ public class Engine {
/**
* Checks all analyzers to see if an extension is supported.
*
* @param ext a file extension
* @param file a file extension
* @return true or false depending on whether or not the file extension is supported
*/
public boolean supportsExtension(String ext) {
if (ext == null) {
public boolean accept(File file) {
if (file == null) {
return false;
}
boolean scan = false;
for (FileTypeAnalyzer a : this.fileTypeAnalyzers) {
/* note, we can't break early on this loop as the analyzers need to know if
they have files to work on prior to initialization */
scan |= a.supportsExtension(ext);
scan |= a.accept(file);
}
return scan;
}
@@ -510,7 +511,7 @@ public class Engine {
/**
* Checks the CPE Index to ensure documents exists. If none exist a NoDataException is thrown.
*
* @throws NoDataException thrown if no data exists in the CPE Index
* @throws NoDataException thrown if no data exists in the CPE Index
* @throws DatabaseException thrown if there is an exception opening the database
*/
private void ensureDataExists() throws NoDataException, DatabaseException {

7
dependency-check-core/src/main/java/org/owasp/dependencycheck/agent/DependencyCheckScanAgent.java
View File

@@ -872,7 +872,7 @@ public class DependencyCheckScanAgent {
r.generateReports(outDirectory.getCanonicalPath(), this.reportFormat.name());
} catch (IOException ex) {
LOGGER.error(
"Unexpected exception occurred during analysis; please see the verbose error log for more details.");
"Unexpected exception occurred during analysis; please see the verbose error log for more details.");
LOGGER.debug("", ex);
} catch (Throwable ex) {
LOGGER.error(
@@ -1058,8 +1058,9 @@ public class DependencyCheckScanAgent {
}
}
if (summary.length() > 0) {
LOGGER.warn("\n\nOne or more dependencies were identified with known vulnerabilities:\n\n{}\n\nSee the dependency-check report for more details.\n\n",
summary.toString());
LOGGER.warn("\n\nOne or more dependencies were identified with known vulnerabilities:\n\n{}\n\n"
+ "See the dependency-check report for more details.\n\n",
summary.toString());
}
}

77
dependency-check-core/src/main/java/org/owasp/dependencycheck/analyzer/AbstractFileTypeAnalyzer.java
View File

@@ -17,9 +17,6 @@
*/
package org.owasp.dependencycheck.analyzer;
import java.util.Collections;
import java.util.HashSet;
import java.util.Set;
import org.owasp.dependencycheck.Engine;
import org.owasp.dependencycheck.analyzer.exception.AnalysisException;
import org.owasp.dependencycheck.dependency.Dependency;
@@ -28,6 +25,12 @@ import org.owasp.dependencycheck.utils.Settings;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.FileFilter;
import java.util.Collections;
import java.util.HashSet;
import java.util.Set;
/**
* The base FileTypeAnalyzer that all analyzers that have specific file types they analyze should extend.
*
@@ -37,8 +40,7 @@ public abstract class AbstractFileTypeAnalyzer extends AbstractAnalyzer implemen
//<editor-fold defaultstate="collapsed" desc="Constructor">
/**
* Base constructor that all children must call. This checks the configuration to determine if the analyzer is
* enabled.
* Base constructor that all children must call. This checks the configuration to determine if the analyzer is enabled.
*/
public AbstractFileTypeAnalyzer() {
reset();
@@ -100,19 +102,16 @@ public abstract class AbstractFileTypeAnalyzer extends AbstractAnalyzer implemen
//<editor-fold defaultstate="collapsed" desc="Abstract methods children must implement">
/**
* <p>
* Returns a list of supported file extensions. An example would be an analyzer that inspected java jar files. The
* getSupportedExtensions function would return a set with a single element "jar".</p>
* Returns the {@link java.io.FileFilter} used to determine which files are to be analyzed. An example would be an analyzer
* that inspected Java jar files. Implementors may use {@link org.owasp.dependencycheck.utils.FileFilterBuilder}.</p>
*
* @return the file filter used to determine which files are to be analyzed
* <p/>
* <p>
* <b>Note:</b> when implementing this the extensions returned MUST be lowercase.</p>
*
* @return The file extensions supported by this analyzer.
*
* <p>
* If the analyzer returns null it will not cause additional files to be analyzed but will be executed against every
* file loaded</p>
* If the analyzer returns null it will not cause additional files to be analyzed, but will be executed against every file
* loaded.</p>
*/
protected abstract Set<String> getSupportedExtensions();
protected abstract FileFilter getFileFilter();
/**
* Initializes the file type analyzer.
@@ -122,8 +121,8 @@ public abstract class AbstractFileTypeAnalyzer extends AbstractAnalyzer implemen
protected abstract void initializeFileTypeAnalyzer() throws Exception;
/**
* Analyzes a given dependency. If the dependency is an archive, such as a WAR or EAR, the contents are extracted,
* scanned, and added to the list of dependencies within the engine.
* Analyzes a given dependency. If the dependency is an archive, such as a WAR or EAR, the contents are extracted, scanned,
* and added to the list of dependencies within the engine.
*
* @param dependency the dependency to analyze
* @param engine the engine scanning
@@ -171,8 +170,8 @@ public abstract class AbstractFileTypeAnalyzer extends AbstractAnalyzer implemen
}
/**
* Analyzes a given dependency. If the dependency is an archive, such as a WAR or EAR, the contents are extracted,
* scanned, and added to the list of dependencies within the engine.
* Analyzes a given dependency. If the dependency is an archive, such as a WAR or EAR, the contents are extracted, scanned,
* and added to the list of dependencies within the engine.
*
* @param dependency the dependency to analyze
* @param engine the engine scanning
@@ -185,38 +184,28 @@ public abstract class AbstractFileTypeAnalyzer extends AbstractAnalyzer implemen
}
}
/**
* Returns whether or not this analyzer can process the given extension.
*
* @param extension the file extension to test for support.
* @return whether or not the specified file extension is supported by this analyzer.
*/
@Override
public final boolean supportsExtension(String extension) {
if (!enabled) {
return false;
}
final Set<String> ext = getSupportedExtensions();
if (ext == null) {
LOGGER.error("The '{}' analyzer is misconfigured and does not have any file extensions;"
+ " it will be disabled", getName());
return false;
} else {
final boolean match = ext.contains(extension);
if (match) {
filesMatched = match;
public boolean accept(File pathname) {
final FileFilter filter = getFileFilter();
boolean accepted = false;
if (null == filter) {
LOGGER.error("The '{}' analyzer is misconfigured and does not have a file filter; it will be disabled", getName());
} else if (enabled) {
accepted = filter.accept(pathname);
if (accepted) {
filesMatched = true;
}
return match;
}
return accepted;
}
//</editor-fold>
//</editor-fold>
//<editor-fold defaultstate="collapsed" desc="Static utility methods">
/**
* <p>
* Utility method to help in the creation of the extensions set. This constructs a new Set that can be used in a
* final static declaration.</p>
*
* Utility method to help in the creation of the extensions set. This constructs a new Set that can be used in a final static
* declaration.</p>
* <p/>
* <p>
* This implementation was copied from
* http://stackoverflow.com/questions/2041778/initialize-java-hashset-values-by-construction</p>
@@ -226,9 +215,9 @@ public abstract class AbstractFileTypeAnalyzer extends AbstractAnalyzer implemen
*/
protected static Set<String> newHashSet(String... strings) {
final Set<String> set = new HashSet<String>();
Collections.addAll(set, strings);
return set;
}
//</editor-fold>
}

39
dependency-check-core/src/main/java/org/owasp/dependencycheck/analyzer/ArchiveAnalyzer.java
View File

@@ -20,6 +20,7 @@ package org.owasp.dependencycheck.analyzer;
import java.io.BufferedInputStream;
import java.io.BufferedOutputStream;
import java.io.File;
import java.io.FileFilter;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
@@ -31,6 +32,7 @@ import java.util.Enumeration;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import org.apache.commons.compress.archivers.ArchiveEntry;
import org.apache.commons.compress.archivers.ArchiveInputStream;
import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;
@@ -44,6 +46,7 @@ import org.owasp.dependencycheck.Engine;
import org.owasp.dependencycheck.analyzer.exception.AnalysisException;
import org.owasp.dependencycheck.analyzer.exception.ArchiveExtractionException;
import org.owasp.dependencycheck.dependency.Dependency;
import org.owasp.dependencycheck.utils.FileFilterBuilder;
import org.owasp.dependencycheck.utils.FileUtils;
import org.owasp.dependencycheck.utils.Settings;
import org.slf4j.Logger;
@@ -51,8 +54,8 @@ import org.slf4j.LoggerFactory;
/**
* <p>
* An analyzer that extracts files from archives and ensures any supported files contained within the archive are added
* to the dependency list.</p>
* An analyzer that extracts files from archives and ensures any supported files contained within the archive are added to the
* dependency list.</p>
*
* @author Jeremy Long
*/
@@ -97,8 +100,8 @@ public class ArchiveAnalyzer extends AbstractFileTypeAnalyzer {
*/
private static final Set<String> ZIPPABLES = newHashSet("zip", "ear", "war", "jar", "sar", "apk", "nupkg");
/**
* The set of file extensions supported by this analyzer. Note for developers, any additions to this list will need
* to be explicitly handled in extractFiles().
* The set of file extensions supported by this analyzer. Note for developers, any additions to this list will need to be
* explicitly handled in extractFiles().
*/
private static final Set<String> EXTENSIONS = newHashSet("tar", "gz", "tgz");
@@ -116,14 +119,11 @@ public class ArchiveAnalyzer extends AbstractFileTypeAnalyzer {
EXTENSIONS.addAll(ZIPPABLES);
}
/**
* Returns a list of file EXTENSIONS supported by this analyzer.
*
* @return a list of file EXTENSIONS supported by this analyzer.
*/
private static final FileFilter FILTER = FileFilterBuilder.newInstance().addExtensions(EXTENSIONS).build();
@Override
public Set<String> getSupportedExtensions() {
return EXTENSIONS;
protected FileFilter getFileFilter() {
return FILTER;
}
/**
@@ -193,8 +193,8 @@ public class ArchiveAnalyzer extends AbstractFileTypeAnalyzer {
}
/**
* Analyzes a given dependency. If the dependency is an archive, such as a WAR or EAR, the contents are extracted,
* scanned, and added to the list of dependencies within the engine.
* Analyzes a given dependency. If the dependency is an archive, such as a WAR or EAR, the contents are extracted, scanned,
* and added to the list of dependencies within the engine.
*
* @param dependency the dependency to analyze
* @param engine the engine scanning
@@ -229,14 +229,14 @@ public class ArchiveAnalyzer extends AbstractFileTypeAnalyzer {
//TODO - can we get more evidence from the parent? EAR contains module name, etc.
//analyze the dependency (i.e. extract files) if it is a supported type.
if (this.supportsExtension(d.getFileExtension()) && scanDepth < MAX_SCAN_DEPTH) {
if (this.accept(d.getActualFile()) && scanDepth < MAX_SCAN_DEPTH) {
scanDepth += 1;
analyze(d, engine);
scanDepth -= 1;
}
}
}
if (this.REMOVE_FROM_ANALYSIS.contains(dependency.getFileExtension())) {
if (REMOVE_FROM_ANALYSIS.contains(dependency.getFileExtension())) {
if ("zip".equals(dependency.getFileExtension()) && isZipFileActuallyJarFile(dependency)) {
final File tdir = getNextTempDirectory();
final String fileName = dependency.getFileName();
@@ -320,9 +320,9 @@ public class ArchiveAnalyzer extends AbstractFileTypeAnalyzer {
extractArchive(new TarArchiveInputStream(new BufferedInputStream(fis)), destination, engine);
} else if ("gz".equals(archiveExt) || "tgz".equals(archiveExt)) {
final String uncompressedName = GzipUtils.getUncompressedFilename(archive.getName());
final String uncompressedExt = FileUtils.getFileExtension(uncompressedName).toLowerCase();
if (engine.supportsExtension(uncompressedExt)) {
decompressFile(new GzipCompressorInputStream(new BufferedInputStream(fis)), new File(destination, uncompressedName));
File f = new File(destination, uncompressedName);
if (engine.accept(f)) {
decompressFile(new GzipCompressorInputStream(new BufferedInputStream(fis)), f);
}
}
} catch (ArchiveExtractionException ex) {
@@ -362,8 +362,7 @@ public class ArchiveAnalyzer extends AbstractFileTypeAnalyzer {
}
} else {
final File file = new File(destination, entry.getName());
final String ext = FileUtils.getFileExtension(file.getName());
if (engine.supportsExtension(ext)) {
if (engine.accept(file)) {
LOGGER.debug("Extracting '{}'", file.getPath());
BufferedOutputStream bos = null;
FileOutputStream fos = null;

50
dependency-check-core/src/main/java/org/owasp/dependencycheck/analyzer/AssemblyAnalyzer.java
View File

@@ -17,36 +17,37 @@
*/
package org.owasp.dependencycheck.analyzer;
import ch.qos.cal10n.IMessageConveyor;
import ch.qos.cal10n.MessageConveyor;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileFilter;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.util.ArrayList;
import java.util.List;
import java.util.Locale;
import java.util.Set;
import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.xpath.XPath;
import javax.xml.xpath.XPathExpressionException;
import javax.xml.xpath.XPathFactory;
import ch.qos.cal10n.IMessageConveyor;
import ch.qos.cal10n.MessageConveyor;
import org.owasp.dependencycheck.Engine;
import org.owasp.dependencycheck.analyzer.exception.AnalysisException;
import org.owasp.dependencycheck.dependency.Confidence;
import org.owasp.dependencycheck.dependency.Dependency;
import org.owasp.dependencycheck.dependency.Evidence;
import org.owasp.dependencycheck.utils.DCResources;
import org.owasp.dependencycheck.utils.FileFilterBuilder;
import org.owasp.dependencycheck.utils.Settings;
import org.slf4j.cal10n.LocLogger;
import org.slf4j.cal10n.LocLoggerFactory;
import org.w3c.dom.Document;
import org.xml.sax.SAXException;
import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.xpath.XPath;
import javax.xml.xpath.XPathExpressionException;
import javax.xml.xpath.XPathFactory;
import java.util.ArrayList;
import java.util.List;
import java.util.Locale;
/**
* Analyzer for getting company, product, and version information from a .NET assembly.
*
@@ -66,7 +67,7 @@ public class AssemblyAnalyzer extends AbstractFileTypeAnalyzer {
/**
* The list of supported extensions
*/
private static final Set<String> SUPPORTED_EXTENSIONS = newHashSet("dll", "exe");
private static final String[] SUPPORTED_EXTENSIONS = {"dll", "exe"};
/**
* The temp value for GrokAssembly.exe
*/
@@ -78,15 +79,15 @@ public class AssemblyAnalyzer extends AbstractFileTypeAnalyzer {
/**
* Message Conveyer
*/
private final IMessageConveyor MESSAGE_CONVERYOR = new MessageConveyor(Locale.getDefault());
private static final IMessageConveyor MESSAGE_CONVERYOR = new MessageConveyor(Locale.getDefault());
/**
* LocLoggerFactory for localized logger
*/
private final LocLoggerFactory LLFACTORY = new LocLoggerFactory(MESSAGE_CONVERYOR);
private static final LocLoggerFactory LLFACTORY = new LocLoggerFactory(MESSAGE_CONVERYOR);
/**
* Logger
*/
private final LocLogger LOGGER = LLFACTORY.getLocLogger(AssemblyAnalyzer.class);
private static final LocLogger LOGGER = LLFACTORY.getLocLogger(AssemblyAnalyzer.class);
/**
* Builds the beginnings of a List for ProcessBuilder
@@ -284,6 +285,11 @@ public class AssemblyAnalyzer extends AbstractFileTypeAnalyzer {
builder = DocumentBuilderFactory.newInstance().newDocumentBuilder();
}
/**
* Removes resources used from the local file system.
*
* @throws Exception thrown if there is a problem closing the analyzer
*/
@Override
public void close() throws Exception {
super.close();
@@ -296,14 +302,12 @@ public class AssemblyAnalyzer extends AbstractFileTypeAnalyzer {
}
}
/**
* Gets the set of extensions supported by this analyzer.
*
* @return the list of supported extensions
*/
private static final FileFilter FILTER = FileFilterBuilder.newInstance().addExtensions(
SUPPORTED_EXTENSIONS).build();
@Override
public Set<String> getSupportedExtensions() {
return SUPPORTED_EXTENSIONS;
protected FileFilter getFileFilter() {
return FILTER;
}
/**

35
dependency-check-core/src/main/java/org/owasp/dependencycheck/analyzer/AutoconfAnalyzer.java
View File

@@ -17,23 +17,24 @@
*/
package org.owasp.dependencycheck.analyzer;
import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.Set;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.apache.commons.io.FileUtils;
import org.owasp.dependencycheck.Engine;
import org.owasp.dependencycheck.analyzer.exception.AnalysisException;
import org.owasp.dependencycheck.dependency.Confidence;
import org.owasp.dependencycheck.dependency.Dependency;
import org.owasp.dependencycheck.dependency.EvidenceCollection;
import org.owasp.dependencycheck.utils.FileFilterBuilder;
import org.owasp.dependencycheck.utils.Settings;
import org.owasp.dependencycheck.utils.UrlStringUtils;
import java.io.File;
import java.io.FileFilter;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
/**
* Used to analyze Autoconf input files named configure.ac or configure.in. Files simply named "configure" are also analyzed,
* assuming they are generated by Autoconf, and contain certain special package descriptor variables.
@@ -71,8 +72,7 @@ public class AutoconfAnalyzer extends AbstractFileTypeAnalyzer {
/**
* The set of file extensions supported by this analyzer.
*/
private static final Set<String> EXTENSIONS = newHashSet("ac", "in",
CONFIGURE);
private static final String[] EXTENSIONS = {"ac", "in"};
/**
* Matches AC_INIT variables in the output configure script.
@@ -104,13 +104,19 @@ public class AutoconfAnalyzer extends AbstractFileTypeAnalyzer {
}
/**
* Returns a list of file EXTENSIONS supported by this analyzer.
* The file filter used to determine which files this analyzer supports.
*/
private static final FileFilter FILTER = FileFilterBuilder.newInstance().addFilenames(CONFIGURE).addExtensions(
EXTENSIONS).build();
/**
* Returns the FileFilter
*
* @return a list of file EXTENSIONS supported by this analyzer.
* @return the FileFilter
*/
@Override
public Set<String> getSupportedExtensions() {
return EXTENSIONS;
protected FileFilter getFileFilter() {
return FILTER;
}
/**
@@ -128,6 +134,7 @@ public class AutoconfAnalyzer extends AbstractFileTypeAnalyzer {
*
* @return the phase that the analyzer is intended to run in.
*/
@Override
public AnalysisPhase getAnalysisPhase() {
return ANALYSIS_PHASE;
}

4
dependency-check-core/src/main/java/org/owasp/dependencycheck/analyzer/CPEAnalyzer.java
View File

@@ -511,8 +511,8 @@ public class CPEAnalyzer implements Analyzer {
}
for (VulnerableSoftware vs : cpes) {
DependencyVersion dbVer;
if (vs.getRevision() != null && !vs.getRevision().isEmpty()) {
dbVer = DependencyVersionUtil.parseVersion(vs.getVersion() + "." + vs.getRevision());
if (vs.getUpdate() != null && !vs.getUpdate().isEmpty()) {
dbVer = DependencyVersionUtil.parseVersion(vs.getVersion() + "." + vs.getUpdate());
} else {
dbVer = DependencyVersionUtil.parseVersion(vs.getVersion());
}

39
dependency-check-core/src/main/java/org/owasp/dependencycheck/analyzer/CentralAnalyzer.java
View File

@@ -17,12 +17,6 @@
*/
package org.owasp.dependencycheck.analyzer;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.net.URL;
import java.util.List;
import java.util.Set;
import org.apache.commons.io.FileUtils;
import org.owasp.dependencycheck.Engine;
import org.owasp.dependencycheck.analyzer.exception.AnalysisException;
@@ -32,13 +26,21 @@ import org.owasp.dependencycheck.dependency.Confidence;
import org.owasp.dependencycheck.dependency.Dependency;
import org.owasp.dependencycheck.dependency.Evidence;
import org.owasp.dependencycheck.xml.pom.PomUtils;
import org.owasp.dependencycheck.utils.DownloadFailedException;
import org.owasp.dependencycheck.utils.Downloader;
import org.owasp.dependencycheck.utils.InvalidSettingException;
import org.owasp.dependencycheck.utils.Settings;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.FileFilter;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.net.URL;
import java.util.List;
import org.owasp.dependencycheck.utils.DownloadFailedException;
import org.owasp.dependencycheck.utils.Downloader;
import org.owasp.dependencycheck.utils.FileFilterBuilder;
import org.owasp.dependencycheck.utils.InvalidSettingException;
import org.owasp.dependencycheck.utils.Settings;
/**
* Analyzer which will attempt to locate a dependency, and the GAV information, by querying Central for the dependency's SHA-1
* digest.
@@ -65,7 +67,7 @@ public class CentralAnalyzer extends AbstractFileTypeAnalyzer {
/**
* The types of files on which this will work.
*/
private static final Set<String> SUPPORTED_EXTENSIONS = newHashSet("jar");
private static final String SUPPORTED_EXTENSIONS = "jar";
/**
* The analyzer should be disabled if there are errors, so this is a flag to determine if such an error has occurred.
@@ -164,13 +166,13 @@ public class CentralAnalyzer extends AbstractFileTypeAnalyzer {
}
/**
* Returns the extensions for which this Analyzer runs.
*
* @return the extensions for which this Analyzer runs
* The file filter used to determine which files this analyzer supports.
*/
private static final FileFilter FILTER = FileFilterBuilder.newInstance().addExtensions(SUPPORTED_EXTENSIONS).build();
@Override
public Set<String> getSupportedExtensions() {
return SUPPORTED_EXTENSIONS;
protected FileFilter getFileFilter() {
return FILTER;
}
/**
@@ -206,7 +208,7 @@ public class CentralAnalyzer extends AbstractFileTypeAnalyzer {
pomFile = File.createTempFile("pom", ".xml", baseDir);
if (!pomFile.delete()) {
LOGGER.warn("Unable to fetch pom.xml for {} from Central; "
+ "this could result in undetected CPE/CVEs.", dependency.getFileName());
+ "this could result in undetected CPE/CVEs.", dependency.getFileName());
LOGGER.debug("Unable to delete temp file");
}
LOGGER.debug("Downloading {}", ma.getPomUrl());
@@ -215,7 +217,7 @@ public class CentralAnalyzer extends AbstractFileTypeAnalyzer {
} catch (DownloadFailedException ex) {
LOGGER.warn("Unable to download pom.xml for {} from Central; "
+ "this could result in undetected CPE/CVEs.", dependency.getFileName());
+ "this could result in undetected CPE/CVEs.", dependency.getFileName());
} finally {
if (pomFile != null && !FileUtils.deleteQuietly(pomFile)) {
pomFile.deleteOnExit();
@@ -233,5 +235,4 @@ public class CentralAnalyzer extends AbstractFileTypeAnalyzer {
errorFlag = true;
}
}
}

12
dependency-check-core/src/main/java/org/owasp/dependencycheck/analyzer/FileTypeAnalyzer.java
View File

@@ -17,20 +17,14 @@
*/
package org.owasp.dependencycheck.analyzer;
import java.io.FileFilter;
/**
* An Analyzer that scans specific file types.
*
* @author Jeremy Long
*/
public interface FileTypeAnalyzer extends Analyzer {
/**
* Returns whether or not this analyzer can process the given extension.
*
* @param extension the file extension to test for support.
* @return whether or not the specified file extension is supported by this analyzer.
*/
boolean supportsExtension(String extension);
public interface FileTypeAnalyzer extends Analyzer, FileFilter {
/**
* Resets the analyzers state.

27
dependency-check-core/src/main/java/org/owasp/dependencycheck/analyzer/JarAnalyzer.java
View File

@@ -17,14 +17,7 @@
*/
package org.owasp.dependencycheck.analyzer;
import java.io.BufferedOutputStream;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.OutputStream;
import java.io.Reader;
import java.io.*;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Enumeration;
@@ -47,6 +40,7 @@ import org.owasp.dependencycheck.analyzer.exception.AnalysisException;
import org.owasp.dependencycheck.dependency.Confidence;
import org.owasp.dependencycheck.dependency.Dependency;
import org.owasp.dependencycheck.dependency.EvidenceCollection;
import org.owasp.dependencycheck.utils.FileFilterBuilder;
import org.owasp.dependencycheck.xml.pom.License;
import org.owasp.dependencycheck.xml.pom.PomUtils;
import org.owasp.dependencycheck.xml.pom.Model;
@@ -168,16 +162,21 @@ public class JarAnalyzer extends AbstractFileTypeAnalyzer {
/**
* The set of file extensions supported by this analyzer.
*/
private static final Set<String> EXTENSIONS = newHashSet("jar", "war");
private static final String[] EXTENSIONS = {"jar", "war"};
/**
* Returns a list of file EXTENSIONS supported by this analyzer.
* The file filter used to determine which files this analyzer supports.
*/
private static final FileFilter FILTER = FileFilterBuilder.newInstance().addExtensions(EXTENSIONS).build();
/**
* Returns the FileFilter.
*
* @return a list of file EXTENSIONS supported by this analyzer.
* @return the FileFilter
*/
@Override
public Set<String> getSupportedExtensions() {
return EXTENSIONS;
protected FileFilter getFileFilter() {
return FILTER;
}
/**
@@ -388,7 +387,7 @@ public class JarAnalyzer extends AbstractFileTypeAnalyzer {
* @param dependency the dependency being analyzed
* @return returns the POM object
* @throws AnalysisException is thrown if there is an exception extracting or parsing the POM
* {@link org.owasp.dependencycheck.jaxb.pom.generated.Model} object
* {@link org.owasp.dependencycheck.xml.pom.Model} object
*/
private Model extractPom(String path, JarFile jar, Dependency dependency) throws AnalysisException {
InputStream input = null;

141
dependency-check-core/src/main/java/org/owasp/dependencycheck/analyzer/JavaScriptAnalyzer.java
View File

@@ -1,141 +0,0 @@
/*
* This file is part of dependency-check-core.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
* Copyright (c) 2014 Jeremy Long. All Rights Reserved.
*/
package org.owasp.dependencycheck.analyzer;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileReader;
import java.io.IOException;
import java.util.Set;
import java.util.regex.Pattern;
import org.owasp.dependencycheck.Engine;
import org.owasp.dependencycheck.analyzer.exception.AnalysisException;
import org.owasp.dependencycheck.dependency.Dependency;
import org.owasp.dependencycheck.utils.Settings;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
*
* Used to analyze a JavaScript file to gather information to aid in identification of a CPE identifier.
*
* @author Jeremy Long
*/
public class JavaScriptAnalyzer extends AbstractFileTypeAnalyzer {
/**
* The logger.
*/
private static final Logger LOGGER = LoggerFactory.getLogger(JavaScriptAnalyzer.class);
//<editor-fold defaultstate="collapsed" desc="All standard implementation details of Analyzer">
/**
* The name of the analyzer.
*/
private static final String ANALYZER_NAME = "JavaScript Analyzer";
/**
* The phase that this analyzer is intended to run in.
*/
private static final AnalysisPhase ANALYSIS_PHASE = AnalysisPhase.INFORMATION_COLLECTION;
/**
* The set of file extensions supported by this analyzer.
*/
private static final Set<String> EXTENSIONS = newHashSet("js");
/**
* Returns a list of file EXTENSIONS supported by this analyzer.
*
* @return a list of file EXTENSIONS supported by this analyzer.
*/
@Override
public Set<String> getSupportedExtensions() {
return EXTENSIONS;
}
/**
* Returns the name of the analyzer.
*
* @return the name of the analyzer.
*/
@Override
public String getName() {
return ANALYZER_NAME;
}
/**
* Returns the phase that the analyzer is intended to run in.
*
* @return the phase that the analyzer is intended to run in.
*/
@Override
public AnalysisPhase getAnalysisPhase() {
return ANALYSIS_PHASE;
}
//</editor-fold>
/**
* Returns the key used in the properties file to reference the analyzer's enabled property.
*
* @return the analyzer's enabled property setting key
*/
@Override
protected String getAnalyzerEnabledSettingKey() {
return Settings.KEYS.ANALYZER_JAVASCRIPT_ENABLED;
}
/**
* Loads a specified JavaScript file and collects information from the copyright information contained within.
*
* @param dependency the dependency to analyze.
* @param engine the engine that is scanning the dependencies
* @throws AnalysisException is thrown if there is an error reading the JavaScript file.
*/
@Override
public void analyzeFileType(Dependency dependency, Engine engine) throws AnalysisException {
BufferedReader fin = null;
try {
// /\*([^\*][^/]|[\r\n\f])+?\*/
final Pattern extractComments = Pattern.compile("(/\\*([^*]|[\\r\\n]|(\\*+([^*/]|[\\r\\n])))*\\*+/)|(//.*)", Pattern.MULTILINE);
File file = dependency.getActualFile();
fin = new BufferedReader(new FileReader(file));
StringBuilder sb = new StringBuilder(2000);
String text;
while ((text = fin.readLine()) != null) {
sb.append(text);
}
} catch (FileNotFoundException ex) {
final String msg = String.format("Dependency file not found: '%s'", dependency.getActualFilePath());
throw new AnalysisException(msg, ex);
} catch (IOException ex) {
LOGGER.error("", ex);
} finally {
if (fin != null) {
try {
fin.close();
} catch (IOException ex) {
LOGGER.trace("", ex);
}
}
}
}
@Override
protected void initializeFileTypeAnalyzer() throws Exception {
}
}

41
dependency-check-core/src/main/java/org/owasp/dependencycheck/analyzer/NexusAnalyzer.java
View File

@@ -17,12 +17,6 @@
*/
package org.owasp.dependencycheck.analyzer;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.Set;
import org.apache.commons.io.FileUtils;
import org.owasp.dependencycheck.Engine;
import org.owasp.dependencycheck.analyzer.exception.AnalysisException;
@@ -32,13 +26,21 @@ import org.owasp.dependencycheck.dependency.Confidence;
import org.owasp.dependencycheck.dependency.Dependency;
import org.owasp.dependencycheck.dependency.Evidence;
import org.owasp.dependencycheck.xml.pom.PomUtils;
import org.owasp.dependencycheck.utils.InvalidSettingException;
import org.owasp.dependencycheck.utils.DownloadFailedException;
import org.owasp.dependencycheck.utils.Downloader;
import org.owasp.dependencycheck.utils.Settings;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.FileFilter;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.net.MalformedURLException;
import java.net.URL;
import org.owasp.dependencycheck.utils.DownloadFailedException;
import org.owasp.dependencycheck.utils.Downloader;
import org.owasp.dependencycheck.utils.FileFilterBuilder;
import org.owasp.dependencycheck.utils.InvalidSettingException;
import org.owasp.dependencycheck.utils.Settings;
/**
* Analyzer which will attempt to locate a dependency on a Nexus service by SHA-1 digest of the dependency.
*
@@ -78,7 +80,7 @@ public class NexusAnalyzer extends AbstractFileTypeAnalyzer {
/**
* The types of files on which this will work.
*/
private static final Set<String> SUPPORTED_EXTENSIONS = newHashSet("jar");
private static final String SUPPORTED_EXTENSIONS = "jar";
/**
* The Nexus Search to be set up for this analyzer.
@@ -184,13 +186,18 @@ public class NexusAnalyzer extends AbstractFileTypeAnalyzer {
}
/**
* Returns the extensions for which this Analyzer runs.
* The file filter used to determine which files this analyzer supports.
*/
private static final FileFilter FILTER = FileFilterBuilder.newInstance().addExtensions(SUPPORTED_EXTENSIONS).build();
/**
* Returns the FileFilter
*
* @return the extensions for which this Analyzer runs
* @return the FileFilter
*/
@Override
public Set<String> getSupportedExtensions() {
return SUPPORTED_EXTENSIONS;
protected FileFilter getFileFilter() {
return FILTER;
}
/**
@@ -223,7 +230,7 @@ public class NexusAnalyzer extends AbstractFileTypeAnalyzer {
pomFile = File.createTempFile("pom", ".xml", baseDir);
if (!pomFile.delete()) {
LOGGER.warn("Unable to fetch pom.xml for {} from Nexus repository; "
+ "this could result in undetected CPE/CVEs.", dependency.getFileName());
+ "this could result in undetected CPE/CVEs.", dependency.getFileName());
LOGGER.debug("Unable to delete temp file");
}
LOGGER.debug("Downloading {}", ma.getPomUrl());
@@ -231,7 +238,7 @@ public class NexusAnalyzer extends AbstractFileTypeAnalyzer {
PomUtils.analyzePOM(dependency, pomFile);
} catch (DownloadFailedException ex) {
LOGGER.warn("Unable to download pom.xml for {} from Nexus repository; "
+ "this could result in undetected CPE/CVEs.", dependency.getFileName());
+ "this could result in undetected CPE/CVEs.", dependency.getFileName());
} finally {
if (pomFile != null && !FileUtils.deleteQuietly(pomFile)) {
pomFile.deleteOnExit();

26
dependency-check-core/src/main/java/org/owasp/dependencycheck/analyzer/NuspecAnalyzer.java
View File

@@ -17,10 +17,6 @@
*/
package org.owasp.dependencycheck.analyzer;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.util.Set;
import org.owasp.dependencycheck.Engine;
import org.owasp.dependencycheck.analyzer.exception.AnalysisException;
import org.owasp.dependencycheck.data.nuget.NugetPackage;
@@ -29,10 +25,16 @@ import org.owasp.dependencycheck.data.nuget.NuspecParser;
import org.owasp.dependencycheck.data.nuget.XPathNuspecParser;
import org.owasp.dependencycheck.dependency.Confidence;
import org.owasp.dependencycheck.dependency.Dependency;
import org.owasp.dependencycheck.utils.FileFilterBuilder;
import org.owasp.dependencycheck.utils.Settings;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.FileFilter;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
/**
* Analyzer which will parse a Nuspec file to gather module information.
*
@@ -58,7 +60,7 @@ public class NuspecAnalyzer extends AbstractFileTypeAnalyzer {
/**
* The types of files on which this will work.
*/
private static final Set<String> SUPPORTED_EXTENSIONS = newHashSet("nuspec");
private static final String SUPPORTED_EXTENSIONS = "nuspec";
/**
* Initializes the analyzer once before any analysis is performed.
@@ -100,13 +102,19 @@ public class NuspecAnalyzer extends AbstractFileTypeAnalyzer {
}
/**
* Returns the extensions for which this Analyzer runs.
* The file filter used to determine which files this analyzer supports.
*/
private static final FileFilter FILTER = FileFilterBuilder.newInstance().addExtensions(
SUPPORTED_EXTENSIONS).build();
/**
* Returns the FileFilter
*
* @return the extensions for which this Analyzer runs
* @return the FileFilter
*/
@Override
public Set<String> getSupportedExtensions() {
return SUPPORTED_EXTENSIONS;
protected FileFilter getFileFilter() {
return FILTER;
}
/**

44
dependency-check-core/src/main/java/org/owasp/dependencycheck/analyzer/PythonDistributionAnalyzer.java
View File

@@ -17,17 +17,6 @@
*/
package org.owasp.dependencycheck.analyzer;
import java.io.BufferedInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FilenameFilter;
import java.util.Set;
import java.util.regex.Pattern;
import javax.mail.MessagingException;
import javax.mail.internet.InternetHeaders;
import org.apache.commons.io.filefilter.NameFileFilter;
import org.apache.commons.io.filefilter.SuffixFileFilter;
import org.apache.commons.io.input.AutoCloseInputStream;
@@ -37,13 +26,19 @@ import org.owasp.dependencycheck.analyzer.exception.AnalysisException;
import org.owasp.dependencycheck.dependency.Confidence;
import org.owasp.dependencycheck.dependency.Dependency;
import org.owasp.dependencycheck.dependency.EvidenceCollection;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.mail.MessagingException;
import javax.mail.internet.InternetHeaders;
import java.io.*;
import java.util.regex.Pattern;
import org.owasp.dependencycheck.utils.ExtractionException;
import org.owasp.dependencycheck.utils.ExtractionUtil;
import org.owasp.dependencycheck.utils.FileFilterBuilder;
import org.owasp.dependencycheck.utils.FileUtils;
import org.owasp.dependencycheck.utils.Settings;
import org.owasp.dependencycheck.utils.UrlStringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* Used to analyze a Wheel or egg distribution files, or their contents in unzipped form, and collect information that can be used
@@ -86,11 +81,10 @@ public class PythonDistributionAnalyzer extends AbstractFileTypeAnalyzer {
/**
* The set of file extensions supported by this analyzer.
*/
private static final Set<String> EXTENSIONS = newHashSet("whl", "egg",
"zip", METADATA, PKG_INFO);
private static final String[] EXTENSIONS = {"whl", "egg", "zip"};
/**
* Used to match on egg archive candidate extenssions.
* Used to match on egg archive candidate extensions.
*/
private static final Pattern EGG_OR_ZIP = Pattern.compile("egg|zip");
@@ -114,23 +108,29 @@ public class PythonDistributionAnalyzer extends AbstractFileTypeAnalyzer {
/**
* Filter that detects files named "METADATA".
*/
private static final FilenameFilter METADATA_FILTER = new NameFileFilter(
private static final NameFileFilter METADATA_FILTER = new NameFileFilter(
METADATA);
/**
* Filter that detects files named "PKG-INFO".
*/
private static final FilenameFilter PKG_INFO_FILTER = new NameFileFilter(
private static final NameFileFilter PKG_INFO_FILTER = new NameFileFilter(
PKG_INFO);
/**
* Returns a list of file EXTENSIONS supported by this analyzer.
* The file filter used to determine which files this analyzer supports.
*/
private static final FileFilter FILTER = FileFilterBuilder.newInstance().addFileFilters(
METADATA_FILTER, PKG_INFO_FILTER).addExtensions(EXTENSIONS).build();
/**
* Returns the FileFilter
*
* @return a list of file EXTENSIONS supported by this analyzer.
* @return the FileFilter
*/
@Override
public Set<String> getSupportedExtensions() {
return EXTENSIONS;
protected FileFilter getFileFilter() {
return FILTER;
}
/**

41
dependency-check-core/src/main/java/org/owasp/dependencycheck/analyzer/PythonPackageAnalyzer.java
View File

@@ -17,17 +17,6 @@
*/
package org.owasp.dependencycheck.analyzer;
import java.io.File;
import java.io.FileFilter;
import java.io.IOException;
import java.net.MalformedURLException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Set;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.filefilter.NameFileFilter;
import org.apache.commons.io.filefilter.SuffixFileFilter;
@@ -36,11 +25,21 @@ import org.owasp.dependencycheck.analyzer.exception.AnalysisException;
import org.owasp.dependencycheck.dependency.Confidence;
import org.owasp.dependencycheck.dependency.Dependency;
import org.owasp.dependencycheck.dependency.EvidenceCollection;
import org.owasp.dependencycheck.utils.FileFilterBuilder;
import org.owasp.dependencycheck.utils.Settings;
import org.owasp.dependencycheck.utils.UrlStringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.FileFilter;
import java.io.IOException;
import java.net.MalformedURLException;
import java.util.ArrayList;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
/**
* Used to analyze a Python package, and collect information that can be used to determine the associated CPE.
*
@@ -63,8 +62,7 @@ public class PythonPackageAnalyzer extends AbstractFileTypeAnalyzer {
/**
* Filename extensions for files to be analyzed.
*/
private static final Set<String> EXTENSIONS = Collections
.unmodifiableSet(Collections.singleton("py"));
private static final String EXTENSIONS = "py";
/**
* Pattern for matching the module docstring in a source file.
@@ -135,13 +133,18 @@ public class PythonPackageAnalyzer extends AbstractFileTypeAnalyzer {
}
/**
* Returns the set of supported file extensions.
* The file filter used to determine which files this analyzer supports.
*/
private static final FileFilter FILTER = FileFilterBuilder.newInstance().addExtensions(EXTENSIONS).build();
/**
* Returns the FileFilter
*
* @return the set of supported file extensions
* @return the FileFilter
*/
@Override
protected Set<String> getSupportedExtensions() {
return EXTENSIONS;
protected FileFilter getFileFilter() {
return FILTER;
}
/**
@@ -209,12 +212,12 @@ public class PythonPackageAnalyzer extends AbstractFileTypeAnalyzer {
*/
private boolean analyzeFileContents(Dependency dependency, File file)
throws AnalysisException {
String contents = "";
String contents;
try {
contents = FileUtils.readFileToString(file).trim();
} catch (IOException e) {
throw new AnalysisException(
"Problem occured while reading dependency file.", e);
"Problem occurred while reading dependency file.", e);
}
boolean found = false;
if (!contents.isEmpty()) {

92
dependency-check-core/src/main/java/org/owasp/dependencycheck/data/nvdcve/ConnectionFactory.java
View File

@@ -35,9 +35,8 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* Loads the configured database driver and returns the database connection. If the embedded H2 database is used
* obtaining a connection will ensure the database file exists and that the appropriate table structure has been
* created.
* Loads the configured database driver and returns the database connection. If the embedded H2 database is used obtaining a
* connection will ensure the database file exists and that the appropriate table structure has been created.
*
* @author Jeremy Long
*/
@@ -55,6 +54,10 @@ public final class ConnectionFactory {
* Resource location for SQL file used to create the database schema.
*/
public static final String DB_STRUCTURE_RESOURCE = "data/initialize.sql";
/**
* Resource location for SQL file used to create the database schema.
*/
public static final String DB_STRUCTURE_UPDATE_RESOURCE = "data/upgrade_%s.sql";
/**
* The database driver used to connect to the database.
*/
@@ -79,8 +82,8 @@ public final class ConnectionFactory {
}
/**
* Initializes the connection factory. Ensuring that the appropriate drivers are loaded and that a connection can be
* made successfully.
* Initializes the connection factory. Ensuring that the appropriate drivers are loaded and that a connection can be made
* successfully.
*
* @throws DatabaseException thrown if we are unable to connect to the database
*/
@@ -114,8 +117,7 @@ public final class ConnectionFactory {
try {
connectionString = Settings.getConnectionString(
Settings.KEYS.DB_CONNECTION_STRING,
Settings.KEYS.DB_FILE_NAME,
Settings.KEYS.DB_VERSION);
Settings.KEYS.DB_FILE_NAME);
} catch (IOException ex) {
LOGGER.debug(
"Unable to retrieve the database connection string", ex);
@@ -162,13 +164,12 @@ public final class ConnectionFactory {
LOGGER.debug("", dex);
throw new DatabaseException("Unable to create the database structure");
}
} else {
try {
ensureSchemaVersion(conn);
} catch (DatabaseException dex) {
LOGGER.debug("", dex);
throw new DatabaseException("Database schema does not match this version of dependency-check");
}
}
try {
ensureSchemaVersion(conn);
} catch (DatabaseException dex) {
LOGGER.debug("", dex);
throw new DatabaseException("Database schema does not match this version of dependency-check", dex);
}
} finally {
if (conn != null) {
@@ -183,8 +184,8 @@ public final class ConnectionFactory {
/**
* Cleans up resources and unloads any registered database drivers. This needs to be called to ensure the driver is
* unregistered prior to the finalize method being called as during shutdown the class loader used to load the
* driver may be unloaded prior to the driver being de-registered.
* unregistered prior to the finalize method being called as during shutdown the class loader used to load the driver may be
* unloaded prior to the driver being de-registered.
*/
public static synchronized void cleanup() {
if (driver != null) {
@@ -229,8 +230,7 @@ public final class ConnectionFactory {
*/
private static boolean h2DataFileExists() throws IOException {
final File dir = Settings.getDataDirectory();
final String name = Settings.getString(Settings.KEYS.DB_FILE_NAME);
final String fileName = String.format(name, DB_SCHEMA_VERSION);
final String fileName = Settings.getString(Settings.KEYS.DB_FILE_NAME);
final File file = new File(dir, fileName);
return file.exists();
}
@@ -278,6 +278,55 @@ public final class ConnectionFactory {
}
}
/**
* Updates the database schema by loading the upgrade script for the version specified. The intended use is that if the
* current schema version is 2.9 then we would call updateSchema(conn, "2.9"). This would load the upgrade_2.9.sql file and
* execute it against the database. The upgrade script must update the 'version' in the properties table.
*
* @param conn the database connection object
* @param schema the current schema version that is being upgraded
* @throws DatabaseException thrown if there is an exception upgrading the database schema
*/
private static void updateSchema(Connection conn, String schema) throws DatabaseException {
LOGGER.debug("Updating database structure");
InputStream is;
InputStreamReader reader;
BufferedReader in = null;
String updateFile = null;
try {
updateFile = String.format(DB_STRUCTURE_UPDATE_RESOURCE, schema);
is = ConnectionFactory.class.getClassLoader().getResourceAsStream(updateFile);
reader = new InputStreamReader(is, "UTF-8");
in = new BufferedReader(reader);
final StringBuilder sb = new StringBuilder(2110);
String tmp;
while ((tmp = in.readLine()) != null) {
sb.append(tmp);
}
Statement statement = null;
try {
statement = conn.createStatement();
statement.execute(sb.toString());
} catch (SQLException ex) {
LOGGER.debug("", ex);
throw new DatabaseException("Unable to update database schema", ex);
} finally {
DBUtils.closeStatement(statement);
}
} catch (IOException ex) {
final String msg = String.format("Upgrade SQL file does not exist: %s", updateFile);
throw new DatabaseException(msg, ex);
} finally {
if (in != null) {
try {
in.close();
} catch (IOException ex) {
LOGGER.trace("", ex);
}
}
}
}
/**
* Uses the provided connection to check the specified schema version within the database.
*
@@ -288,12 +337,13 @@ public final class ConnectionFactory {
ResultSet rs = null;
CallableStatement cs = null;
try {
//TODO convert this to use DatabaseProperties
cs = conn.prepareCall("SELECT value FROM properties WHERE id = 'version'");
rs = cs.executeQuery();
if (rs.next()) {
final boolean isWrongSchema = !DB_SCHEMA_VERSION.equals(rs.getString(1));
if (isWrongSchema) {
throw new DatabaseException("Incorrect database schema; unable to continue");
if (!DB_SCHEMA_VERSION.equals(rs.getString(1))) {
LOGGER.debug("Updating from version: " + rs.getString(1));
updateSchema(conn, rs.getString(1));
}
} else {
throw new DatabaseException("Database schema is missing");

42
dependency-check-core/src/main/java/org/owasp/dependencycheck/data/nvdcve/CveDB.java
View File

@@ -19,6 +19,7 @@ package org.owasp.dependencycheck.data.nvdcve;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.sql.CallableStatement;
import java.sql.Connection;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
@@ -766,8 +767,8 @@ public class CveDB {
DependencyVersion cpeVersion;
if (cpe.getVersion() != null && !cpe.getVersion().isEmpty()) {
String versionText;
if (cpe.getRevision() != null && !cpe.getRevision().isEmpty()) {
versionText = String.format("%s.%s", cpe.getVersion(), cpe.getRevision());
if (cpe.getUpdate() != null && !cpe.getUpdate().isEmpty()) {
versionText = String.format("%s.%s", cpe.getVersion(), cpe.getUpdate());
} else {
versionText = cpe.getVersion();
}
@@ -777,4 +778,41 @@ public class CveDB {
}
return cpeVersion;
}
/**
* Deletes unused dictionary entries from the database.
*/
public void deleteUnusedCpe() {
CallableStatement cs = null;
try {
cs = getConnection().prepareCall(statementBundle.getString("DELETE_UNUSED_DICT_CPE"));
cs.executeUpdate();
} catch (SQLException ex) {
LOGGER.error("Unable to delete CPE dictionary entries", ex);
} finally {
DBUtils.closeStatement(cs);
}
}
/**
* Merges CPE entries into the database.
*
* @param cpe the CPE identifier
* @param vendor the CPE vendor
* @param product the CPE product
*/
public void addCpe(String cpe, String vendor, String product) {
PreparedStatement ps = null;
try {
ps = getConnection().prepareCall(statementBundle.getString("ADD_DICT_CPE"));
ps.setString(1, cpe);
ps.setString(2, vendor);
ps.setString(3, product);
ps.executeUpdate();
} catch (SQLException ex) {
LOGGER.error("Unable to add CPE dictionary entry", ex);
} finally {
DBUtils.closeStatement(ps);
}
}
}

33
dependency-check-core/src/main/java/org/owasp/dependencycheck/data/nvdcve/DatabaseProperties.java
View File

@@ -24,7 +24,7 @@ import java.util.Map;
import java.util.Map.Entry;
import java.util.Properties;
import java.util.TreeMap;
import org.owasp.dependencycheck.data.update.NvdCveInfo;
import org.owasp.dependencycheck.data.update.nvd.NvdCveInfo;
import org.owasp.dependencycheck.data.update.exception.UpdateException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -41,20 +41,28 @@ public class DatabaseProperties {
*/
private static final Logger LOGGER = LoggerFactory.getLogger(DatabaseProperties.class);
/**
* Modified key word, used as a key to store information about the modified file (i.e. the containing the last 8
* days of updates)..
* Modified key word, used as a key to store information about the modified file (i.e. the containing the last 8 days of
* updates)..
*/
public static final String MODIFIED = "Modified";
/**
* The properties file key for the last updated field - used to store the last updated time of the Modified NVD CVE
* xml file.
* The properties file key for the last updated field - used to store the last updated time of the Modified NVD CVE xml file.
*/
public static final String LAST_UPDATED = "NVD CVE Modified";
/**
* Stores the last updated time for each of the NVD CVE files. These timestamps should be updated if we process the
* modified file within 7 days of the last update.
* Stores the last updated time for each of the NVD CVE files. These timestamps should be updated if we process the modified
* file within 7 days of the last update.
*/
public static final String LAST_UPDATED_BASE = "NVD CVE ";
/**
* The key for the last time the CPE data was updated.
*/
public static final String LAST_CPE_UPDATE = "LAST_CPE_UPDATE";
/**
* The key for the database schema version.
*/
public static final String VERSION = "version";
/**
* A collection of properties about the data.
*/
@@ -116,8 +124,7 @@ public class DatabaseProperties {
}
/**
* Returns the property value for the given key. If the key is not contained in the underlying properties null is
* returned.
* Returns the property value for the given key. If the key is not contained in the underlying properties null is returned.
*
* @param key the property key
* @return the value of the property
@@ -127,8 +134,8 @@ public class DatabaseProperties {
}
/**
* Returns the property value for the given key. If the key is not contained in the underlying properties the
* default value is returned.
* Returns the property value for the given key. If the key is not contained in the underlying properties the default value is
* returned.
*
* @param key the property key
* @param defaultValue the default value
@@ -148,8 +155,8 @@ public class DatabaseProperties {
}
/**
* Returns a map of the meta data from the database properties. This primarily contains timestamps of when the NVD
* CVE information was last updated.
* Returns a map of the meta data from the database properties. This primarily contains timestamps of when the NVD CVE
* information was last updated.
*
* @return a map of the database meta data
*/

86
dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/BaseUpdater.java Normal file
View File

@@ -0,0 +1,86 @@
/*
* This file is part of dependency-check-core.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
* Copyright (c) 2015 Jeremy Long. All Rights Reserved.
*/
package org.owasp.dependencycheck.data.update;
import org.owasp.dependencycheck.data.nvdcve.CveDB;
import org.owasp.dependencycheck.data.nvdcve.DatabaseException;
import org.owasp.dependencycheck.data.nvdcve.DatabaseProperties;
import org.owasp.dependencycheck.data.update.exception.UpdateException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
*
* @author Jeremy Long
*/
public abstract class BaseUpdater {
/**
* Static logger.
*/
private static final Logger LOGGER = LoggerFactory.getLogger(BaseUpdater.class);
/**
* Information about the timestamps and URLs for data that needs to be updated.
*/
private DatabaseProperties properties;
/**
* Reference to the Cve Database.
*/
private CveDB cveDB = null;
protected CveDB getCveDB() {
return cveDB;
}
protected DatabaseProperties getProperties() {
return properties;
}
/**
* Closes the CVE and CPE data stores.
*/
protected void closeDataStores() {
if (cveDB != null) {
try {
cveDB.close();
} catch (Throwable ignore) {
LOGGER.trace("Error closing the database", ignore);
}
}
}
/**
* Opens the data store.
*
* @throws UpdateException thrown if a data store cannot be opened
*/
protected final void openDataStores() throws UpdateException {
if (cveDB != null) {
return;
}
try {
cveDB = new CveDB();
cveDB.open();
} catch (DatabaseException ex) {
closeDataStores();
LOGGER.debug("Database Exception opening databases", ex);
throw new UpdateException("Error updating the database, please see the log file for more details.");
}
properties = cveDB.getDatabaseProperties();
}
}

198
dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/CpeUpdater.java Normal file
View File

@@ -0,0 +1,198 @@
/*
* This file is part of dependency-check-core.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
* Copyright (c) 2015 Jeremy Long. All Rights Reserved.
*/
package org.owasp.dependencycheck.data.update;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.Date;
import java.util.List;
import java.util.zip.GZIPInputStream;
import javax.xml.parsers.ParserConfigurationException;
import javax.xml.parsers.SAXParser;
import javax.xml.parsers.SAXParserFactory;
import org.apache.commons.io.FileUtils;
import static org.owasp.dependencycheck.data.nvdcve.DatabaseProperties.LAST_CPE_UPDATE;
import org.owasp.dependencycheck.data.update.cpe.CPEHandler;
import org.owasp.dependencycheck.data.update.cpe.Cpe;
import org.owasp.dependencycheck.data.update.exception.UpdateException;
import org.owasp.dependencycheck.utils.DateUtil;
import org.owasp.dependencycheck.utils.DownloadFailedException;
import org.owasp.dependencycheck.utils.Downloader;
import org.owasp.dependencycheck.utils.Settings;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.xml.sax.SAXException;
/**
* The CpeUpdater is designed to download the CPE data file from NIST and import the data into the database. However, as this
* currently adds no beneficial data, compared to what is in the CPE data contained in the CVE data files, this class is not
* currently used. The code is being kept as a future update may utilize more data from the CPE xml files.
*
* @author Jeremy Long
*/
public class CpeUpdater extends BaseUpdater implements CachedWebDataSource {
/**
* Static logger.
*/
private static final Logger LOGGER = LoggerFactory.getLogger(CpeUpdater.class);
@Override
public void update() throws UpdateException {
try {
openDataStores();
if (updateNeeded()) {
LOGGER.info("Updating the Common Platform Enumeration (CPE)");
final File xml = downloadCpe();
final List<Cpe> cpes = processXML(xml);
getCveDB().deleteUnusedCpe();
for (Cpe cpe : cpes) {
getCveDB().addCpe(cpe.getValue(), cpe.getVendor(), cpe.getProduct());
}
final Date now = new Date();
getProperties().save(LAST_CPE_UPDATE, Long.toString(now.getTime()));
LOGGER.info("CPE update complete");
}
} finally {
closeDataStores();
}
}
/**
* Downloads the CPE XML file.
*
* @return the file reference to the CPE.xml file
* @throws UpdateException thrown if there is an issue downloading the XML file
*/
private File downloadCpe() throws UpdateException {
File xml;
final URL url;
try {
url = new URL(Settings.getString(Settings.KEYS.CPE_URL));
xml = File.createTempFile("cpe", ".xml", Settings.getTempDirectory());
Downloader.fetchFile(url, xml);
if (url.toExternalForm().endsWith(".xml.gz")) {
extractGzip(xml);
}
} catch (MalformedURLException ex) {
throw new UpdateException("Invalid CPE URL", ex);
} catch (DownloadFailedException ex) {
throw new UpdateException("Unable to download CPE XML file", ex);
} catch (IOException ex) {
throw new UpdateException("Unable to create temporary file to download CPE", ex);
}
return xml;
}
/**
* Parses the CPE XML file to return a list of CPE entries.
*
* @param xml the CPE data file
* @return the list of CPE entries
* @throws UpdateException thrown if there is an issue with parsing the XML file
*/
private List<Cpe> processXML(final File xml) throws UpdateException {
try {
final SAXParserFactory factory = SAXParserFactory.newInstance();
final SAXParser saxParser = factory.newSAXParser();
final CPEHandler handler = new CPEHandler();
saxParser.parse(xml, handler);
return handler.getData();
} catch (ParserConfigurationException ex) {
throw new UpdateException("Unable to parse CPE XML file due to SAX Parser Issue", ex);
} catch (SAXException ex) {
throw new UpdateException("Unable to parse CPE XML file due to SAX Parser Exception", ex);
} catch (IOException ex) {
throw new UpdateException("Unable to parse CPE XML file due to IO Failure", ex);
}
}
/**
* Checks to find the last time the CPE data was refreshed and if it needs to be updated.
*
* @return true if the CPE data should be refreshed
*/
private boolean updateNeeded() {
final Date now = new Date();
final int days = Settings.getInt(Settings.KEYS.CVE_MODIFIED_VALID_FOR_DAYS, 30);
long timestamp = 0;
final String ts = getProperties().getProperty(LAST_CPE_UPDATE);
if (ts != null && ts.matches("^[0-9]+$")) {
timestamp = Long.parseLong(ts);
}
return !DateUtil.withinDateRange(timestamp, now.getTime(), days);
}
/**
* Extracts the file contained in a gzip archive. The extracted file is placed in the exact same path as the file specified.
*
* @param file the archive file
* @throws FileNotFoundException thrown if the file does not exist
* @throws IOException thrown if there is an error extracting the file.
*/
private void extractGzip(File file) throws FileNotFoundException, IOException {
//TODO - move this to a util class as it is duplicative of (copy of) code in the DownloadTask
final String originalPath = file.getPath();
final File gzip = new File(originalPath + ".gz");
if (gzip.isFile() && !gzip.delete()) {
gzip.deleteOnExit();
}
if (!file.renameTo(gzip)) {
throw new IOException("Unable to rename '" + file.getPath() + "'");
}
final File newfile = new File(originalPath);
final byte[] buffer = new byte[4096];
GZIPInputStream cin = null;
FileOutputStream out = null;
try {
cin = new GZIPInputStream(new FileInputStream(gzip));
out = new FileOutputStream(newfile);
int len;
while ((len = cin.read(buffer)) > 0) {
out.write(buffer, 0, len);
}
} finally {
if (cin != null) {
try {
cin.close();
} catch (IOException ex) {
LOGGER.trace("ignore", ex);
}
}
if (out != null) {
try {
out.close();
} catch (IOException ex) {
LOGGER.trace("ignore", ex);
}
}
if (gzip.isFile()) {
FileUtils.deleteQuietly(gzip);
}
}
}
}

229
dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/NvdCveUpdater.java
View File

@@ -18,23 +18,44 @@
package org.owasp.dependencycheck.data.update;
import java.net.MalformedURLException;
import java.util.Calendar;
import java.util.Date;
import java.util.HashSet;
import java.util.Set;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import org.owasp.dependencycheck.data.nvdcve.DatabaseProperties;
import static org.owasp.dependencycheck.data.nvdcve.DatabaseProperties.MODIFIED;
import org.owasp.dependencycheck.data.update.exception.InvalidDataException;
import org.owasp.dependencycheck.data.update.exception.UpdateException;
import org.owasp.dependencycheck.data.update.nvd.DownloadTask;
import org.owasp.dependencycheck.data.update.nvd.NvdCveInfo;
import org.owasp.dependencycheck.data.update.nvd.ProcessTask;
import org.owasp.dependencycheck.data.update.nvd.UpdateableNvdCve;
import org.owasp.dependencycheck.utils.DateUtil;
import org.owasp.dependencycheck.utils.DownloadFailedException;
import org.owasp.dependencycheck.utils.InvalidSettingException;
import org.owasp.dependencycheck.utils.Settings;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* Class responsible for updating the NVD CVE and CPE data stores.
* Class responsible for updating the NVD CVE data.
*
* @author Jeremy Long
*/
public class NvdCveUpdater implements CachedWebDataSource {
public class NvdCveUpdater extends BaseUpdater implements CachedWebDataSource {
/**
* The logger
*/
private static final Logger LOGGER = LoggerFactory.getLogger(NvdCveUpdater.class);
/**
* The max thread pool size to use when downloading files.
*/
public static final int MAX_THREAD_POOL_SIZE = Settings.getInt(Settings.KEYS.MAX_DOWNLOAD_THREAD_POOL_SIZE, 3);
/**
* <p>
@@ -45,9 +66,10 @@ public class NvdCveUpdater implements CachedWebDataSource {
@Override
public void update() throws UpdateException {
try {
final StandardUpdate task = new StandardUpdate();
if (task.isUpdateNeeded()) {
task.update();
openDataStores();
final UpdateableNvdCve updateable = getUpdatesNeeded();
if (updateable.isUpdateNeeded()) {
performUpdate(updateable);
}
} catch (MalformedURLException ex) {
LOGGER.warn(
@@ -61,6 +83,203 @@ public class NvdCveUpdater implements CachedWebDataSource {
"If you are behind a proxy you may need to configure dependency-check to use the proxy.");
}
LOGGER.debug("", ex);
} finally {
closeDataStores();
}
}
/**
* Downloads the latest NVD CVE XML file from the web and imports it into the current CVE Database.
*
* @param updateable a collection of NVD CVE data file references that need to be downloaded and processed to update the
* database
* @throws UpdateException is thrown if there is an error updating the database
*/
public void performUpdate(UpdateableNvdCve updateable) throws UpdateException {
int maxUpdates = 0;
try {
for (NvdCveInfo cve : updateable) {
if (cve.getNeedsUpdate()) {
maxUpdates += 1;
}
}
if (maxUpdates <= 0) {
return;
}
if (maxUpdates > 3) {
LOGGER.info(
"NVD CVE requires several updates; this could take a couple of minutes.");
}
if (maxUpdates > 0) {
openDataStores();
}
final int poolSize = (MAX_THREAD_POOL_SIZE < maxUpdates) ? MAX_THREAD_POOL_SIZE : maxUpdates;
final ExecutorService downloadExecutors = Executors.newFixedThreadPool(poolSize);
final ExecutorService processExecutor = Executors.newSingleThreadExecutor();
final Set<Future<Future<ProcessTask>>> downloadFutures = new HashSet<Future<Future<ProcessTask>>>(maxUpdates);
for (NvdCveInfo cve : updateable) {
if (cve.getNeedsUpdate()) {
final DownloadTask call = new DownloadTask(cve, processExecutor, getCveDB(), Settings.getInstance());
downloadFutures.add(downloadExecutors.submit(call));
}
}
downloadExecutors.shutdown();
//next, move the future future processTasks to just future processTasks
final Set<Future<ProcessTask>> processFutures = new HashSet<Future<ProcessTask>>(maxUpdates);
for (Future<Future<ProcessTask>> future : downloadFutures) {
Future<ProcessTask> task = null;
try {
task = future.get();
} catch (InterruptedException ex) {
downloadExecutors.shutdownNow();
processExecutor.shutdownNow();
LOGGER.debug("Thread was interrupted during download", ex);
throw new UpdateException("The download was interrupted", ex);
} catch (ExecutionException ex) {
downloadExecutors.shutdownNow();
processExecutor.shutdownNow();
LOGGER.debug("Thread was interrupted during download execution", ex);
throw new UpdateException("The execution of the download was interrupted", ex);
}
if (task == null) {
downloadExecutors.shutdownNow();
processExecutor.shutdownNow();
LOGGER.debug("Thread was interrupted during download");
throw new UpdateException("The download was interrupted; unable to complete the update");
} else {
processFutures.add(task);
}
}
for (Future<ProcessTask> future : processFutures) {
try {
final ProcessTask task = future.get();
if (task.getException() != null) {
throw task.getException();
}
} catch (InterruptedException ex) {
processExecutor.shutdownNow();
LOGGER.debug("Thread was interrupted during processing", ex);
throw new UpdateException(ex);
} catch (ExecutionException ex) {
processExecutor.shutdownNow();
LOGGER.debug("Execution Exception during process", ex);
throw new UpdateException(ex);
} finally {
processExecutor.shutdown();
}
}
if (maxUpdates >= 1) { //ensure the modified file date gets written (we may not have actually updated it)
getProperties().save(updateable.get(MODIFIED));
LOGGER.info("Begin database maintenance.");
getCveDB().cleanupDatabase();
LOGGER.info("End database maintenance.");
}
} finally {
closeDataStores();
}
}
/**
* Determines if the index needs to be updated. This is done by fetching the NVD CVE meta data and checking the last update
* date. If the data needs to be refreshed this method will return the NvdCveUrl for the files that need to be updated.
*
* @return the collection of files that need to be updated
* @throws MalformedURLException is thrown if the URL for the NVD CVE Meta data is incorrect
* @throws DownloadFailedException is thrown if there is an error. downloading the NVD CVE download data file
* @throws UpdateException Is thrown if there is an issue with the last updated properties file
*/
protected final UpdateableNvdCve getUpdatesNeeded() throws MalformedURLException, DownloadFailedException, UpdateException {
UpdateableNvdCve updates = null;
try {
updates = retrieveCurrentTimestampsFromWeb();
} catch (InvalidDataException ex) {
final String msg = "Unable to retrieve valid timestamp from nvd cve downloads page";
LOGGER.debug(msg, ex);
throw new DownloadFailedException(msg, ex);
} catch (InvalidSettingException ex) {
LOGGER.debug("Invalid setting found when retrieving timestamps", ex);
throw new DownloadFailedException("Invalid settings", ex);
}
if (updates == null) {
throw new DownloadFailedException("Unable to retrieve the timestamps of the currently published NVD CVE data");
}
if (!getProperties().isEmpty()) {
try {
final long lastUpdated = Long.parseLong(getProperties().getProperty(DatabaseProperties.LAST_UPDATED, "0"));
final Date now = new Date();
final int days = Settings.getInt(Settings.KEYS.CVE_MODIFIED_VALID_FOR_DAYS, 7);
if (lastUpdated == updates.getTimeStamp(MODIFIED)) {
updates.clear(); //we don't need to update anything.
} else if (DateUtil.withinDateRange(lastUpdated, now.getTime(), days)) {
for (NvdCveInfo entry : updates) {
if (MODIFIED.equals(entry.getId())) {
entry.setNeedsUpdate(true);
} else {
entry.setNeedsUpdate(false);
}
}
} else { //we figure out which of the several XML files need to be downloaded.
for (NvdCveInfo entry : updates) {
if (MODIFIED.equals(entry.getId())) {
entry.setNeedsUpdate(true);
} else {
long currentTimestamp = 0;
try {
currentTimestamp = Long.parseLong(getProperties().getProperty(DatabaseProperties.LAST_UPDATED_BASE
+ entry.getId(), "0"));
} catch (NumberFormatException ex) {
LOGGER.debug("Error parsing '{}' '{}' from nvdcve.lastupdated",
DatabaseProperties.LAST_UPDATED_BASE, entry.getId(), ex);
}
if (currentTimestamp == entry.getTimestamp()) {
entry.setNeedsUpdate(false);
}
}
}
}
} catch (NumberFormatException ex) {
LOGGER.warn("An invalid schema version or timestamp exists in the data.properties file.");
LOGGER.debug("", ex);
}
}
return updates;
}
/**
* Retrieves the timestamps from the NVD CVE meta data file.
*
* @return the timestamp from the currently published nvdcve downloads page
* @throws MalformedURLException thrown if the URL for the NVD CCE Meta data is incorrect.
* @throws DownloadFailedException thrown if there is an error downloading the nvd cve meta data file
* @throws InvalidDataException thrown if there is an exception parsing the timestamps
* @throws InvalidSettingException thrown if the settings are invalid
*/
private UpdateableNvdCve retrieveCurrentTimestampsFromWeb()
throws MalformedURLException, DownloadFailedException, InvalidDataException, InvalidSettingException {
final UpdateableNvdCve updates = new UpdateableNvdCve();
updates.add(MODIFIED, Settings.getString(Settings.KEYS.CVE_MODIFIED_20_URL),
Settings.getString(Settings.KEYS.CVE_MODIFIED_12_URL),
false);
final int start = Settings.getInt(Settings.KEYS.CVE_START_YEAR);
final int end = Calendar.getInstance().get(Calendar.YEAR);
final String baseUrl20 = Settings.getString(Settings.KEYS.CVE_SCHEMA_2_0);
final String baseUrl12 = Settings.getString(Settings.KEYS.CVE_SCHEMA_1_2);
for (int i = start; i <= end; i++) {
updates.add(Integer.toString(i), String.format(baseUrl20, i),
String.format(baseUrl12, i),
true);
}
return updates;
}
}

318
dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/StandardUpdate.java
View File

@@ -1,318 +0,0 @@
/*
* This file is part of dependency-check-core.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
* Copyright (c) 2012 Jeremy Long. All Rights Reserved.
*/
package org.owasp.dependencycheck.data.update;
import java.net.MalformedURLException;
import java.util.Calendar;
import java.util.Date;
import java.util.HashSet;
import java.util.Set;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import org.owasp.dependencycheck.data.nvdcve.CveDB;
import org.owasp.dependencycheck.data.nvdcve.DatabaseException;
import org.owasp.dependencycheck.data.nvdcve.DatabaseProperties;
import static org.owasp.dependencycheck.data.nvdcve.DatabaseProperties.MODIFIED;
import org.owasp.dependencycheck.data.update.exception.InvalidDataException;
import org.owasp.dependencycheck.data.update.exception.UpdateException;
import org.owasp.dependencycheck.data.update.task.DownloadTask;
import org.owasp.dependencycheck.data.update.task.ProcessTask;
import org.owasp.dependencycheck.utils.DateUtil;
import org.owasp.dependencycheck.utils.DownloadFailedException;
import org.owasp.dependencycheck.utils.InvalidSettingException;
import org.owasp.dependencycheck.utils.Settings;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* Class responsible for updating the NVDCVE data store.
*
* @author Jeremy Long
*/
public class StandardUpdate {
/**
* Static logger.
*/
private static final Logger LOGGER = LoggerFactory.getLogger(StandardUpdate.class);
/**
* The max thread pool size to use when downloading files.
*/
public static final int MAX_THREAD_POOL_SIZE = Settings.getInt(Settings.KEYS.MAX_DOWNLOAD_THREAD_POOL_SIZE, 3);
/**
* Information about the timestamps and URLs for data that needs to be updated.
*/
private DatabaseProperties properties;
/**
* A collection of updateable NVD CVE items.
*/
private UpdateableNvdCve updateable;
/**
* Reference to the Cve Database.
*/
private CveDB cveDB = null;
/**
* Gets whether or not an update is needed.
*
* @return true or false depending on whether an update is needed
*/
public boolean isUpdateNeeded() {
return updateable.isUpdateNeeded();
}
/**
* Constructs a new Standard Update Task.
*
* @throws MalformedURLException thrown if a configured URL is malformed
* @throws DownloadFailedException thrown if a timestamp cannot be checked on a configured URL
* @throws UpdateException thrown if there is an exception generating the update task
*/
public StandardUpdate() throws MalformedURLException, DownloadFailedException, UpdateException {
openDataStores();
properties = cveDB.getDatabaseProperties();
updateable = updatesNeeded();
}
/**
* <p>
* Downloads the latest NVD CVE XML file from the web and imports it into the current CVE Database.</p>
*
* @throws UpdateException is thrown if there is an error updating the database
*/
public void update() throws UpdateException {
int maxUpdates = 0;
try {
for (NvdCveInfo cve : updateable) {
if (cve.getNeedsUpdate()) {
maxUpdates += 1;
}
}
if (maxUpdates <= 0) {
return;
}
if (maxUpdates > 3) {
LOGGER.info(
"NVD CVE requires several updates; this could take a couple of minutes.");
}
if (maxUpdates > 0) {
openDataStores();
}
final int poolSize = (MAX_THREAD_POOL_SIZE < maxUpdates) ? MAX_THREAD_POOL_SIZE : maxUpdates;
final ExecutorService downloadExecutors = Executors.newFixedThreadPool(poolSize);
final ExecutorService processExecutor = Executors.newSingleThreadExecutor();
final Set<Future<Future<ProcessTask>>> downloadFutures = new HashSet<Future<Future<ProcessTask>>>(maxUpdates);
for (NvdCveInfo cve : updateable) {
if (cve.getNeedsUpdate()) {
final DownloadTask call = new DownloadTask(cve, processExecutor, cveDB, Settings.getInstance());
downloadFutures.add(downloadExecutors.submit(call));
}
}
downloadExecutors.shutdown();
//next, move the future future processTasks to just future processTasks
final Set<Future<ProcessTask>> processFutures = new HashSet<Future<ProcessTask>>(maxUpdates);
for (Future<Future<ProcessTask>> future : downloadFutures) {
Future<ProcessTask> task = null;
try {
task = future.get();
} catch (InterruptedException ex) {
downloadExecutors.shutdownNow();
processExecutor.shutdownNow();
LOGGER.debug("Thread was interrupted during download", ex);
throw new UpdateException("The download was interrupted", ex);
} catch (ExecutionException ex) {
downloadExecutors.shutdownNow();
processExecutor.shutdownNow();
LOGGER.debug("Thread was interrupted during download execution", ex);
throw new UpdateException("The execution of the download was interrupted", ex);
}
if (task == null) {
downloadExecutors.shutdownNow();
processExecutor.shutdownNow();
LOGGER.debug("Thread was interrupted during download");
throw new UpdateException("The download was interrupted; unable to complete the update");
} else {
processFutures.add(task);
}
}
for (Future<ProcessTask> future : processFutures) {
try {
final ProcessTask task = future.get();
if (task.getException() != null) {
throw task.getException();
}
} catch (InterruptedException ex) {
processExecutor.shutdownNow();
LOGGER.debug("Thread was interrupted during processing", ex);
throw new UpdateException(ex);
} catch (ExecutionException ex) {
processExecutor.shutdownNow();
LOGGER.debug("Execution Exception during process", ex);
throw new UpdateException(ex);
} finally {
processExecutor.shutdown();
}
}
if (maxUpdates >= 1) { //ensure the modified file date gets written (we may not have actually updated it)
properties.save(updateable.get(MODIFIED));
LOGGER.info("Begin database maintenance.");
cveDB.cleanupDatabase();
LOGGER.info("End database maintenance.");
}
} finally {
closeDataStores();
}
}
/**
* Determines if the index needs to be updated. This is done by fetching the NVD CVE meta data and checking the last update
* date. If the data needs to be refreshed this method will return the NvdCveUrl for the files that need to be updated.
*
* @return the collection of files that need to be updated
* @throws MalformedURLException is thrown if the URL for the NVD CVE Meta data is incorrect
* @throws DownloadFailedException is thrown if there is an error. downloading the NVD CVE download data file
* @throws UpdateException Is thrown if there is an issue with the last updated properties file
*/
protected final UpdateableNvdCve updatesNeeded() throws MalformedURLException, DownloadFailedException, UpdateException {
UpdateableNvdCve updates = null;
try {
updates = retrieveCurrentTimestampsFromWeb();
} catch (InvalidDataException ex) {
final String msg = "Unable to retrieve valid timestamp from nvd cve downloads page";
LOGGER.debug(msg, ex);
throw new DownloadFailedException(msg, ex);
} catch (InvalidSettingException ex) {
LOGGER.debug("Invalid setting found when retrieving timestamps", ex);
throw new DownloadFailedException("Invalid settings", ex);
}
if (updates == null) {
throw new DownloadFailedException("Unable to retrieve the timestamps of the currently published NVD CVE data");
}
if (!properties.isEmpty()) {
try {
final long lastUpdated = Long.parseLong(properties.getProperty(DatabaseProperties.LAST_UPDATED, "0"));
final Date now = new Date();
final int days = Settings.getInt(Settings.KEYS.CVE_MODIFIED_VALID_FOR_DAYS, 7);
if (lastUpdated == updates.getTimeStamp(MODIFIED)) {
updates.clear(); //we don't need to update anything.
} else if (DateUtil.withinDateRange(lastUpdated, now.getTime(), days)) {
for (NvdCveInfo entry : updates) {
if (MODIFIED.equals(entry.getId())) {
entry.setNeedsUpdate(true);
} else {
entry.setNeedsUpdate(false);
}
}
} else { //we figure out which of the several XML files need to be downloaded.
for (NvdCveInfo entry : updates) {
if (MODIFIED.equals(entry.getId())) {
entry.setNeedsUpdate(true);
} else {
long currentTimestamp = 0;
try {
currentTimestamp = Long.parseLong(properties.getProperty(DatabaseProperties.LAST_UPDATED_BASE + entry.getId(), "0"));
} catch (NumberFormatException ex) {
LOGGER.debug("Error parsing '{}' '{}' from nvdcve.lastupdated",
DatabaseProperties.LAST_UPDATED_BASE, entry.getId(), ex);
}
if (currentTimestamp == entry.getTimestamp()) {
entry.setNeedsUpdate(false);
}
}
}
}
} catch (NumberFormatException ex) {
LOGGER.warn("An invalid schema version or timestamp exists in the data.properties file.");
LOGGER.debug("", ex);
}
}
return updates;
}
/**
* Retrieves the timestamps from the NVD CVE meta data file.
*
* @return the timestamp from the currently published nvdcve downloads page
* @throws MalformedURLException thrown if the URL for the NVD CCE Meta data is incorrect.
* @throws DownloadFailedException thrown if there is an error downloading the nvd cve meta data file
* @throws InvalidDataException thrown if there is an exception parsing the timestamps
* @throws InvalidSettingException thrown if the settings are invalid
*/
private UpdateableNvdCve retrieveCurrentTimestampsFromWeb()
throws MalformedURLException, DownloadFailedException, InvalidDataException, InvalidSettingException {
final UpdateableNvdCve updates = new UpdateableNvdCve();
updates.add(MODIFIED, Settings.getString(Settings.KEYS.CVE_MODIFIED_20_URL),
Settings.getString(Settings.KEYS.CVE_MODIFIED_12_URL),
false);
final int start = Settings.getInt(Settings.KEYS.CVE_START_YEAR);
final int end = Calendar.getInstance().get(Calendar.YEAR);
final String baseUrl20 = Settings.getString(Settings.KEYS.CVE_SCHEMA_2_0);
final String baseUrl12 = Settings.getString(Settings.KEYS.CVE_SCHEMA_1_2);
for (int i = start; i <= end; i++) {
updates.add(Integer.toString(i), String.format(baseUrl20, i),
String.format(baseUrl12, i),
true);
}
return updates;
}
/**
* Closes the CVE and CPE data stores.
*/
protected void closeDataStores() {
if (cveDB != null) {
try {
cveDB.close();
} catch (Throwable ignore) {
LOGGER.trace("Error closing the cveDB", ignore);
}
}
}
/**
* Opens the CVE and CPE data stores.
*
* @throws UpdateException thrown if a data store cannot be opened
*/
protected final void openDataStores() throws UpdateException {
if (cveDB != null) {
return;
}
try {
cveDB = new CveDB();
cveDB.open();
} catch (DatabaseException ex) {
closeDataStores();
LOGGER.debug("Database Exception opening databases", ex);
throw new UpdateException("Error updating the CPE/CVE data, please see the log file for more details.");
}
}
}

364
dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/cpe/CPEHandler.java Normal file
View File

@@ -0,0 +1,364 @@
/*
* This file is part of dependency-check-core.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
* Copyright (c) 2015 Jeremy Long. All Rights Reserved.
*/
package org.owasp.dependencycheck.data.update.cpe;
import java.io.UnsupportedEncodingException;
import java.util.ArrayList;
import java.util.List;
import org.owasp.dependencycheck.data.update.NvdCveUpdater;
import org.owasp.dependencycheck.data.update.exception.InvalidDataException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.xml.sax.Attributes;
import org.xml.sax.SAXException;
import org.xml.sax.helpers.DefaultHandler;
/**
* A SAX Handler that will parse the CPE XML and load it into the databse.
*
* @author Jeremy Long
*/
public class CPEHandler extends DefaultHandler {
/**
* The current CPE schema.
*/
private static final String CURRENT_SCHEMA_VERSION = "2.3";
/**
* The text content of the node being processed. This can be used during the end element event.
*/
private StringBuilder nodeText = null;
/**
* A reference to the current element.
*/
private Element current = new Element();
/**
* The logger.
*/
private static final Logger LOGGER = LoggerFactory.getLogger(NvdCveUpdater.class);
/**
* The list of CPE values.
*/
private List<Cpe> data = new ArrayList<Cpe>();
/**
* Returns the list of CPE values.
*
* @return the list of CPE values
*/
public List<Cpe> getData() {
return data;
}
/**
* Handles the start element event.
*
* @param uri the elements uri
* @param localName the local name
* @param qName the qualified name
* @param attributes the attributes
* @throws SAXException thrown if there is an exception processing the element
*/
@Override
public void startElement(String uri, String localName, String qName, Attributes attributes) throws SAXException {
nodeText = null;
current.setNode(qName);
if (current.isCpeItemNode()) {
final String temp = attributes.getValue("deprecated");
final String value = attributes.getValue("name");
final boolean delete = "true".equalsIgnoreCase(temp);
if (!delete && value.startsWith("cpe:/a:") && value.length() > 7) {
try {
final Cpe cpe = new Cpe(value);
data.add(cpe);
} catch (UnsupportedEncodingException ex) {
LOGGER.debug("Unable to parse the CPE", ex);
} catch (InvalidDataException ex) {
LOGGER.debug("CPE is not the correct format", ex);
}
}
} else if (current.isSchemaVersionNode()) {
nodeText = new StringBuilder(3);
}
// } else if (current.isTitleNode()) {
// //do nothing
// } else if (current.isMetaNode()) {
// //do nothing
// } else if (current.isTimestampNode()) {
// //do nothing
// } else if (current.isCpeListNode()) {
// //do nothing
// } else if (current.isNotesNode()) {
// //do nothing
// } else if (current.isNoteNode()) {
// //do nothing
// } else if (current.isCheckNode()) {
// //do nothing
// } else if (current.isGeneratorNode()) {
// //do nothing
// } else if (current.isProductNameNode()) {
// //do nothing
// } else if (current.isProductVersionNode()) {
// //do nothing
}
/**
* Reads the characters in the current node.
*
* @param ch the char array
* @param start the start position of the data read
* @param length the length of the data read
* @throws SAXException thrown if there is an exception processing the characters
*/
@Override
public void characters(char[] ch, int start, int length) throws SAXException {
if (nodeText != null) {
nodeText.append(ch, start, length);
}
}
/**
* Handles the end element event. Stores the CPE data in the Cve Database if the cpe item node is ending.
*
* @param uri the element's uri
* @param localName the local name
* @param qName the qualified name
* @throws SAXException thrown if there is an exception processing the element
*/
@Override
public void endElement(String uri, String localName, String qName) throws SAXException {
current.setNode(qName);
if (current.isSchemaVersionNode() && !CURRENT_SCHEMA_VERSION.equals(nodeText.toString())) {
throw new SAXException("ERROR: Unexpecgted CPE Schema Version, expected: "
+ CURRENT_SCHEMA_VERSION + ", file is: " + nodeText);
}
// } else if (current.isCpeItemNode()) {
// //do nothing
// } else if (current.isTitleNode()) {
// //do nothing
// } else if (current.isCpeListNode()) {
// //do nothing
// } else if (current.isMetaNode()) {
// //do nothing
// } else if (current.isNotesNode()) {
// //do nothing
// } else if (current.isNoteNode()) {
// //do nothing
// } else if (current.isCheckNode()) {
// //do nothing
// } else if (current.isGeneratorNode()) {
// //do nothing
// } else if (current.isProductNameNode()) {
// //do nothing
// } else if (current.isProductVersionNode()) {
// //do nothing
// else if (current.isTimestampNode()) {
// //do nothing
// } else {
// throw new SAXException("ERROR STATE: Unexpected qName '" + qName + "'");
// }
}
// <editor-fold defaultstate="collapsed" desc="The Element Class that maintains state information about the current node">
/**
* A simple class to maintain information about the current element while parsing the CPE XML.
*/
protected class Element {
/**
* A node type in the CPE Schema 2.2
*/
public static final String CPE_LIST = "cpe-list";
/**
* A node type in the CPE Schema 2.2
*/
public static final String CPE_ITEM = "cpe-item";
/**
* A node type in the CPE Schema 2.2
*/
public static final String TITLE = "title";
/**
* A node type in the CPE Schema 2.2
*/
public static final String NOTES = "notes";
/**
* A node type in the CPE Schema 2.2
*/
public static final String NOTE = "note";
/**
* A node type in the CPE Schema 2.2
*/
public static final String CHECK = "check";
/**
* A node type in the CPE Schema 2.2
*/
public static final String META = "meta:item-metadata";
/**
* A node type in the CPE Schema 2.2
*/
public static final String GENERATOR = "generator";
/**
* A node type in the CPE Schema 2.2
*/
public static final String PRODUCT_NAME = "product_name";
/**
* A node type in the CPE Schema 2.2
*/
public static final String PRODUCT_VERSION = "product_version";
/**
* A node type in the CPE Schema 2.2
*/
public static final String SCHEMA_VERSION = "schema_version";
/**
* A node type in the CPE Schema 2.2
*/
public static final String TIMESTAMP = "timestamp";
/**
* A reference to the current node.
*/
private String node = null;
/**
* Gets the value of node
*
* @return the value of node
*/
public String getNode() {
return this.node;
}
/**
* Sets the value of node
*
* @param node new value of node
*/
public void setNode(String node) {
this.node = node;
}
/**
* Checks if the handler is at the CPE_LIST node
*
* @return true or false
*/
public boolean isCpeListNode() {
return CPE_LIST.equals(node);
}
/**
* Checks if the handler is at the CPE_ITEM node
*
* @return true or false
*/
public boolean isCpeItemNode() {
return CPE_ITEM.equals(node);
}
/**
* Checks if the handler is at the TITLE node
*
* @return true or false
*/
public boolean isTitleNode() {
return TITLE.equals(node);
}
/**
* Checks if the handler is at the NOTES node
*
* @return true or false
*/
public boolean isNotesNode() {
return NOTES.equals(node);
}
/**
* Checks if the handler is at the NOTE node
*
* @return true or false
*/
public boolean isNoteNode() {
return NOTE.equals(node);
}
/**
* Checks if the handler is at the CHECK node
*
* @return true or false
*/
public boolean isCheckNode() {
return CHECK.equals(node);
}
/**
* Checks if the handler is at the META node
*
* @return true or false
*/
public boolean isMetaNode() {
return META.equals(node);
}
/**
* Checks if the handler is at the GENERATOR node
*
* @return true or false
*/
public boolean isGeneratorNode() {
return GENERATOR.equals(node);
}
/**
* Checks if the handler is at the PRODUCT_NAME node
*
* @return true or false
*/
public boolean isProductNameNode() {
return PRODUCT_NAME.equals(node);
}
/**
* Checks if the handler is at the PRODUCT_VERSION node
*
* @return true or false
*/
public boolean isProductVersionNode() {
return PRODUCT_VERSION.equals(node);
}
/**
* Checks if the handler is at the SCHEMA_VERSION node
*
* @return true or false
*/
public boolean isSchemaVersionNode() {
return SCHEMA_VERSION.equals(node);
}
/**
* Checks if the handler is at the TIMESTAMP node
*
* @return true or false
*/
public boolean isTimestampNode() {
return TIMESTAMP.equals(node);
}
}
// </editor-fold>
}

125
dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/cpe/Cpe.java Normal file
View File

@@ -0,0 +1,125 @@
/*
* This file is part of dependency-check-core.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
* Copyright (c) 2015 Jeremy Long. All Rights Reserved.
*/
package org.owasp.dependencycheck.data.update.cpe;
import java.io.UnsupportedEncodingException;
import java.net.URLDecoder;
import org.owasp.dependencycheck.data.update.exception.InvalidDataException;
/**
*
* @author Jeremy Long
*/
public class Cpe {
/**
* Constructs a new Cpe Object by parsing the vendor and product from the CPE identifier value.
*
* @param value the cpe identifier (cpe:/a:vendor:product:version:....)
* @throws UnsupportedEncodingException thrown if UTF-8 is not supported
* @throws InvalidDataException thrown if the CPE provided is not the correct format
*/
public Cpe(String value) throws UnsupportedEncodingException, InvalidDataException {
this.value = value;
final String[] data = value.substring(7).split(":");
if (data.length >= 2) {
vendor = URLDecoder.decode(data[0].replace("+", "%2B"), "UTF-8");
product = URLDecoder.decode(data[1].replace("+", "%2B"), "UTF-8");
} else {
throw new InvalidDataException(String.format("CPE has an invalid format: %s", value));
}
}
/**
* The CPE identifier string (cpe:/a:vendor:product:version).
*/
private String value;
/**
* Get the value of value.
*
* @return the value of value
*/
public String getValue() {
return value;
}
/**
* Set the value of value.
*
* @param value new value of value
*/
public void setValue(String value) {
this.value = value;
}
/**
* The vendor portion of the identifier.
*/
private String vendor;
/**
* Get the value of vendor.
*
* @return the value of vendor
*/
public String getVendor() {
return vendor;
}
/**
* Set the value of vendor.
*
* @param vendor new value of vendor
*/
public void setVendor(String vendor) {
this.vendor = vendor;
}
/**
* The product portion of the identifier.
*/
private String product;
/**
* Get the value of product.
*
* @return the value of product
*/
public String getProduct() {
return product;
}
/**
* Set the value of product.
*
* @param product new value of product
*/
public void setProduct(String product) {
this.product = product;
}
/**
* Returns the full CPE identifier.
*
* @return the full CPE identifier
*/
@Override
public String toString() {
return value;
}
}

7
dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/cpe/package-info.java Normal file
View File

@@ -0,0 +1,7 @@
/**
* Contains classes used to parse the CPE XML file from NIST.<br/><br/>
*
* These classes are not used as they add no value over the existing CPE data contained within the CVE data from the NVD. However,
* we may consider pulling the more descriptive data from the CPE data in the future.
*/
package org.owasp.dependencycheck.data.update.cpe;

10
dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/task/DownloadTask.java → dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/nvd/DownloadTask.java
View File

@@ -15,7 +15,7 @@
*
* Copyright (c) 2013 Jeremy Long. All Rights Reserved.
*/
package org.owasp.dependencycheck.data.update.task;
package org.owasp.dependencycheck.data.update.nvd;
import java.io.File;
import java.io.FileInputStream;
@@ -29,7 +29,6 @@ import java.util.concurrent.Future;
import java.util.zip.GZIPInputStream;
import org.apache.commons.io.FileUtils;
import org.owasp.dependencycheck.data.nvdcve.CveDB;
import org.owasp.dependencycheck.data.update.NvdCveInfo;
import org.owasp.dependencycheck.data.update.exception.UpdateException;
import org.owasp.dependencycheck.utils.DownloadFailedException;
import org.owasp.dependencycheck.utils.Downloader;
@@ -55,8 +54,8 @@ public class DownloadTask implements Callable<Future<ProcessTask>> {
* @param nvdCveInfo the NVD CVE info
* @param processor the processor service to submit the downloaded files to
* @param cveDB the CVE DB to use to store the vulnerability data
* @param settings a reference to the global settings object; this is necessary so that when the thread is started
* the dependencies have a correct reference to the global settings.
* @param settings a reference to the global settings object; this is necessary so that when the thread is started the
* dependencies have a correct reference to the global settings.
* @throws UpdateException thrown if temporary files could not be created
*/
public DownloadTask(NvdCveInfo nvdCveInfo, ExecutorService processor, CveDB cveDB, Settings settings) throws UpdateException {
@@ -248,8 +247,7 @@ public class DownloadTask implements Callable<Future<ProcessTask>> {
}
/**
* Extracts the file contained in a gzip archive. The extracted file is placed in the exact same path as the file
* specified.
* Extracts the file contained in a gzip archive. The extracted file is placed in the exact same path as the file specified.
*
* @param file the archive file
* @throws FileNotFoundException thrown if the file does not exist

2
dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/xml/NvdCve12Handler.java → dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/nvd/NvdCve12Handler.java
View File

@@ -15,7 +15,7 @@
*
* Copyright (c) 2012 Jeremy Long. All Rights Reserved.
*/
package org.owasp.dependencycheck.data.update.xml;
package org.owasp.dependencycheck.data.update.nvd;
import java.util.ArrayList;
import java.util.HashMap;

2
dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/xml/NvdCve20Handler.java → dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/nvd/NvdCve20Handler.java
View File

@@ -15,7 +15,7 @@
*
* Copyright (c) 2012 Jeremy Long. All Rights Reserved.
*/
package org.owasp.dependencycheck.data.update.xml;
package org.owasp.dependencycheck.data.update.nvd;
import java.io.IOException;
import java.util.List;

2
dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/NvdCveInfo.java → dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/nvd/NvdCveInfo.java
View File

@@ -15,7 +15,7 @@
*
* Copyright (c) 2013 Jeremy Long. All Rights Reserved.
*/
package org.owasp.dependencycheck.data.update;
package org.owasp.dependencycheck.data.update.nvd;
/**
* A pojo that contains the Url and timestamp of the current NvdCve XML files.

12
dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/task/ProcessTask.java → dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/nvd/ProcessTask.java
View File

@@ -15,7 +15,7 @@
*
* Copyright (c) 2013 Jeremy Long. All Rights Reserved.
*/
package org.owasp.dependencycheck.data.update.task;
package org.owasp.dependencycheck.data.update.nvd;
import java.io.File;
import java.io.FileNotFoundException;
@@ -31,8 +31,6 @@ import org.owasp.dependencycheck.data.nvdcve.CveDB;
import org.owasp.dependencycheck.data.nvdcve.DatabaseException;
import org.owasp.dependencycheck.data.nvdcve.DatabaseProperties;
import org.owasp.dependencycheck.data.update.exception.UpdateException;
import org.owasp.dependencycheck.data.update.xml.NvdCve12Handler;
import org.owasp.dependencycheck.data.update.xml.NvdCve20Handler;
import org.owasp.dependencycheck.dependency.VulnerableSoftware;
import org.owasp.dependencycheck.utils.Settings;
import org.slf4j.Logger;
@@ -94,8 +92,8 @@ public class ProcessTask implements Callable<ProcessTask> {
*
* @param cveDB the data store object
* @param filePair the download task that contains the URL references to download
* @param settings a reference to the global settings object; this is necessary so that when the thread is started
* the dependencies have a correct reference to the global settings.
* @param settings a reference to the global settings object; this is necessary so that when the thread is started the
* dependencies have a correct reference to the global settings.
*/
public ProcessTask(final CveDB cveDB, final DownloadTask filePair, Settings settings) {
this.cveDB = cveDB;
@@ -108,8 +106,8 @@ public class ProcessTask implements Callable<ProcessTask> {
* Implements the callable interface.
*
* @return this object
* @throws Exception thrown if there is an exception; note that any UpdateExceptions are simply added to the tasks
* exception collection
* @throws Exception thrown if there is an exception; note that any UpdateExceptions are simply added to the tasks exception
* collection
*/
@Override
public ProcessTask call() throws Exception {

14
dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/UpdateableNvdCve.java → dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/nvd/UpdateableNvdCve.java
View File

@@ -15,7 +15,7 @@
*
* Copyright (c) 2012 Jeremy Long. All Rights Reserved.
*/
package org.owasp.dependencycheck.data.update;
package org.owasp.dependencycheck.data.update.nvd;
import java.net.MalformedURLException;
import java.net.URL;
@@ -27,8 +27,8 @@ import org.owasp.dependencycheck.utils.DownloadFailedException;
import org.owasp.dependencycheck.utils.Downloader;
/**
* Contains a collection of updateable NvdCveInfo objects. This is used to determine which files need to be downloaded
* and processed.
* Contains a collection of updateable NvdCveInfo objects. This is used to determine which files need to be downloaded and
* processed.
*
* @author Jeremy Long
*/
@@ -67,8 +67,7 @@ public class UpdateableNvdCve implements java.lang.Iterable<NvdCveInfo>, Iterato
*
* @param id the key for the item to be added
* @param url the URL to download the item
* @param oldUrl the URL for the old version of the item (the NVD CVE old schema still contains useful data we
* need).
* @param oldUrl the URL for the old version of the item (the NVD CVE old schema still contains useful data we need).
* @throws MalformedURLException thrown if the URL provided is invalid
* @throws DownloadFailedException thrown if the download fails.
*/
@@ -81,8 +80,7 @@ public class UpdateableNvdCve implements java.lang.Iterable<NvdCveInfo>, Iterato
*
* @param id the key for the item to be added
* @param url the URL to download the item
* @param oldUrl the URL for the old version of the item (the NVD CVE old schema still contains useful data we
* need).
* @param oldUrl the URL for the old version of the item (the NVD CVE old schema still contains useful data we need).
* @param needsUpdate whether or not the data needs to be updated
* @throws MalformedURLException thrown if the URL provided is invalid
* @throws DownloadFailedException thrown if the download fails.
@@ -175,7 +173,7 @@ public class UpdateableNvdCve implements java.lang.Iterable<NvdCveInfo>, Iterato
* @param key the key to lookup the return value
* @return the NvdCveInfo object stored using the specified key
*/
NvdCveInfo get(String key) {
public NvdCveInfo get(String key) {
return collection.get(key);
}

4
dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/nvd/package-info.java Normal file
View File

@@ -0,0 +1,4 @@
/**
* Contains classes used to download, parse, and load the NVD CVE data from NIST into the local database.<br/><br/>
*/
package org.owasp.dependencycheck.data.update.nvd;

4
dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/task/package-info.java
View File

@@ -1,4 +0,0 @@
/**
* A collection of callable/runnable tasks used to speed up the update process.
*/
package org.owasp.dependencycheck.data.update.task;

8
dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/xml/package-info.java
View File

@@ -1,8 +0,0 @@
/**
* Contains classes used to parse the NVD CVE XML file.<br/><br/>
*
* The basic use is that the Importer is called to import an NVD CVE file. The Importer instantiates an Indexer object (which
* extends Index). The Indexer creates a partial-unmarshalling SAX parser (implemented in the NvdCveXmlFilter) that extracts
* VulnerabilityTypes (aka Entry) from the NVD CVE data file and stores these into a Lucene Index.
*/
package org.owasp.dependencycheck.data.update.xml;

31
dependency-check-core/src/main/java/org/owasp/dependencycheck/dependency/VulnerableSoftware.java
View File

@@ -57,8 +57,7 @@ public class VulnerableSoftware extends IndexEntry implements Serializable, Comp
/**
* <p>
* Parses a name attribute value, from the cpe.xml, into its corresponding parts: vendor, product, version,
* revision.</p>
* Parses a name attribute value, from the cpe.xml, into its corresponding parts: vendor, product, version, update.</p>
* <p>
* Example:</p>
* <code>&nbsp;&nbsp;&nbsp;cpe:/a:apache:struts:1.1:rc2</code>
@@ -85,7 +84,7 @@ public class VulnerableSoftware extends IndexEntry implements Serializable, Comp
version = urlDecode(data[2]);
}
if (data.length >= 4) {
revision = urlDecode(data[3]);
update = urlDecode(data[3]);
}
if (data.length >= 5) {
edition = urlDecode(data[4]);
@@ -297,26 +296,26 @@ public class VulnerableSoftware extends IndexEntry implements Serializable, Comp
this.version = version;
}
/**
* The product revision version.
* The product update version.
*/
private String revision;
private String update;
/**
* Get the value of revision.
* Get the value of update.
*
* @return the value of revision
* @return the value of update
*/
public String getRevision() {
return revision;
public String getUpdate() {
return update;
}
/**
* Set the value of revision.
* Set the value of update.
*
* @param revision new value of revision
* @param update new value of update
*/
public void setRevision(String revision) {
this.revision = revision;
public void setUpdate(String update) {
this.update = update;
}
/**
* The product edition.
@@ -363,11 +362,9 @@ public class VulnerableSoftware extends IndexEntry implements Serializable, Comp
}
/**
* Call {@link java.net.URLDecoder#decode(String)} to URL decode using the
* default encoding.
* Call {@link java.net.URLDecoder#decode(String)} to URL decode using the default encoding.
*
* @param text
* www-form-encoded URL to decode
* @param text www-form-encoded URL to decode
* @return the newly decoded String
*/
@SuppressWarnings("deprecation")

2
dependency-check-core/src/main/java/org/owasp/dependencycheck/utils/DCResources.java
View File

@@ -22,7 +22,7 @@ import ch.qos.cal10n.Locale;
import ch.qos.cal10n.LocaleData;
/**
* Created by colezlaw on 6/13/15.
* @author colezlaw
*/
@BaseName("dependencycheck-resources")
@LocaleData(defaultCharset = "UTF-8",

8
dependency-check-core/src/main/java/org/owasp/dependencycheck/utils/ExtractionUtil.java
View File

@@ -17,8 +17,6 @@
*/
package org.owasp.dependencycheck.utils;
import static org.owasp.dependencycheck.utils.FileUtils.getFileExtension;
import java.io.BufferedInputStream;
import java.io.BufferedOutputStream;
import java.io.Closeable;
@@ -109,8 +107,7 @@ public final class ExtractionUtil {
}
} else {
final File file = new File(extractTo, entry.getName());
final String ext = getFileExtension(file.getName());
if (engine == null || engine.supportsExtension(ext)) {
if (engine == null || engine.accept(file)) {
BufferedOutputStream bos = null;
FileOutputStream fos;
try {
@@ -227,7 +224,7 @@ public final class ExtractionUtil {
final File file = new File(destination, entry.getName());
if (filter.accept(file.getParentFile(), file.getName())) {
LOGGER.debug("Extracting '{}'",
file.getPath());
file.getPath());
BufferedOutputStream bos = null;
FileOutputStream fos = null;
try {
@@ -303,5 +300,4 @@ public final class ExtractionUtil {
}
}
}
}

138
dependency-check-core/src/main/java/org/owasp/dependencycheck/utils/FileFilterBuilder.java Normal file
View File

@@ -0,0 +1,138 @@
/*
* This file is part of dependency-check-core.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
* Copyright (c) 2015 Institute for Defense Analyses. All Rights Reserved.
*/
package org.owasp.dependencycheck.utils;
import org.apache.commons.io.IOCase;
import org.apache.commons.io.filefilter.IOFileFilter;
import org.apache.commons.io.filefilter.NameFileFilter;
import org.apache.commons.io.filefilter.OrFileFilter;
import org.apache.commons.io.filefilter.SuffixFileFilter;
import java.io.FileFilter;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
/**
* <p>
* Utility class for building useful {@link FileFilter} instances for
* {@link org.owasp.dependencycheck.analyzer.AbstractFileTypeAnalyzer} implementations. The built filter uses {@link OrFileFilter}
* to logically OR the given filter conditions. Example usage:</p>
*
* <pre>
* FileFilter filter = FileFilterBuilder.newInstance().addExtensions("jar", "war").build();
* </pre>
*
* @author Dale Visser <dvisser@ida.org>
* @see <a href="https://en.wikipedia.org/wiki/Builder_pattern">Builder pattern</a>
*/
public class FileFilterBuilder {
/**
* A set of filenames to filter.
*/
private final Set<String> filenames = new HashSet<String>();
/**
* A set of extensions to filter.
*/
private final Set<String> extensions = new HashSet<String>();
/**
* An array list of file filters.
*/
private final List<IOFileFilter> fileFilters = new ArrayList<IOFileFilter>();
/**
* Create a new instance and return it. This method is for convenience in using the builder pattern within a single statement.
*
* @return a new builder instance
*/
public static FileFilterBuilder newInstance() {
return new FileFilterBuilder();
}
/**
* Add to the set of filenames to accept for analysis. Case-sensitivity is assumed.
*
* @param names one or more filenames to accept for analysis
* @return this builder
*/
public FileFilterBuilder addFilenames(String... names) {
filenames.addAll(Arrays.asList(names));
return this;
}
/**
* Add to the set of file extensions to accept for analysis. Case-insensitivity is assumed.
*
* @param extensions one or more file extensions to accept for analysis
* @return this builder
*/
public FileFilterBuilder addExtensions(String... extensions) {
return this.addExtensions(Arrays.asList(extensions));
}
/**
* Add to the set of file extensions to accept for analysis. Case-insensitivity is assumed.
*
* @param extensions one or more file extensions to accept for analysis
* @return this builder
*/
public FileFilterBuilder addExtensions(Iterable<String> extensions) {
for (String extension : extensions) {
// Ultimately, SuffixFileFilter will be used, and the "." needs to be explicit.
this.extensions.add(extension.startsWith(".") ? extension : "." + extension);
}
return this;
}
/**
* Add to a list of {@link IOFileFilter} instances to consult for whether to accept a file for analysis.
*
* @param filters one or more file filters to consult for whether to accept for analysis
* @return this builder
*/
public FileFilterBuilder addFileFilters(IOFileFilter... filters) {
fileFilters.addAll(Arrays.asList(filters));
return this;
}
/**
* Builds the filter and returns it.
*
* @return a filter that is the logical OR of all the conditions provided by the add... methods
* @throws IllegalStateException if no add... method has been called with one or more arguments
*/
public FileFilter build() {
if (filenames.isEmpty() && extensions.isEmpty() && fileFilters.isEmpty()) {
throw new IllegalStateException("May only be invoked after at least one add... method has been invoked.");
}
final OrFileFilter filter = new OrFileFilter();
if (!filenames.isEmpty()) {
filter.addFileFilter(new NameFileFilter(new ArrayList<String>(filenames)));
}
if (!extensions.isEmpty()) {
filter.addFileFilter(new SuffixFileFilter(new ArrayList<String>(extensions), IOCase.INSENSITIVE));
}
for (IOFileFilter iof : fileFilters) {
filter.addFileFilter(iof);
}
return filter;
}
}

6
dependency-check-core/src/main/java/org/owasp/dependencycheck/xml/pom/PomUtils.java
View File

@@ -48,8 +48,7 @@ public final class PomUtils {
*
* @param file the pom.xml file
* @return returns a
* @throws AnalysisException is thrown if there is an exception extracting or parsing the POM
* {@link org.owasp.dependencycheck.jaxb.pom.generated.Model} object
* @throws AnalysisException is thrown if there is an exception extracting or parsing the POM {@link Model} object
*/
public static Model readPom(File file) throws AnalysisException {
Model model = null;
@@ -78,8 +77,7 @@ public final class PomUtils {
* @param path the path to the pom.xml file within the jar file
* @param jar the jar file to extract the pom from
* @return returns a
* @throws AnalysisException is thrown if there is an exception extracting or parsing the POM
* {@link org.owasp.dependencycheck.jaxb.pom.generated.Model} object
* @throws AnalysisException is thrown if there is an exception extracting or parsing the POM {@link Model} object
*/
public static Model readPom(String path, JarFile jar) throws AnalysisException {
final ZipEntry entry = jar.getEntry(path);

3
dependency-check-core/src/main/resources/data/dbStatements.properties
View File

@@ -34,3 +34,6 @@ SELECT_PROPERTY=SELECT id, value FROM properties WHERE id = ?
INSERT_PROPERTY=INSERT INTO properties (id, value) VALUES (?, ?)
UPDATE_PROPERTY=UPDATE properties SET value = ? WHERE id = ?
DELETE_PROPERTY=DELETE FROM properties WHERE id = ?
DELETE_UNUSED_DICT_CPE=DELETE FROM cpeEntry WHERE dictionaryEntry=true AND id NOT IN (SELECT cpeEntryId FROM software)
ADD_DICT_CPE=MERGE INTO cpeEntry (cpe, vendor, product, dictionaryEntry) KEY(cpe) VALUES(?,?,?,true)

7
dependency-check-core/src/main/resources/data/upgrade_2.9.sql Normal file
View File

@@ -0,0 +1,7 @@
--the following is not currently used.
--ALTER TABLE cpeEntry ADD COLUMN IF NOT EXISTS dictionaryEntry BOOLEAN;
--ALTER TABLE cpeEntry ALTER COLUMN dictionaryEntry SET DEFAULT FALSE;
--UPDATE cpeEntry SET dictionaryEntry=false;
--UPDATE Properties SET value='3.0' WHERE ID='version';

4
dependency-check-core/src/main/resources/dependencycheck.properties
View File

@@ -17,7 +17,7 @@ engine.version.url=http://jeremylong.github.io/DependencyCheck/current.txt
# below contains a %s then the data.directory will replace the %s.
data.directory=[JAR]/data
#if the filename has a %s it will be replaced with the current expected version
data.file_name=cve.%s.h2.db
data.file_name=dc.h2.db
data.version=2.9
data.connection_string=jdbc:h2:file:%s;FILE_LOCK=SERIALIZED;AUTOCOMMIT=ON;
#data.connection_string=jdbc:mysql://localhost:3306/dependencycheck
@@ -53,6 +53,8 @@ cve.url-1.2.base=https://nvd.nist.gov/download/nvdcve-%d.xml.gz
cve.url-2.0.base=https://nvd.nist.gov/feeds/xml/cve/nvdcve-2.0-%d.xml.gz
#cve.url-2.0.base=http://static.nvd.nist.gov/feeds/xml/cve/nvdcve-2.0-%d.xml
cpe.validfordays=30
cpe.url=http://static.nvd.nist.gov/feeds/xml/cpe/dictionary/official-cpe-dictionary_v2.3.xml.gz
# file type analyzer settings:
analyzer.archive.enabled=true