Merge pull request #217 from dwvisser/python-github-pr

Add Analyzers for Python Distribution Files and Packages

Former-commit-id: 972a63525744bbbd510b4ae2d92028dc3079787a

dependency-check-core/src/main/java/org/owasp/dependencycheck/Engine.java

@@ -318,16 +318,17 @@ public class Engine {
             return null;
         }
         final String fileName = file.getName();
-        final String extension = FileUtils.getFileExtension(fileName);
+        String extension = FileUtils.getFileExtension(fileName);
+        if (null == extension) {
+            extension = fileName;
+        }
         Dependency dependency = null;
-        if (extension != null) {
-            if (supportsExtension(extension)) {
-                dependency = new Dependency(file);
-                dependencies.add(dependency);
+        if (supportsExtension(extension)) {
+            dependency = new Dependency(file);
+            if (extension == fileName){
+                dependency.setFileExtension(extension);
             }
-        } else {
-            final String msg = String.format("No file extension found on file '%s'. The file was not analyzed.", file.toString());
-            LOGGER.log(Level.FINE, msg);
+            dependencies.add(dependency);
         }
         return dependency;
     }

dependency-check-core/src/main/java/org/owasp/dependencycheck/analyzer/PythonDistributionAnalyzer.java

@@ -0,0 +1,344 @@
+/*
+ * This file is part of dependency-check-core.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * Copyright (c) 2015 Institute for Defense Analyses. All Rights Reserved.
+ */
+package org.owasp.dependencycheck.analyzer;
+
+import java.io.BufferedInputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.FilenameFilter;
+import java.net.MalformedURLException;
+import java.util.Set;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+import java.util.regex.Pattern;
+
+import javax.mail.MessagingException;
+import javax.mail.internet.InternetHeaders;
+
+import org.apache.commons.io.filefilter.NameFileFilter;
+import org.apache.commons.io.filefilter.SuffixFileFilter;
+import org.apache.commons.io.input.AutoCloseInputStream;
+import org.apache.commons.lang.StringUtils;
+import org.owasp.dependencycheck.Engine;
+import org.owasp.dependencycheck.analyzer.exception.AnalysisException;
+import org.owasp.dependencycheck.dependency.Confidence;
+import org.owasp.dependencycheck.dependency.Dependency;
+import org.owasp.dependencycheck.dependency.EvidenceCollection;
+import org.owasp.dependencycheck.utils.ExtractionException;
+import org.owasp.dependencycheck.utils.ExtractionUtil;
+import org.owasp.dependencycheck.utils.FileUtils;
+import org.owasp.dependencycheck.utils.Settings;
+import org.owasp.dependencycheck.utils.UrlStringUtils;
+
+/**
+ * Used to analyze a Wheel or egg distriution files, or their contents in
+ * unzipped form, and collect information that can be used to determine the
+ * associated CPE.
+ *
+ * @author Dale Visser <dvisser@ida.org>
+ */
+public class PythonDistributionAnalyzer extends AbstractFileTypeAnalyzer {
+
+	/**
+	 * Name of egg metatdata files to analyze.
+	 */
+	private static final String PKG_INFO = "PKG-INFO";
+
+	/**
+	 * Name of wheel metadata files to analyze.
+	 */
+	private static final String METADATA = "METADATA";
+
+	/**
+	 * The logger.
+	 */
+	private static final Logger LOGGER = Logger
+			.getLogger(PythonDistributionAnalyzer.class.getName());
+
+	/**
+	 * The count of directories created during analysis. This is used for
+	 * creating temporary directories.
+	 */
+	private static int dirCount = 0;
+
+	/**
+	 * The name of the analyzer.
+	 */
+	private static final String ANALYZER_NAME = "Python Distribution Analyzer";
+	/**
+	 * The phase that this analyzer is intended to run in.
+	 */
+	private static final AnalysisPhase ANALYSIS_PHASE = AnalysisPhase.INFORMATION_COLLECTION;
+
+	/**
+	 * The set of file extensions supported by this analyzer.
+	 */
+	private static final Set<String> EXTENSIONS = newHashSet("whl", "egg",
+			"zip", METADATA, PKG_INFO);
+
+	/**
+	 * Used to match on egg archive candidate extenssions.
+	 */
+	private static final Pattern EGG_OR_ZIP = Pattern.compile("egg|zip");
+
+	/**
+	 * The parent directory for the individual directories per archive.
+	 */
+	private File tempFileLocation;
+
+	/**
+	 * Filter that detects *.dist-info files (but doesn't verify they are
+	 * directories.
+	 */
+	private static final FilenameFilter DIST_INFO_FILTER = new SuffixFileFilter(
+			".dist-info");
+
+	/**
+	 * Filter that detects files named "METADATA".
+	 */
+	private static final FilenameFilter EGG_INFO_FILTER = new NameFileFilter(
+			"EGG-INFO");
+
+	/**
+	 * Filter that detects files named "METADATA".
+	 */
+	private static final FilenameFilter METADATA_FILTER = new NameFileFilter(
+			METADATA);
+
+	/**
+	 * Filter that detects files named "PKG-INFO".
+	 */
+	private static final FilenameFilter PKG_INFO_FILTER = new NameFileFilter(
+			PKG_INFO);
+
+	/**
+	 * Returns a list of file EXTENSIONS supported by this analyzer.
+	 *
+	 * @return a list of file EXTENSIONS supported by this analyzer.
+	 */
+	@Override
+	public Set<String> getSupportedExtensions() {
+		return EXTENSIONS;
+	}
+
+	/**
+	 * Returns the name of the analyzer.
+	 *
+	 * @return the name of the analyzer.
+	 */
+	@Override
+	public String getName() {
+		return ANALYZER_NAME;
+	}
+
+	/**
+	 * Returns the phase that the analyzer is intended to run in.
+	 *
+	 * @return the phase that the analyzer is intended to run in.
+	 */
+	public AnalysisPhase getAnalysisPhase() {
+		return ANALYSIS_PHASE;
+	}
+
+	/**
+	 * Returns the key used in the properties file to reference the analyzer's
+	 * enabled property.
+	 *
+	 * @return the analyzer's enabled property setting key
+	 */
+	@Override
+	protected String getAnalyzerEnabledSettingKey() {
+		return Settings.KEYS.ANALYZER_PYTHON_DISTRIBUTION_ENABLED;
+	}
+
+	@Override
+	protected void analyzeFileType(Dependency dependency, Engine engine)
+			throws AnalysisException {
+		if ("whl".equals(dependency.getFileExtension())) {
+			collectMetadataFromArchiveFormat(dependency, DIST_INFO_FILTER,
+					METADATA_FILTER);
+		} else if (EGG_OR_ZIP.matcher(
+				StringUtils.stripToEmpty(dependency.getFileExtension()))
+				.matches()) {
+			collectMetadataFromArchiveFormat(dependency, EGG_INFO_FILTER,
+					PKG_INFO_FILTER);
+		} else {
+			final File actualFile = dependency.getActualFile();
+			final String name = actualFile.getName();
+			final boolean metadata = METADATA.equals(name);
+			if (metadata || PKG_INFO.equals(name)) {
+				final File parent = actualFile.getParentFile();
+				final String parentName = parent.getName();
+				dependency.setDisplayFileName(parentName + "/" + name);
+				if (parent.isDirectory()
+						&& (metadata && parentName.endsWith(".dist-info")
+								|| parentName.endsWith(".egg-info") || "EGG-INFO"
+									.equals(parentName))) {
+					collectWheelMetadata(dependency, actualFile);
+				}
+			}
+		}
+	}
+
+	private void collectMetadataFromArchiveFormat(Dependency dependency,
+			FilenameFilter folderFilter, FilenameFilter metadataFilter)
+			throws AnalysisException {
+		final File temp = getNextTempDirectory();
+		LOGGER.fine(String.format("%s exists? %b", temp, temp.exists()));
+		try {
+			ExtractionUtil.extractFilesUsingFilter(
+					new File(dependency.getActualFilePath()), temp,
+					metadataFilter);
+		} catch (ExtractionException ex) {
+			throw new AnalysisException(ex);
+		}
+
+		collectWheelMetadata(
+				dependency,
+				getMatchingFile(getMatchingFile(temp, folderFilter),
+						metadataFilter));
+	}
+
+	/**
+	 * Makes sure a usable temporary directory is available.
+	 */
+	@Override
+	protected void initializeFileTypeAnalyzer() throws Exception {
+		final File baseDir = Settings.getTempDirectory();
+		tempFileLocation = File.createTempFile("check", "tmp", baseDir);
+		if (!tempFileLocation.delete()) {
+			final String msg = String.format(
+					"Unable to delete temporary file '%s'.",
+					tempFileLocation.getAbsolutePath());
+			throw new AnalysisException(msg);
+		}
+		if (!tempFileLocation.mkdirs()) {
+			final String msg = String.format(
+					"Unable to create directory '%s'.",
+					tempFileLocation.getAbsolutePath());
+			throw new AnalysisException(msg);
+		}
+	}
+
+	/**
+	 * Deletes any files extracted from the Wheel during analysis.
+	 */
+	@Override
+	public void close() {
+		if (tempFileLocation != null && tempFileLocation.exists()) {
+			LOGGER.log(Level.FINE, "Attempting to delete temporary files");
+			final boolean success = FileUtils.delete(tempFileLocation);
+			if (!success) {
+				LOGGER.log(Level.WARNING,
+						"Failed to delete some temporary files, see the log for more details");
+			}
+		}
+	}
+
+	/**
+	 * Gathers evidence from the METADATA file.
+	 *
+	 * @param dependency
+	 *            the dependency being analyzed
+	 * @throws MalformedURLException
+	 */
+	private static void collectWheelMetadata(Dependency dependency, File file)
+			throws AnalysisException {
+		final InternetHeaders headers = getManifestProperties(file);
+		addPropertyToEvidence(headers, dependency.getVersionEvidence(),
+				"Version", Confidence.HIGHEST);
+		addPropertyToEvidence(headers, dependency.getProductEvidence(), "Name",
+				Confidence.HIGHEST);
+		final String url = headers.getHeader("Home-page", null);
+		final EvidenceCollection vendorEvidence = dependency
+				.getVendorEvidence();
+		if (StringUtils.isNotBlank(url)) {
+			if (UrlStringUtils.isUrl(url)) {
+				vendorEvidence.addEvidence(METADATA, "vendor", url,
+						Confidence.MEDIUM);
+			}
+		}
+		addPropertyToEvidence(headers, vendorEvidence, "Author", Confidence.LOW);
+		final String summary = headers.getHeader("Summary", null);
+		if (StringUtils.isNotBlank(summary)) {
+			JarAnalyzer
+					.addDescription(dependency, summary, METADATA, "summary");
+		}
+	}
+
+	private static void addPropertyToEvidence(InternetHeaders headers,
+			EvidenceCollection evidence, String property, Confidence confidence) {
+		final String value = headers.getHeader(property, null);
+		LOGGER.fine(String.format("Property: %s, Value: %s\n", property, value));
+		if (StringUtils.isNotBlank(value)) {
+			evidence.addEvidence(METADATA, property, value, confidence);
+		}
+	}
+
+	private static final File getMatchingFile(File folder, FilenameFilter filter) {
+		File result = null;
+		final File[] matches = folder.listFiles(filter);
+		if (null != matches && 1 == matches.length) {
+			result = matches[0];
+		}
+		return result;
+	}
+
+	private static InternetHeaders getManifestProperties(File manifest) {
+		final InternetHeaders result = new InternetHeaders();
+		if (null == manifest) {
+			LOGGER.fine("Manifest file not found.");
+		} else {
+			try {
+				result.load(new AutoCloseInputStream(new BufferedInputStream(
+						new FileInputStream(manifest))));
+			} catch (MessagingException e) {
+				LOGGER.log(Level.WARNING, e.getMessage(), e);
+			} catch (FileNotFoundException e) {
+				LOGGER.log(Level.WARNING, e.getMessage(), e);
+			}
+		}
+		return result;
+	}
+
+	/**
+	 * Retrieves the next temporary destingation directory for extracting an
+	 * archive.
+	 *
+	 * @return a directory
+	 * @throws AnalysisException
+	 *             thrown if unable to create temporary directory
+	 */
+	private File getNextTempDirectory() throws AnalysisException {
+		File directory;
+
+		// getting an exception for some directories not being able to be
+		// created; might be because the directory already exists?
+		do {
+			dirCount += 1;
+			directory = new File(tempFileLocation, String.valueOf(dirCount));
+		} while (directory.exists());
+		if (!directory.mkdirs()) {
+			throw new AnalysisException(String.format(
+					"Unable to create temp directory '%s'.",
+					directory.getAbsolutePath()));
+		}
+		return directory;
+	}
+}

dependency-check-core/src/main/java/org/owasp/dependencycheck/analyzer/PythonPackageAnalyzer.java

@@ -0,0 +1,284 @@
+/*
+ * This file is part of dependency-check-core.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * Copyright (c) 2015 Institute for Defense Analyses. All Rights Reserved.
+ */
+package org.owasp.dependencycheck.analyzer;
+
+import java.io.File;
+import java.io.FileFilter;
+import java.io.IOException;
+import java.net.MalformedURLException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Set;
+import java.util.logging.Logger;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.io.filefilter.NameFileFilter;
+import org.apache.commons.io.filefilter.SuffixFileFilter;
+import org.owasp.dependencycheck.Engine;
+import org.owasp.dependencycheck.analyzer.exception.AnalysisException;
+import org.owasp.dependencycheck.dependency.Confidence;
+import org.owasp.dependencycheck.dependency.Dependency;
+import org.owasp.dependencycheck.dependency.EvidenceCollection;
+import org.owasp.dependencycheck.utils.Settings;
+import org.owasp.dependencycheck.utils.UrlStringUtils;
+
+/**
+ * Used to analyze a Python package, and collect information that can be used to
+ * determine the associated CPE.
+ *
+ * @author Dale Visser <dvisser@ida.org>
+ */
+public class PythonPackageAnalyzer extends AbstractFileTypeAnalyzer {
+
+	/**
+	 * Used when compiling file scanning regex patterns.
+	 */
+	private static final int REGEX_OPTIONS = Pattern.DOTALL
+			| Pattern.CASE_INSENSITIVE;
+
+	/**
+	 * The logger.
+	 */
+	private static final Logger LOGGER = Logger
+			.getLogger(PythonDistributionAnalyzer.class.getName());
+
+	/**
+	 * Filename extensions for files to be analyzed.
+	 */
+	private static final Set<String> EXTENSIONS = Collections
+			.unmodifiableSet(Collections.singleton("py"));
+
+	/**
+	 * Pattern for matching the module docstring in a source file.
+	 */
+	private static final Pattern MODULE_DOCSTRING = Pattern.compile(
+			"^(['\\\"]{3})(.*?)\\1", REGEX_OPTIONS);
+
+	/**
+	 * Matches assignments to version variables in Python source code.
+	 */
+	private static final Pattern VERSION_PATTERN = Pattern.compile(
+			"\\b(__)?version(__)? *= *(['\"]+)(\\d+\\.\\d+.*?)\\3",
+			REGEX_OPTIONS);
+
+	/**
+	 * Matches assignments to title variables in Python source code.
+	 */
+	private static final Pattern TITLE_PATTERN = compileAssignPattern("title");
+
+	/**
+	 * Matches assignments to summary variables in Python source code.
+	 */
+	private static final Pattern SUMMARY_PATTERN = compileAssignPattern("summary");
+
+	/**
+	 * Matches assignments to URL/URL variables in Python source code.
+	 */
+	private static final Pattern URI_PATTERN = compileAssignPattern("ur[il]");
+
+	/**
+	 * Matches assignments to home page variables in Python source code.
+	 */
+	private static final Pattern HOMEPAGE_PATTERN = compileAssignPattern("home_?page");
+
+	/**
+	 * Matches assignments to author variables in Python source code.
+	 */
+	private static final Pattern AUTHOR_PATTERN = compileAssignPattern("author");
+
+	/**
+	 * Filter that detects files named "__init__.py".
+	 */
+	private static final FileFilter INIT_PY_FILTER = new NameFileFilter(
+			"__init__.py");
+
+	private static final FileFilter PY_FILTER = new SuffixFileFilter(".py");
+
+	/**
+	 * Returns the name of the Python Package Analyzer.
+	 */
+	@Override
+	public String getName() {
+		return "Python Package Analyzer";
+	}
+
+	/**
+	 * Tell that we are used for information collection.
+	 */
+	@Override
+	public AnalysisPhase getAnalysisPhase() {
+		return AnalysisPhase.INFORMATION_COLLECTION;
+	}
+
+	/**
+	 * Return the set of supported file extensions.
+	 */
+	@Override
+	protected Set<String> getSupportedExtensions() {
+		return EXTENSIONS;
+	}
+
+	/**
+	 * No-op initializer implementation.
+	 */
+	@Override
+	protected void initializeFileTypeAnalyzer() throws Exception {
+		// Nothing to do here.
+	}
+
+	private static Pattern compileAssignPattern(String name) {
+		return Pattern.compile(
+				String.format("\\b(__)?%s(__)?\\b *= *(['\"]+)(.*?)\\3", name),
+				REGEX_OPTIONS);
+	}
+
+	@Override
+	protected void analyzeFileType(Dependency dependency, Engine engine)
+			throws AnalysisException {
+		final File file = dependency.getActualFile();
+		final File parent = file.getParentFile();
+		final String parentName = parent.getName();
+		boolean found = false;
+		if (INIT_PY_FILTER.accept(file)) {
+			for (final File sourcefile : parent.listFiles(PY_FILTER)) {
+				found |= analyzeFileContents(dependency, sourcefile);
+			}
+		}
+		if (found) {
+			dependency.setDisplayFileName(parentName + "/__init__.py");
+			dependency.getProductEvidence().addEvidence(file.getName(),
+					"PackageName", parentName, Confidence.MEDIUM);
+		} else {
+			// copy, alter and set in case some other thread is iterating over
+			final List<Dependency> deps = new ArrayList<Dependency>(
+					engine.getDependencies());
+			deps.remove(dependency);
+			engine.setDependencies(deps);
+		}
+	}
+
+	/**
+	 * This should gather information from leading docstrings, file comments,
+	 * and assignments to __version__, __title__, __summary__, __uri__, __url__,
+	 * __home*page__, __author__, and their all caps equivalents.
+	 *
+	 * @return whether evidence was found
+	 */
+	private boolean analyzeFileContents(Dependency dependency, File file)
+			throws AnalysisException {
+		String contents = "";
+		try {
+			contents = FileUtils.readFileToString(file).trim();
+		} catch (IOException e) {
+			throw new AnalysisException(
+					"Problem occured while reading dependency file.", e);
+		}
+		boolean found = false;
+		if (!contents.isEmpty()) {
+			final String source = file.getName();
+			found = gatherEvidence(VERSION_PATTERN, contents, source,
+					dependency.getVersionEvidence(), "SourceVersion",
+					Confidence.MEDIUM);
+			found |= addSummaryInfo(dependency, SUMMARY_PATTERN, 4, contents,
+					source, "summary");
+			if (INIT_PY_FILTER.accept(file)) {
+				found |= addSummaryInfo(dependency, MODULE_DOCSTRING, 2,
+						contents, source, "docstring");
+			}
+			found |= gatherEvidence(TITLE_PATTERN, contents, source,
+					dependency.getProductEvidence(), "SourceTitle",
+					Confidence.LOW);
+			final EvidenceCollection vendorEvidence = dependency
+					.getVendorEvidence();
+			found |= gatherEvidence(AUTHOR_PATTERN, contents, source,
+					vendorEvidence, "SourceAuthor", Confidence.MEDIUM);
+			try {
+				found |= gatherHomePageEvidence(URI_PATTERN, vendorEvidence,
+						source, "URL", contents);
+				found |= gatherHomePageEvidence(HOMEPAGE_PATTERN,
+						vendorEvidence, source, "HomePage", contents);
+			} catch (MalformedURLException e) {
+				LOGGER.warning(e.getMessage());
+			}
+		}
+		return found;
+	}
+
+	private boolean addSummaryInfo(Dependency dependency, Pattern pattern,
+			int group, String contents, String source, String key) {
+		final Matcher matcher = pattern.matcher(contents);
+		final boolean found = matcher.find();
+		if (found) {
+			JarAnalyzer.addDescription(dependency, matcher.group(group),
+					source, key);
+		}
+		return found;
+	}
+
+	private boolean gatherHomePageEvidence(Pattern pattern,
+			EvidenceCollection evidence, String source, String name,
+			String contents) throws MalformedURLException {
+		final Matcher matcher = pattern.matcher(contents);
+		boolean found = false;
+		if (matcher.find()) {
+			final String url = matcher.group(4);
+			if (UrlStringUtils.isUrl(url)) {
+				found = true;
+				evidence.addEvidence(source, name, url, Confidence.MEDIUM);
+			}
+		}
+		return found;
+	}
+
+	/**
+	 * Gather evidence from a Python source file usin the given string
+	 * assignment regex pattern.
+	 *
+	 * @param pattern
+	 *            to scan contents with
+	 * @param contents
+	 *            of Python source file
+	 * @param source
+	 *            for storing evidence
+	 * @param evidence
+	 *            to store evidence in
+	 * @param name
+	 *            of evidence
+	 * @param confidence
+	 *            in evidence
+	 * @return whether evidence was found
+	 */
+	private boolean gatherEvidence(Pattern pattern, String contents,
+			String source, EvidenceCollection evidence, String name,
+			Confidence confidence) {
+		final Matcher matcher = pattern.matcher(contents);
+		final boolean found = matcher.find();
+		if (found) {
+			evidence.addEvidence(source, name, matcher.group(4), confidence);
+		}
+		return found;
+	}
+
+	@Override
+	protected String getAnalyzerEnabledSettingKey() {
+		return Settings.KEYS.ANALYZER_PYTHON_PACKAGE_ENABLED;
+	}
+}

dependency-check-core/src/main/java/org/owasp/dependencycheck/dependency/Dependency.java

@@ -222,7 +222,7 @@ public class Dependency implements Serializable, Comparable<Dependency> {
     }
 
     /**
-     * Sets the file name of the dependency.
+     * Sets the file extension of the dependency.
      *
      * @param fileExtension the file name of the dependency
      */

dependency-check-core/src/main/java/org/owasp/dependencycheck/utils/ExtractionUtil.java

@@ -17,19 +17,29 @@
  */
 package org.owasp.dependencycheck.utils;
 
+import static org.owasp.dependencycheck.utils.FileUtils.getFileExtension;
+
 import java.io.BufferedInputStream;
 import java.io.BufferedOutputStream;
+import java.io.Closeable;
 import java.io.File;
 import java.io.FileInputStream;
 import java.io.FileNotFoundException;
 import java.io.FileOutputStream;
+import java.io.FilenameFilter;
 import java.io.IOException;
+import java.io.InputStream;
 import java.util.logging.Level;
 import java.util.logging.Logger;
 import java.util.zip.ZipEntry;
 import java.util.zip.ZipInputStream;
+
+import org.apache.commons.compress.archivers.ArchiveEntry;
+import org.apache.commons.compress.archivers.ArchiveInputStream;
+import org.apache.commons.compress.archivers.zip.ZipArchiveInputStream;
 import org.owasp.dependencycheck.Engine;
-import static org.owasp.dependencycheck.utils.FileUtils.getFileExtension;
+import org.owasp.dependencycheck.analyzer.exception.AnalysisException;
+import org.owasp.dependencycheck.analyzer.exception.ArchiveExtractionException;
 
 /**
  *
@@ -106,12 +116,7 @@ public final class ExtractionUtil {
                         try {
                             fos = new FileOutputStream(file);
                             bos = new BufferedOutputStream(fos, BUFFER_SIZE);
-                            int count;
-                            final byte[] data = new byte[BUFFER_SIZE];
-                            while ((count = zis.read(data, 0, BUFFER_SIZE)) != -1) {
-                                bos.write(data, 0, count);
-                            }
-                            bos.flush();
+                            transferUsingBuffer(zis, bos);
                         } catch (FileNotFoundException ex) {
                             LOGGER.log(Level.FINE, null, ex);
                             final String msg = String.format("Unable to find file '%s'.", file.getName());
@@ -121,13 +126,7 @@ public final class ExtractionUtil {
                             final String msg = String.format("IO Exception while parsing file '%s'.", file.getName());
                             throw new ExtractionException(msg, ex);
                         } finally {
-                            if (bos != null) {
-                                try {
-                                    bos.close();
-                                } catch (IOException ex) {
-                                    LOGGER.log(Level.FINEST, null, ex);
-                                }
-                            }
+                            closeStream(bos);
                         }
                     }
                 }
@@ -137,11 +136,157 @@ public final class ExtractionUtil {
             LOGGER.log(Level.FINE, msg, ex);
             throw new ExtractionException(msg, ex);
         } finally {
-            try {
-                zis.close();
-            } catch (IOException ex) {
-                LOGGER.log(Level.FINEST, null, ex);
-            }
+            closeStream(zis);
         }
     }
+
+	/**
+	 * Extracts the contents of an archive into the specified directory.
+	 *
+	 * @param archive
+	 *            an archive file such as a WAR or EAR
+	 * @param destination
+	 *            a directory to extract the contents to
+	 * @param filter
+	 *            determines which files get extracted
+	 * @throws ExtractionException
+	 *             thrown if the archive is not found
+	 */
+	public static void extractFilesUsingFilter(File archive, File destination,
+			FilenameFilter filter) throws ExtractionException {
+		if (archive == null || destination == null) {
+			return;
+		}
+
+		FileInputStream fis = null;
+		try {
+			fis = new FileInputStream(archive);
+		} catch (FileNotFoundException ex) {
+			LOGGER.log(Level.FINE, null, ex);
+			throw new ExtractionException("Archive file was not found.", ex);
+		}
+		try {
+			extractArchive(new ZipArchiveInputStream(new BufferedInputStream(
+					fis)), destination, filter);
+		} catch (ArchiveExtractionException ex) {
+			final String msg = String.format(
+					"Exception extracting archive '%s'.", archive.getName());
+			LOGGER.log(Level.WARNING, msg);
+			LOGGER.log(Level.FINE, null, ex);
+		} finally {
+			try {
+				fis.close();
+			} catch (IOException ex) {
+				LOGGER.log(Level.FINE, null, ex);
+			}
+		}
+	}
+
+	/**
+	 * Extracts files from an archive.
+	 *
+	 * @param input
+	 *            the archive to extract files from
+	 * @param destination
+	 *            the location to write the files too
+	 * @param filter
+	 *            determines which files get extracted
+	 * @throws ArchiveExtractionException
+	 *             thrown if there is an exception extracting files from the
+	 *             archive
+	 */
+	private static void extractArchive(ArchiveInputStream input,
+			File destination, FilenameFilter filter)
+			throws ArchiveExtractionException {
+		ArchiveEntry entry;
+		try {
+			while ((entry = input.getNextEntry()) != null) {
+				if (entry.isDirectory()) {
+					final File dir = new File(destination, entry.getName());
+					if (!dir.exists()) {
+						if (!dir.mkdirs()) {
+							final String msg = String.format(
+									"Unable to create directory '%s'.",
+									dir.getAbsolutePath());
+							throw new AnalysisException(msg);
+						}
+					}
+				} else {
+					extractFile(input, destination, filter, entry);
+				}
+			}
+		} catch (IOException ex) {
+			throw new ArchiveExtractionException(ex);
+		} catch (Throwable ex) {
+			throw new ArchiveExtractionException(ex);
+		} finally {
+			closeStream(input);
+		}
+	}
+
+	private static void extractFile(ArchiveInputStream input, File destination,
+			FilenameFilter filter, ArchiveEntry entry) throws ExtractionException {
+		final File file = new File(destination, entry.getName());
+		if (filter.accept(file.getParentFile(), file.getName())) {
+			final String extracting = String.format("Extracting '%s'",
+					file.getPath());
+			LOGGER.fine(extracting);
+			BufferedOutputStream bos = null;
+			FileOutputStream fos = null;
+			try {
+				createParentFile(file);
+				fos = new FileOutputStream(file);
+				bos = new BufferedOutputStream(fos, BUFFER_SIZE);
+				transferUsingBuffer(input, bos);
+			} catch (FileNotFoundException ex) {
+				LOGGER.log(Level.FINE, null, ex);
+				final String msg = String.format("Unable to find file '%s'.",
+						file.getName());
+				throw new ExtractionException(msg, ex);
+			} catch (IOException ex) {
+				LOGGER.log(Level.FINE, null, ex);
+				final String msg = String
+						.format("IO Exception while parsing file '%s'.",
+								file.getName());
+				throw new ExtractionException(msg, ex);
+			} finally {
+				closeStream(bos);
+				closeStream(fos);
+			}
+		}
+	}
+
+	private static void transferUsingBuffer(InputStream input,
+			BufferedOutputStream bos) throws IOException {
+		int count;
+		final byte[] data = new byte[BUFFER_SIZE];
+		while ((count = input.read(data, 0, BUFFER_SIZE)) != -1) {
+			bos.write(data, 0, count);
+		}
+		bos.flush();
+	}
+
+	private static void closeStream(Closeable stream) {
+		if (stream != null) {
+			try {
+				stream.close();
+			} catch (IOException ex) {
+				LOGGER.log(Level.FINEST, null, ex);
+			}
+		}
+	}
+
+	private static void createParentFile(final File file)
+			throws ExtractionException {
+		final File parent = file.getParentFile();
+		if (!parent.isDirectory()) {
+			if (!parent.mkdirs()) {
+				final String msg = String.format(
+						"Unable to build directory '%s'.",
+						parent.getAbsolutePath());
+				throw new ExtractionException(msg);
+			}
+		}
+	}
+
 }

dependency-check-core/src/main/resources/META-INF/services/org.owasp.dependencycheck.analyzer.Analyzer

@@ -11,4 +11,6 @@ org.owasp.dependencycheck.analyzer.VulnerabilitySuppressionAnalyzer
 org.owasp.dependencycheck.analyzer.CentralAnalyzer
 org.owasp.dependencycheck.analyzer.NexusAnalyzer
 org.owasp.dependencycheck.analyzer.NuspecAnalyzer
-org.owasp.dependencycheck.analyzer.AssemblyAnalyzer
+org.owasp.dependencycheck.analyzer.AssemblyAnalyzer
+org.owasp.dependencycheck.analyzer.PythonDistributionAnalyzer
+org.owasp.dependencycheck.analyzer.PythonPackageAnalyzer

dependency-check-core/src/main/resources/dependencycheck-base-suppression.xml

@@ -83,5 +83,21 @@
         <gav regex="true">org\.opensaml:xmltooling:.*</gav>
         <cpe>cpe:/a:internet2:opensaml</cpe>
     </suppress>
-
+    <suppress base="true">
+        <notes><![CDATA[
+        Suppresses false positives for python:python.
+        ]]></notes>
+        <filePath regex="true">.*(\.(whl|egg)|\b(site|dist)-packages\b.*)</filePath>
+        <cpe>cpe:/a:python:python</cpe>
+        <cpe>cpe:/a:python_software_foundation:python</cpe>
+        <cpe>cpe:/a:class:class</cpe>
+        <cpe>cpe:/a:file:file</cpe>
+        <cpe>cpe:/a:gnupg:gnupg</cpe>
+        <cpe>cpe:/a:mongodb:mongodb</cpe>
+        <cpe>cpe:/a:mozilla:mozilla</cpe>
+        <cpe>cpe:/a:openssl:openssl</cpe>
+        <cpe>cpe:/a:sendfile:sendfile</cpe>
+        <cpe>cpe:/a:sendmail:sendmail</cpe>
+        <cpe>cpe:/a:yacc:yacc</cpe>
+    </suppress>
 </suppressions>