Added capability to scan Python egg distribution format (.egg and .zip).

Changed copyright notices to IDA, and added as possible copyyright holder in
checkstyle-header file, and some whitespace fixes.


Former-commit-id: ac4288dff7c3c40e64dc733791c80035f73cc602

dependency-check-core/pom.xml

@@ -528,9 +528,9 @@ Copyright (c) 2012 Jeremy Long. All Rights Reserved.
             <optional>true</optional>
         </dependency>
         <dependency>
-        	<groupId>com.sun.mail</groupId>
+            <groupId>com.sun.mail</groupId>
-        	<artifactId>mailapi</artifactId>
+            <artifactId>mailapi</artifactId>
-        	<version>1.5.2</version>
+            <version>1.5.2</version>
         </dependency>
     </dependencies>
     <profiles>

dependency-check-core/src/main/java/org/owasp/dependencycheck/Engine.java

@@ -320,13 +320,13 @@ public class Engine {
         final String fileName = file.getName();
         String extension = FileUtils.getFileExtension(fileName);
         if (null == extension) {
-        	extension = fileName;
+            extension = fileName;
         }
         Dependency dependency = null;
         if (supportsExtension(extension)) {
             dependency = new Dependency(file);
             if (extension == fileName){
-            	dependency.setFileExtension(extension);
+                dependency.setFileExtension(extension);
             }
             dependencies.add(dependency);
         }

dependency-check-core/src/main/java/org/owasp/dependencycheck/analyzer/PythonDistributionAnalyzer.java

@@ -13,7 +13,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  *
- * Copyright (c) 2012 Jeremy Long. All Rights Reserved.
+ * Copyright (c) 2015 Institute for Defense Analyses. All Rights Reserved.
  */
 package org.owasp.dependencycheck.analyzer;
 
@@ -53,6 +53,9 @@ import org.owasp.dependencycheck.utils.Settings;
  */
 public class PythonDistributionAnalyzer extends AbstractFileTypeAnalyzer {
 
+	/**
+	 * Name of egg metatdata files to analyze.
+	 */
 	private static final String PKG_INFO = "PKG-INFO";
 
 	/**
@@ -84,14 +87,24 @@ public class PythonDistributionAnalyzer extends AbstractFileTypeAnalyzer {
 	/**
 	 * The set of file extensions supported by this analyzer.
 	 */
-	private static final Set<String> EXTENSIONS = newHashSet("whl", METADATA,
+	private static final Set<String> EXTENSIONS = newHashSet("whl", "egg",
-			PKG_INFO);
+			"zip", METADATA, PKG_INFO);
 
 	/**
 	 * Pattern that captures the vendor from a home page URL.
 	 */
 	private static final Pattern HOMEPAGE_VENDOR = Pattern
-			.compile("^[a-zA-Z]+://.*\\.(.+)\\.[a-zA-Z]+/?.*$");
+			.compile("^[a-zA-Z]+://(.*)$");
+
+	/**
+	 * Used to split the subdomains of a host name.
+	 */
+	private static final Pattern DOT = Pattern.compile("\\.");
+
+	/**
+	 * Used to match on egg archive candidate extenssions.
+	 */
+	private static final Pattern EGG_OR_ZIP = Pattern.compile("egg|zip");
 
 	/**
 	 * The parent directory for the individual directories per archive.
@@ -105,6 +118,12 @@ public class PythonDistributionAnalyzer extends AbstractFileTypeAnalyzer {
 	private static final FilenameFilter DIST_INFO_FILTER = new SuffixFileFilter(
 			".dist-info");
 
+	/**
+	 * Filter that detects files named "METADATA".
+	 */
+	private static final FilenameFilter EGG_INFO_FILTER = new NameFileFilter(
+			"EGG-INFO");
+
 	/**
 	 * Filter that detects files named "METADATA".
 	 */
@@ -112,11 +131,10 @@ public class PythonDistributionAnalyzer extends AbstractFileTypeAnalyzer {
 			METADATA);
 
 	/**
-	 * Constructs a new PythonDistributionAnalyzer.
+	 * Filter that detects files named "PKG-INFO".
 	 */
-	public PythonDistributionAnalyzer() {
+	private static final FilenameFilter PKG_INFO_FILTER = new NameFileFilter(
-		super();
+			PKG_INFO);
-	}
 
 	/**
 	 * Returns a list of file EXTENSIONS supported by this analyzer.
@@ -147,8 +165,6 @@ public class PythonDistributionAnalyzer extends AbstractFileTypeAnalyzer {
 		return ANALYSIS_PHASE;
 	}
 
-	// </editor-fold>
-
 	/**
 	 * Returns the key used in the properties file to reference the analyzer's
 	 * enabled property.
@@ -164,22 +180,13 @@ public class PythonDistributionAnalyzer extends AbstractFileTypeAnalyzer {
 	protected void analyzeFileType(Dependency dependency, Engine engine)
 			throws AnalysisException {
 		if ("whl".equals(dependency.getFileExtension())) {
-			final File tmpWheelFolder = getNextTempDirectory();
+			collectMetadataFromArchiveFormat(dependency, DIST_INFO_FILTER,
-			LOGGER.fine(String.format("%s exists? %b", tmpWheelFolder,
+					METADATA_FILTER);
-					tmpWheelFolder.exists()));
+		} else if (EGG_OR_ZIP.matcher(
-			try {
+				StringUtils.stripToEmpty(dependency.getFileExtension()))
-				ExtractionUtil.extractFilesUsingFilter(
+				.matches()) {
-						new File(dependency.getActualFilePath()),
+			collectMetadataFromArchiveFormat(dependency, EGG_INFO_FILTER,
-						tmpWheelFolder, METADATA_FILTER);
+					PKG_INFO_FILTER);
-			} catch (ExtractionException ex) {
-				throw new AnalysisException(ex);
-			}
-
-			collectWheelMetadata(
-					dependency,
-					getMatchingFile(
-							getMatchingFile(tmpWheelFolder, DIST_INFO_FILTER),
-							METADATA_FILTER));
 		} else {
 			final File actualFile = dependency.getActualFile();
 			final String name = actualFile.getName();
@@ -189,14 +196,34 @@ public class PythonDistributionAnalyzer extends AbstractFileTypeAnalyzer {
 				final String parentName = parent.getName();
 				dependency.setDisplayFileName(parentName + "/" + name);
 				if (parent.isDirectory()
-						&& ((metadata && parentName.endsWith(".dist-info")) || parentName
+						&& (metadata && parentName.endsWith(".dist-info")
-								.endsWith(".egg-info"))) {
+								|| parentName.endsWith(".egg-info") || "EGG-INFO"
+									.equals(parentName))) {
 					collectWheelMetadata(dependency, actualFile);
 				}
 			}
 		}
 	}
 
+	private void collectMetadataFromArchiveFormat(Dependency dependency,
+			FilenameFilter folderFilter, FilenameFilter metadataFilter)
+			throws AnalysisException {
+		final File temp = getNextTempDirectory();
+		LOGGER.fine(String.format("%s exists? %b", temp, temp.exists()));
+		try {
+			ExtractionUtil.extractFilesUsingFilter(
+					new File(dependency.getActualFilePath()), temp,
+					metadataFilter);
+		} catch (ExtractionException ex) {
+			throw new AnalysisException(ex);
+		}
+
+		collectWheelMetadata(
+				dependency,
+				getMatchingFile(getMatchingFile(temp, folderFilter),
+						metadataFilter));
+	}
+
 	/**
 	 * Makes sure a usable temporary directory is available.
 	 */
@@ -251,8 +278,10 @@ public class PythonDistributionAnalyzer extends AbstractFileTypeAnalyzer {
 		if (StringUtils.isNotBlank(url)) {
 			final Matcher matcher = HOMEPAGE_VENDOR.matcher(url);
 			if (matcher.matches()) {
+				final String[] subdomains = DOT.split(matcher.group(1));
 				vendorEvidence.addEvidence(METADATA, "vendor",
-						matcher.group(1), Confidence.MEDIUM);
+						subdomains[Math.max(0, subdomains.length - 2)],
+						Confidence.MEDIUM);
 			}
 		}
 		addPropertyToEvidence(headers, vendorEvidence, "Author", Confidence.LOW);

dependency-check-core/src/test/java/org/owasp/dependencycheck/analyzer/PythonDistributionAnalyzerTest.java

@@ -13,7 +13,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  *
- * Copyright (c) 2012 Dale Visser. All Rights Reserved.
+ * Copyright (c) 2015 Institute for Defense Analyses. All Rights Reserved.
  */
 package org.owasp.dependencycheck.analyzer;
 
@@ -23,6 +23,7 @@ import static org.junit.Assert.assertTrue;
 import java.util.Arrays;
 import java.util.HashSet;
 
+import org.apache.commons.lang.StringUtils;
 import org.junit.Test;
 import org.owasp.dependencycheck.BaseTest;
 import org.owasp.dependencycheck.analyzer.exception.AnalysisException;
@@ -30,11 +31,52 @@ import org.owasp.dependencycheck.dependency.Dependency;
 import org.owasp.dependencycheck.dependency.Evidence;
 
 /**
+ * Unit tests for PythonDistributionAnalyzer.
  *
  * @author Dale Visser <dvisser@ida.org>
  */
 public class PythonDistributionAnalyzerTest extends BaseTest {
 
+	/**
+	 * Test of getName method, of class PythonDistributionAnalyzer.
+	 */
+	@Test
+	public void testGetName() {
+		assertEquals("Analyzer name wrong.", "Python Distribution Analyzer",
+				new PythonDistributionAnalyzer().getName());
+	}
+
+	/**
+	 * Test of getSupportedExtensions method, of class JarAnalyzer.
+	 */
+	@Test
+	public void testGetSupportedExtensions() {
+		final String[] expected = { "whl", "egg", "zip", "METADATA", "PKG-INFO" };
+		assertEquals("Supported extensions should just have the following: "
+				+ StringUtils.join(expected, ", "),
+				new HashSet<String>(Arrays.asList(expected)),
+				new PythonDistributionAnalyzer().getSupportedExtensions());
+	}
+	
+	/**
+	 * Test of supportsExtension method, of class PythonDistributionAnalyzer.
+	 */
+	@Test
+	public void testSupportsExtension() {
+		final PythonDistributionAnalyzer analyzer = new PythonDistributionAnalyzer();
+		assertTrue("Should support \"whl\" extension.",
+				analyzer.supportsExtension("whl"));
+		assertTrue("Should support \"egg\" extension.",
+				analyzer.supportsExtension("egg"));
+		assertTrue("Should support \"zip\" extension.",
+				analyzer.supportsExtension("zip"));
+		assertTrue("Should support \"METADATA\" extension.",
+				analyzer.supportsExtension("METADATA"));
+		assertTrue("Should support \"PKG-INFO\" extension.",
+				analyzer.supportsExtension("PKG-INFO"));
+	}
+
+
 	/**
 	 * Test of inspect method, of class JarAnalyzer.
 	 *
@@ -44,7 +86,7 @@ public class PythonDistributionAnalyzerTest extends BaseTest {
 	@Test
 	public void testAnalyzeWheel() throws AnalysisException {
 		djangoAssertions(new Dependency(BaseTest.getResourceAsFile(this,
-				"Django-1.7.2-py2.py3-none-any.whl")));
+				"python/Django-1.7.2-py2.py3-none-any.whl")));
 	}
 
 	/**
@@ -56,7 +98,7 @@ public class PythonDistributionAnalyzerTest extends BaseTest {
 	@Test
 	public void testAnalyzeSitePackage() throws AnalysisException {
 		final Dependency result = new Dependency(BaseTest.getResourceAsFile(
-				this, "site-packages/Django-1.7.2.dist-info/METADATA"));
+				this, "python/site-packages/Django-1.7.2.dist-info/METADATA"));
 		djangoAssertions(result);
 		assertEquals("Django-1.7.2.dist-info/METADATA",
 				result.getDisplayFileName());
@@ -78,54 +120,39 @@ public class PythonDistributionAnalyzerTest extends BaseTest {
 	}
 
 	@Test
-	public void testAnalyzeEggInfo() throws AnalysisException {
+	public void testAnalyzeEggInfoFolder() throws AnalysisException {
+		eggtestAssertions("python/site-packages/EggTest.egg-info/PKG-INFO");
+	}
+
+	@Test
+	public void testAnalyzeEggArchive() throws AnalysisException {
+		eggtestAssertions("python/dist/EggTest-0.0.1-py2.7.egg");
+	}
+
+	@Test
+	public void testAnalyzeEggArchiveNamedZip() throws AnalysisException {
+		eggtestAssertions("python/dist/EggTest-0.0.1-py2.7.zip");
+	}
+
+	@Test
+	public void testAnalyzeEggFolder() throws AnalysisException {
+		eggtestAssertions("python/site-packages/EggTest-0.0.1-py2.7.egg/EGG-INFO/PKG-INFO");
+	}
+
+	private void eggtestAssertions(final String resource)
+			throws AnalysisException {
 		final Dependency result = new Dependency(BaseTest.getResourceAsFile(
-				this, "site-packages/eggutils-0.0.2-py2.7.egg-info/PKG-INFO"));
+				this, resource));
 		new PythonDistributionAnalyzer().analyze(result, null);
-		assertTrue("Expected vendor evidence to contain \"python\".", result
+		assertTrue("Expected vendor evidence to contain \"example\".", result
-				.getVendorEvidence().toString().contains("python"));
+				.getVendorEvidence().toString().contains("example"));
 		boolean found = false;
 		for (final Evidence e : result.getVersionEvidence()) {
-			if ("Version".equals(e.getName()) && "0.0.2".equals(e.getValue())) {
+			if ("Version".equals(e.getName()) && "0.0.1".equals(e.getValue())) {
 				found = true;
 				break;
 			}
 		}
-		assertTrue("Version 0.0.2 not found in eggutils dependency.", found);
+		assertTrue("Version 0.0.1 not found in EggTest dependency.", found);
-	}
-
-	/**
-	 * Test of getSupportedExtensions method, of class JarAnalyzer.
-	 */
-	@Test
-	public void testGetSupportedExtensions() {
-		assertEquals(
-				"Supported extensions should just be \"whl\", \"METADATA\" and \"PKG-INFO\".",
-				new HashSet<String>(Arrays
-						.asList("whl", "METADATA", "PKG-INFO")),
-				new PythonDistributionAnalyzer().getSupportedExtensions());
-	}
-
-	/**
-	 * Test of getName method, of class PythonDistributionAnalyzer.
-	 */
-	@Test
-	public void testGetName() {
-		assertEquals("Analyzer name wrong.", "Python Distribution Analyzer",
-				new PythonDistributionAnalyzer().getName());
-	}
-
-	/**
-	 * Test of supportsExtension method, of class PythonDistributionAnalyzer.
-	 */
-	@Test
-	public void testSupportsExtension() {
-		final PythonDistributionAnalyzer analyzer = new PythonDistributionAnalyzer();
-		assertTrue("Should support \"whl\" extension.",
-				analyzer.supportsExtension("whl"));
-		assertTrue("Should support \"METADATA\" extension.",
-				analyzer.supportsExtension("METADATA"));
-		assertTrue("Should support \"METADATA\" extension.",
-				analyzer.supportsExtension("PKG-INFO"));
 	}
 }

dependency-check-core/src/test/resources/python/eggtest/__init__.py

@@ -0,0 +1 @@
+from eggtest import main

dependency-check-core/src/test/resources/python/eggtest/main.py

@@ -0,0 +1 @@
+print 'Hello from eggtest!'

dependency-check-core/src/test/resources/python/setup.py

@@ -0,0 +1,9 @@
+from setuptools import setup
+
+setup(name = 'EggTest',
+      packages = ['eggtest'],
+      version = '0.0.1',
+      description = 'Simple project for producing an .egg.',
+      url = 'http://example.org/eggtest',
+      author = 'Dale Visser',
+      author_email = 'dvisser@ida.org')

dependency-check-core/src/test/resources/python/site-packages/EggTest-0.0.1-py2.7.egg/EGG-INFO/PKG-INFO

@@ -0,0 +1,10 @@
+Metadata-Version: 1.0
+Name: EggTest
+Version: 0.0.1
+Summary: Simple project for producing an .egg.
+Home-page: http://example.org/eggtest
+Author: Dale Visser
+Author-email: dvisser@ida.org
+License: UNKNOWN
+Description: UNKNOWN
+Platform: UNKNOWN

dependency-check-core/src/test/resources/python/site-packages/EggTest-0.0.1-py2.7.egg/EGG-INFO/SOURCES.txt

@@ -0,0 +1,7 @@
+setup.py
+EggTest.egg-info/PKG-INFO
+EggTest.egg-info/SOURCES.txt
+EggTest.egg-info/dependency_links.txt
+EggTest.egg-info/top_level.txt
+eggtest/__init__.py
+eggtest/main.py

dependency-check-core/src/test/resources/python/site-packages/EggTest-0.0.1-py2.7.egg/EGG-INFO/top_level.txt

@@ -0,0 +1 @@
+eggtest

dependency-check-core/src/test/resources/python/site-packages/EggTest-0.0.1-py2.7.egg/EGG-INFO/zip-safe

@@ -0,0 +1 @@
+

dependency-check-core/src/test/resources/python/site-packages/EggTest-0.0.1-py2.7.egg/eggtest/__init__.py

@@ -0,0 +1 @@
+from eggtest import main

dependency-check-core/src/test/resources/python/site-packages/EggTest-0.0.1-py2.7.egg/eggtest/main.py

@@ -0,0 +1 @@
+print 'Hello from eggtest!'

dependency-check-core/src/test/resources/python/site-packages/EggTest.egg-info/PKG-INFO

@@ -0,0 +1,10 @@
+Metadata-Version: 1.0
+Name: EggTest
+Version: 0.0.1
+Summary: Simple project for producing an .egg.
+Home-page: http://example.org/eggtest
+Author: Dale Visser
+Author-email: dvisser@ida.org
+License: UNKNOWN
+Description: UNKNOWN
+Platform: UNKNOWN

dependency-check-core/src/test/resources/python/site-packages/EggTest.egg-info/SOURCES.txt

@@ -0,0 +1,7 @@
+setup.py
+EggTest.egg-info/PKG-INFO
+EggTest.egg-info/SOURCES.txt
+EggTest.egg-info/dependency_links.txt
+EggTest.egg-info/top_level.txt
+eggtest/__init__.py
+eggtest/main.py

dependency-check-core/src/test/resources/python/site-packages/EggTest.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

dependency-check-core/src/test/resources/python/site-packages/EggTest.egg-info/top_level.txt

@@ -0,0 +1 @@
+eggtest

dependency-check-core/src/test/resources/site-packages/eggutils-0.0.2-py2.7.egg-info/PKG-INFO

@@ -1,28 +0,0 @@
-Metadata-Version: 1.0
-Name: eggutils
-Version: 0.0.2
-Summary: A set of utilities to create/manipulate eggs
-Home-page: http://pypi.python.org/pypi/eggutils
-Author: David Cournapeau
-Author-email: cournape@gmail.com
-License: BSD
-Description: Set of utilities to manipulate and create eggs without setuptools, and outside
-        distutils context (i.e. without running setup).
-        
-        Making an egg containing DLL
-        ============================
-        
-        Given Windows model for DLLs, when several python packages depends on the same
-        dll to be shared between extensions, it may be useful to have a "DLL egg" which
-        put the dlls within the python installation such as the dll are automatically
-        found by any extension to the corresponding python interpreter.
-        
-        Usage:
-        
-        ::
-                make-dll-egg -m PKG-INFO foo.dll bar.dll
-        
-        This will create a DLL with metadata taken from the PKG-INFO file, containing
-        both foo and bar dlls.
-        
-Platform: any

dependency-check-core/src/test/resources/site-packages/eggutils-0.0.2-py2.7.egg-info/SOURCES.txt

@@ -1,10 +0,0 @@
-README
-setup.cfg
-setup.py
-eggutils/__init__.py
-eggutils/eggutils.py
-eggutils.egg-info/PKG-INFO
-eggutils.egg-info/SOURCES.txt
-eggutils.egg-info/dependency_links.txt
-eggutils.egg-info/entry_points.txt
-eggutils.egg-info/top_level.txt

dependency-check-core/src/test/resources/site-packages/eggutils-0.0.2-py2.7.egg-info/entry_points.txt

@@ -1,3 +0,0 @@
-[console_scripts]
-make-dll-egg = eggutils.eggutils:wrap_main
-

dependency-check-core/src/test/resources/site-packages/eggutils-0.0.2-py2.7.egg-info/installed-files.txt

@@ -1,11 +0,0 @@
-../eggutils/__init__.py
-../eggutils/eggutils.py
-../eggutils/__init__.pyc
-../eggutils/eggutils.pyc
-./
-SOURCES.txt
-dependency_links.txt
-top_level.txt
-entry_points.txt
-PKG-INFO
-../../../../bin/make-dll-egg

dependency-check-core/src/test/resources/site-packages/eggutils-0.0.2-py2.7.egg-info/top_level.txt

@@ -1 +0,0 @@
-eggutils

src/main/config/checkstyle-header.txt

@@ -13,6 +13,6 @@
 ^ \* See the License for the specific language governing permissions and\s*$
 ^ \* limitations under the License\.\s*$
 ^ \*\s*$
-^ \* Copyright \(c\) 201[0-9] (Jeremy Long|Steve Springett)\. All Rights Reserved\.\s*$
+^ \* Copyright \(c\) 201[0-9] (Jeremy Long|Steve Springett|Institute for Defense Analyses)\. All Rights Reserved\.\s*$
 ^ \*/\s*$
 ^package