Merge pull request #302 from dwvisser/290-add-bz2-format

Add bzip2 format to ArchiveAnalyzer

dependency-check-core/src/main/java/org/owasp/dependencycheck/analyzer/ArchiveAnalyzer.java

@@ -17,22 +17,6 @@
  */
 package org.owasp.dependencycheck.analyzer;
 
-import java.io.BufferedInputStream;
-import java.io.BufferedOutputStream;
-import java.io.File;
-import java.io.FileFilter;
-import java.io.FileInputStream;
-import java.io.FileNotFoundException;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.Enumeration;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-
 import org.apache.commons.compress.archivers.ArchiveEntry;
 import org.apache.commons.compress.archivers.ArchiveInputStream;
 import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;
@@ -40,6 +24,8 @@ import org.apache.commons.compress.archivers.zip.ZipArchiveEntry;
 import org.apache.commons.compress.archivers.zip.ZipArchiveInputStream;
 import org.apache.commons.compress.archivers.zip.ZipFile;
 import org.apache.commons.compress.compressors.CompressorInputStream;
+import org.apache.commons.compress.compressors.bzip2.BZip2CompressorInputStream;
+import org.apache.commons.compress.compressors.bzip2.BZip2Utils;
 import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream;
 import org.apache.commons.compress.compressors.gzip.GzipUtils;
 import org.owasp.dependencycheck.Engine;
@@ -52,6 +38,9 @@ import org.owasp.dependencycheck.utils.Settings;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import java.io.*;
+import java.util.*;
+
 /**
  * <p>
  * An analyzer that extracts files from archives and ensures any supported files contained within the archive are added to the
@@ -100,20 +89,21 @@ public class ArchiveAnalyzer extends AbstractFileTypeAnalyzer {
      */
     private static final Set<String> ZIPPABLES = newHashSet("zip", "ear", "war", "jar", "sar", "apk", "nupkg");
     /**
-     * The set of file extensions supported by this analyzer. Note for developers, any additions to this list will need to be
+     * The set of file extensions supported by this analyzer. Note for developers, any additions to this list will need
-     * explicitly handled in extractFiles().
+     * to be explicitly handled in {@link #extractFiles(File, File, Engine)}.
      */
-    private static final Set<String> EXTENSIONS = newHashSet("tar", "gz", "tgz");
+    private static final Set<String> EXTENSIONS = newHashSet("tar", "gz", "tgz", "bz2", "tbz2");
 
     /**
      * Detects files with extensions to remove from the engine's collection of dependencies.
      */
-    private static final FileFilter REMOVE_FROM_ANALYSIS = FileFilterBuilder.newInstance().addExtensions("zip", "tar", "gz", "tgz").build();
+    private static final FileFilter REMOVE_FROM_ANALYSIS =
+            FileFilterBuilder.newInstance().addExtensions("zip", "tar", "gz", "tgz", "bz2", "tbz2").build();
 
     static {
         final String additionalZipExt = Settings.getString(Settings.KEYS.ADDITIONAL_ZIP_EXTENSIONS);
         if (additionalZipExt != null) {
-            final Set<String> ext = new HashSet<String>(Arrays.asList(additionalZipExt));
+            final Set<String> ext = new HashSet<String>(Collections.singletonList(additionalZipExt));
             ZIPPABLES.addAll(ext);
         }
         EXTENSIONS.addAll(ZIPPABLES);
@@ -215,15 +205,8 @@ public class ArchiveAnalyzer extends AbstractFileTypeAnalyzer {
         extractFiles(f, tmpDir, engine);
 
         //make a copy
-        List<Dependency> dependencies = new ArrayList<Dependency>(engine.getDependencies());
+        final Set<Dependency> dependencySet = findMoreDependencies(engine, tmpDir);
-        engine.scan(tmpDir);
+        if (!dependencySet.isEmpty()) {
-        List<Dependency> newDependencies = engine.getDependencies();
-        if (dependencies.size() != newDependencies.size()) {
-            //get the new dependencies
-            final Set<Dependency> dependencySet = new HashSet<Dependency>();
-            dependencySet.addAll(newDependencies);
-            dependencySet.removeAll(dependencies);
-
             for (Dependency d : dependencySet) {
                 //fix the dependency's display name and path
                 final String displayPath = String.format("%s%s",
@@ -245,6 +228,13 @@ public class ArchiveAnalyzer extends AbstractFileTypeAnalyzer {
             }
         }
         if (REMOVE_FROM_ANALYSIS.accept(dependency.getActualFile())) {
+            addDisguisedJarsToDependencies(dependency, engine);
+            engine.getDependencies().remove(dependency);
+        }
+        Collections.sort(engine.getDependencies());
+    }
+
+    private void addDisguisedJarsToDependencies(Dependency dependency, Engine engine) throws AnalysisException {
         if (ZIP_FILTER.accept(dependency.getActualFile()) && isZipFileActuallyJarFile(dependency)) {
             final File tdir = getNextTempDirectory();
             final String fileName = dependency.getFileName();
@@ -254,16 +244,10 @@ public class ArchiveAnalyzer extends AbstractFileTypeAnalyzer {
             final File tmpLoc = new File(tdir, fileName.substring(0, fileName.length() - 3) + "jar");
             try {
                 org.apache.commons.io.FileUtils.copyFile(tdir, tmpLoc);
-                    dependencies = new ArrayList<Dependency>(engine.getDependencies());
+                final Set<Dependency> dependencySet = findMoreDependencies(engine, tmpLoc);
-                    engine.scan(tmpLoc);
+                if (!dependencySet.isEmpty()) {
-                    newDependencies = engine.getDependencies();
-                    if (dependencies.size() != newDependencies.size()) {
-                        //get the new dependencies
-                        final Set<Dependency> dependencySet = new HashSet<Dependency>();
-                        dependencySet.addAll(newDependencies);
-                        dependencySet.removeAll(dependencies);
                     if (dependencySet.size() != 1) {
-                            LOGGER.info("Deep copy of ZIP to JAR file resulted in more then one dependency?");
+                        LOGGER.info("Deep copy of ZIP to JAR file resulted in more than one dependency?");
                     }
                     for (Dependency d : dependencySet) {
                         //fix the dependency's display name and path
@@ -275,10 +259,34 @@ public class ArchiveAnalyzer extends AbstractFileTypeAnalyzer {
                 LOGGER.debug("Unable to perform deep copy on '{}'", dependency.getActualFile().getPath(), ex);
             }
         }
-            engine.getDependencies().remove(dependency);
     }
-        Collections.sort(engine.getDependencies());
+
+    private static final Set<Dependency> EMPTY_DEPENDENCY_SET = Collections.emptySet();
+
+    /**
+     * Scan the given file/folder, and return any new dependencies found.
+     *
+     * @param engine used to scan
+     * @param file target of scanning
+     * @return any dependencies that weren't known to the engine before
+     */
+    private static Set<Dependency> findMoreDependencies(Engine engine, File file) {
+        List<Dependency> before = new ArrayList<Dependency>(engine.getDependencies());
+        engine.scan(file);
+        List<Dependency> after = engine.getDependencies();
+        final boolean sizeChanged = before.size() != after.size();
+        final Set<Dependency> newDependencies;
+        if (sizeChanged) {
+            //get the new dependencies
+            newDependencies = new HashSet<Dependency>();
+            newDependencies.addAll(after);
+            newDependencies.removeAll(before);
+        } else {
+            newDependencies = EMPTY_DEPENDENCY_SET;
         }
+        return newDependencies;
+    }
+
 
     /**
      * Retrieves the next temporary directory to extract an archive too.
@@ -309,11 +317,8 @@ public class ArchiveAnalyzer extends AbstractFileTypeAnalyzer {
      * @throws AnalysisException thrown if the archive is not found
      */
     private void extractFiles(File archive, File destination, Engine engine) throws AnalysisException {
-        if (archive == null || destination == null) {
+        if (archive != null && destination != null) {
-            return;
+            FileInputStream fis;
-        }
-
-        FileInputStream fis = null;
             try {
                 fis = new FileInputStream(archive);
             } catch (FileNotFoundException ex) {
@@ -332,6 +337,12 @@ public class ArchiveAnalyzer extends AbstractFileTypeAnalyzer {
                     if (engine.accept(f)) {
                         decompressFile(new GzipCompressorInputStream(new BufferedInputStream(fis)), f);
                     }
+                } else if ("bz2".equals(archiveExt) || "tbz2".equals(archiveExt)) {
+                    final String uncompressedName = BZip2Utils.getUncompressedFilename(archive.getName());
+                    final File f = new File(destination, uncompressedName);
+                    if (engine.accept(f)) {
+                        decompressFile(new BZip2CompressorInputStream(new BufferedInputStream(fis)), f);
+                    }
                 }
             } catch (ArchiveExtractionException ex) {
                 LOGGER.warn("Exception extracting archive '{}'.", archive.getName());
@@ -340,10 +351,7 @@ public class ArchiveAnalyzer extends AbstractFileTypeAnalyzer {
                 LOGGER.warn("Exception reading archive '{}'.", archive.getName());
                 LOGGER.debug("", ex);
             } finally {
-            try {
+                close(fis);
-                fis.close();
-            } catch (IOException ex) {
-                LOGGER.debug("", ex);
             }
         }
     }
@@ -360,17 +368,24 @@ public class ArchiveAnalyzer extends AbstractFileTypeAnalyzer {
         ArchiveEntry entry;
         try {
             while ((entry = input.getNextEntry()) != null) {
+                final File file = new File(destination, entry.getName());
                 if (entry.isDirectory()) {
-                    final File d = new File(destination, entry.getName());
+                    if (!file.exists() && !file.mkdirs()) {
-                    if (!d.exists()) {
+                        final String msg = String.format("Unable to create directory '%s'.", file.getAbsolutePath());
-                        if (!d.mkdirs()) {
-                            final String msg = String.format("Unable to create directory '%s'.", d.getAbsolutePath());
                         throw new AnalysisException(msg);
                     }
+                } else if (engine.accept(file)) {
+                    extractAcceptedFile(input, file);
                 }
-                } else {
+            }
-                    final File file = new File(destination, entry.getName());
+        } catch (Throwable ex) {
-                    if (engine.accept(file)) {
+            throw new ArchiveExtractionException(ex);
+        } finally {
+            close(input);
+        }
+    }
+
+    private static void extractAcceptedFile(ArchiveInputStream input, File file) throws AnalysisException {
         LOGGER.debug("Extracting '{}'", file.getPath());
         BufferedOutputStream bos = null;
         FileOutputStream fos = null;
@@ -399,36 +414,8 @@ public class ArchiveAnalyzer extends AbstractFileTypeAnalyzer {
             final String msg = String.format("IO Exception while parsing file '%s'.", file.getName());
             throw new AnalysisException(msg, ex);
         } finally {
-                            if (bos != null) {
+            close(bos);
-                                try {
+            close(fos);
-                                    bos.close();
-                                } catch (IOException ex) {
-                                    LOGGER.trace("", ex);
-                                }
-                            }
-                            if (fos != null) {
-                                try {
-                                    fos.close();
-                                } catch (IOException ex) {
-                                    LOGGER.trace("", ex);
-                                }
-                            }
-                        }
-                    }
-                }
-            }
-        } catch (IOException ex) {
-            throw new ArchiveExtractionException(ex);
-        } catch (Throwable ex) {
-            throw new ArchiveExtractionException(ex);
-        } finally {
-            if (input != null) {
-                try {
-                    input.close();
-                } catch (IOException ex) {
-                    LOGGER.trace("", ex);
-                }
-            }
         }
     }
 
@@ -445,7 +432,7 @@ public class ArchiveAnalyzer extends AbstractFileTypeAnalyzer {
         try {
             out = new FileOutputStream(outputFile);
             final byte[] buffer = new byte[BUFFER_SIZE];
-            int n = 0;
+            int n; // = 0
             while (-1 != (n = inputStream.read(buffer))) {
                 out.write(buffer, 0, n);
             }
@@ -456,15 +443,24 @@ public class ArchiveAnalyzer extends AbstractFileTypeAnalyzer {
             LOGGER.debug("", ex);
             throw new ArchiveExtractionException(ex);
         } finally {
-            if (out != null) {
+            close(out);
+        }
+    }
+
+    /**
+     * Close the given {@link Closeable} instance, ignoring nulls, and logging any thrown {@link IOException}.
+     *
+     * @param closeable to be closed
+     */
+    private static void close(Closeable closeable){
+        if (null != closeable) {
             try {
-                    out.close();
+                closeable.close();
             } catch (IOException ex) {
                 LOGGER.trace("", ex);
             }
         }
     }
-    }
 
     /**
      * Attempts to determine if a zip file is actually a JAR file.

dependency-check-core/src/test/java/org/owasp/dependencycheck/analyzer/ArchiveAnalyzerIntegrationTest.java

@@ -51,6 +51,8 @@ public class ArchiveAnalyzerIntegrationTest extends AbstractDatabaseTestCase {
         expResult.add("tar");
         expResult.add("gz");
         expResult.add("tgz");
+        expResult.add("bz2");
+        expResult.add("tbz2");
         for (String ext : expResult) {
             assertTrue(ext, instance.accept(new File("test." + ext)));
         }
@@ -197,28 +199,31 @@ public class ArchiveAnalyzerIntegrationTest extends AbstractDatabaseTestCase {
         }
     }
 
-//    /**
+    /**
-//     * Test of analyze method, of class ArchiveAnalyzer.
+     * Test of analyze method, of class ArchiveAnalyzer.
-//     */
+     */
-//    @Test
+    @Test
-//    public void testNestedZipFolder() throws Exception {
+    public void testAnalyzeTarBz2() throws Exception {
-//        ArchiveAnalyzer instance = new ArchiveAnalyzer();
+        ArchiveAnalyzer instance = new ArchiveAnalyzer();
-//        try {
+        instance.accept(new File("zip")); //ensure analyzer is "enabled"
-//            instance.initialize();
+        try {
-//
+            instance.initialize();
-//            File file = new File(this.getClass().getClassLoader().getResource("nested.zip").getPath());
+            File file = BaseTest.getResourceAsFile(this, "file.tar.bz2");
-//            Settings.setBoolean(Settings.KEYS.AUTO_UPDATE, false);
+            Settings.setBoolean(Settings.KEYS.AUTO_UPDATE, false);
-//            Settings.setBoolean(Settings.KEYS.ANALYZER_NEXUS_ENABLED, false);
+            Settings.setBoolean(Settings.KEYS.ANALYZER_NEXUS_ENABLED, false);
-//            Settings.setBoolean(Settings.KEYS.ANALYZER_CENTRAL_ENABLED, false);
+            Settings.setBoolean(Settings.KEYS.ANALYZER_CENTRAL_ENABLED, false);
-//            Engine engine = new Engine();
+            Engine engine = new Engine();
-//
+            int initial_size = engine.getDependencies().size();
-//            engine.scan(file);
+            engine.scan(file);
-//            engine.analyzeDependencies();
+            engine.analyzeDependencies();
-//
+            int ending_size = engine.getDependencies().size();
-//        } finally {
+            engine.cleanup();
-//            instance.close();
+            assertTrue(initial_size < ending_size);
-//        }
+        } finally {
-//    }
+            instance.close();
+        }
+    }
+
     /**
      * Test of analyze method, of class ArchiveAnalyzer.
      */
@@ -248,6 +253,31 @@ public class ArchiveAnalyzerIntegrationTest extends AbstractDatabaseTestCase {
         }
     }
 
+    /**
+     * Test of analyze method, of class ArchiveAnalyzer.
+     */
+    @Test
+    public void testAnalyzeTbz2() throws Exception {
+        ArchiveAnalyzer instance = new ArchiveAnalyzer();
+        instance.accept(new File("zip")); //ensure analyzer is "enabled"
+        try {
+            instance.initialize();
+            File file = BaseTest.getResourceAsFile(this, "file.tbz2");
+            Settings.setBoolean(Settings.KEYS.AUTO_UPDATE, false);
+            Settings.setBoolean(Settings.KEYS.ANALYZER_NEXUS_ENABLED, false);
+            Settings.setBoolean(Settings.KEYS.ANALYZER_CENTRAL_ENABLED, false);
+            Engine engine = new Engine();
+            int initial_size = engine.getDependencies().size();
+            engine.scan(file);
+            engine.analyzeDependencies();
+            int ending_size = engine.getDependencies().size();
+            engine.cleanup();
+            assertTrue(initial_size < ending_size);
+        } finally {
+            instance.close();
+        }
+    }
+
     /**
      * Test of analyze method, of class ArchiveAnalyzer.
      */