github-mirrors/DependencyCheck
1
0
Fork 0
mirror of https://github.com/ysoftdevs/DependencyCheck.git synced 2026-01-14 07:43:40 +01:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #302 from dwvisser/290-add-bz2-format

Browse Source 
Add bzip2 format to ArchiveAnalyzer
This commit is contained in:
Jeremy Long
2015-08-06 07:02:21 -04:00
parent 921b0eb229 1f074ff400
commit 707de56612
4 changed files with 214 additions and 188 deletions
Download Patch File Download Diff File Expand all files Collapse all files

328
dependency-check-core/src/main/java/org/owasp/dependencycheck/analyzer/ArchiveAnalyzer.java
View File

@@ -17,22 +17,6 @@
*/
package org.owasp.dependencycheck.analyzer;
import java.io.BufferedInputStream;
import java.io.BufferedOutputStream;
import java.io.File;
import java.io.FileFilter;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.Enumeration;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import org.apache.commons.compress.archivers.ArchiveEntry;
import org.apache.commons.compress.archivers.ArchiveInputStream;
import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;
@@ -40,6 +24,8 @@ import org.apache.commons.compress.archivers.zip.ZipArchiveEntry;
import org.apache.commons.compress.archivers.zip.ZipArchiveInputStream;
import org.apache.commons.compress.archivers.zip.ZipFile;
import org.apache.commons.compress.compressors.CompressorInputStream;
import org.apache.commons.compress.compressors.bzip2.BZip2CompressorInputStream;
import org.apache.commons.compress.compressors.bzip2.BZip2Utils;
import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream;
import org.apache.commons.compress.compressors.gzip.GzipUtils;
import org.owasp.dependencycheck.Engine;
@@ -52,6 +38,9 @@ import org.owasp.dependencycheck.utils.Settings;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.*;
import java.util.*;
/**
* <p>
* An analyzer that extracts files from archives and ensures any supported files contained within the archive are added to the
@@ -100,20 +89,21 @@ public class ArchiveAnalyzer extends AbstractFileTypeAnalyzer {
*/
private static final Set<String> ZIPPABLES = newHashSet("zip", "ear", "war", "jar", "sar", "apk", "nupkg");
/**
* The set of file extensions supported by this analyzer. Note for developers, any additions to this list will need to be
* explicitly handled in extractFiles().
* The set of file extensions supported by this analyzer. Note for developers, any additions to this list will need
* to be explicitly handled in {@link #extractFiles(File, File, Engine)}.
*/
private static final Set<String> EXTENSIONS = newHashSet("tar", "gz", "tgz");
private static final Set<String> EXTENSIONS = newHashSet("tar", "gz", "tgz", "bz2", "tbz2");
/**
* Detects files with extensions to remove from the engine's collection of dependencies.
*/
private static final FileFilter REMOVE_FROM_ANALYSIS = FileFilterBuilder.newInstance().addExtensions("zip", "tar", "gz", "tgz").build();
private static final FileFilter REMOVE_FROM_ANALYSIS =
FileFilterBuilder.newInstance().addExtensions("zip", "tar", "gz", "tgz", "bz2", "tbz2").build();
static {
final String additionalZipExt = Settings.getString(Settings.KEYS.ADDITIONAL_ZIP_EXTENSIONS);
if (additionalZipExt != null) {
final Set<String> ext = new HashSet<String>(Arrays.asList(additionalZipExt));
final Set<String> ext = new HashSet<String>(Collections.singletonList(additionalZipExt));
ZIPPABLES.addAll(ext);
}
EXTENSIONS.addAll(ZIPPABLES);
@@ -205,7 +195,7 @@ public class ArchiveAnalyzer extends AbstractFileTypeAnalyzer {
* and added to the list of dependencies within the engine.
*
* @param dependency the dependency to analyze
* @param engine the engine scanning
* @param engine the engine scanning
* @throws AnalysisException thrown if there is an analysis exception
*/
@Override
@@ -215,15 +205,8 @@ public class ArchiveAnalyzer extends AbstractFileTypeAnalyzer {
extractFiles(f, tmpDir, engine);
//make a copy
List<Dependency> dependencies = new ArrayList<Dependency>(engine.getDependencies());
engine.scan(tmpDir);
List<Dependency> newDependencies = engine.getDependencies();
if (dependencies.size() != newDependencies.size()) {
//get the new dependencies
final Set<Dependency> dependencySet = new HashSet<Dependency>();
dependencySet.addAll(newDependencies);
dependencySet.removeAll(dependencies);
final Set<Dependency> dependencySet = findMoreDependencies(engine, tmpDir);
if (!dependencySet.isEmpty()) {
for (Dependency d : dependencySet) {
//fix the dependency's display name and path
final String displayPath = String.format("%s%s",
@@ -245,41 +228,66 @@ public class ArchiveAnalyzer extends AbstractFileTypeAnalyzer {
}
}
if (REMOVE_FROM_ANALYSIS.accept(dependency.getActualFile())) {
if (ZIP_FILTER.accept(dependency.getActualFile()) && isZipFileActuallyJarFile(dependency)) {
final File tdir = getNextTempDirectory();
final String fileName = dependency.getFileName();
LOGGER.info(String.format("The zip file '%s' appears to be a JAR file, making a copy and analyzing it as a JAR.", fileName));
final File tmpLoc = new File(tdir, fileName.substring(0, fileName.length() - 3) + "jar");
try {
org.apache.commons.io.FileUtils.copyFile(tdir, tmpLoc);
dependencies = new ArrayList<Dependency>(engine.getDependencies());
engine.scan(tmpLoc);
newDependencies = engine.getDependencies();
if (dependencies.size() != newDependencies.size()) {
//get the new dependencies
final Set<Dependency> dependencySet = new HashSet<Dependency>();
dependencySet.addAll(newDependencies);
dependencySet.removeAll(dependencies);
if (dependencySet.size() != 1) {
LOGGER.info("Deep copy of ZIP to JAR file resulted in more then one dependency?");
}
for (Dependency d : dependencySet) {
//fix the dependency's display name and path
d.setFilePath(dependency.getFilePath());
d.setDisplayFileName(dependency.getFileName());
}
}
} catch (IOException ex) {
LOGGER.debug("Unable to perform deep copy on '{}'", dependency.getActualFile().getPath(), ex);
}
}
addDisguisedJarsToDependencies(dependency, engine);
engine.getDependencies().remove(dependency);
}
Collections.sort(engine.getDependencies());
}
private void addDisguisedJarsToDependencies(Dependency dependency, Engine engine) throws AnalysisException {
if (ZIP_FILTER.accept(dependency.getActualFile()) && isZipFileActuallyJarFile(dependency)) {
final File tdir = getNextTempDirectory();
final String fileName = dependency.getFileName();
LOGGER.info(String.format("The zip file '%s' appears to be a JAR file, making a copy and analyzing it as a JAR.", fileName));
final File tmpLoc = new File(tdir, fileName.substring(0, fileName.length() - 3) + "jar");
try {
org.apache.commons.io.FileUtils.copyFile(tdir, tmpLoc);
final Set<Dependency> dependencySet = findMoreDependencies(engine, tmpLoc);
if (!dependencySet.isEmpty()) {
if (dependencySet.size() != 1) {
LOGGER.info("Deep copy of ZIP to JAR file resulted in more than one dependency?");
}
for (Dependency d : dependencySet) {
//fix the dependency's display name and path
d.setFilePath(dependency.getFilePath());
d.setDisplayFileName(dependency.getFileName());
}
}
} catch (IOException ex) {
LOGGER.debug("Unable to perform deep copy on '{}'", dependency.getActualFile().getPath(), ex);
}
}
}
private static final Set<Dependency> EMPTY_DEPENDENCY_SET = Collections.emptySet();
/**
* Scan the given file/folder, and return any new dependencies found.
*
* @param engine used to scan
* @param file target of scanning
* @return any dependencies that weren't known to the engine before
*/
private static Set<Dependency> findMoreDependencies(Engine engine, File file) {
List<Dependency> before = new ArrayList<Dependency>(engine.getDependencies());
engine.scan(file);
List<Dependency> after = engine.getDependencies();
final boolean sizeChanged = before.size() != after.size();
final Set<Dependency> newDependencies;
if (sizeChanged) {
//get the new dependencies
newDependencies = new HashSet<Dependency>();
newDependencies.addAll(after);
newDependencies.removeAll(before);
} else {
newDependencies = EMPTY_DEPENDENCY_SET;
}
return newDependencies;
}
/**
* Retrieves the next temporary directory to extract an archive too.
*
@@ -303,47 +311,47 @@ public class ArchiveAnalyzer extends AbstractFileTypeAnalyzer {
/**
* Extracts the contents of an archive into the specified directory.
*
* @param archive an archive file such as a WAR or EAR
* @param archive an archive file such as a WAR or EAR
* @param destination a directory to extract the contents to
* @param engine the scanning engine
* @param engine the scanning engine
* @throws AnalysisException thrown if the archive is not found
*/
private void extractFiles(File archive, File destination, Engine engine) throws AnalysisException {
if (archive == null || destination == null) {
return;
}
FileInputStream fis = null;
try {
fis = new FileInputStream(archive);
} catch (FileNotFoundException ex) {
LOGGER.debug("", ex);
throw new AnalysisException("Archive file was not found.", ex);
}
final String archiveExt = FileUtils.getFileExtension(archive.getName()).toLowerCase();
try {
if (ZIPPABLES.contains(archiveExt)) {
extractArchive(new ZipArchiveInputStream(new BufferedInputStream(fis)), destination, engine);
} else if ("tar".equals(archiveExt)) {
extractArchive(new TarArchiveInputStream(new BufferedInputStream(fis)), destination, engine);
} else if ("gz".equals(archiveExt) || "tgz".equals(archiveExt)) {
final String uncompressedName = GzipUtils.getUncompressedFilename(archive.getName());
final File f = new File(destination, uncompressedName);
if (engine.accept(f)) {
decompressFile(new GzipCompressorInputStream(new BufferedInputStream(fis)), f);
}
}
} catch (ArchiveExtractionException ex) {
LOGGER.warn("Exception extracting archive '{}'.", archive.getName());
LOGGER.debug("", ex);
} catch (IOException ex) {
LOGGER.warn("Exception reading archive '{}'.", archive.getName());
LOGGER.debug("", ex);
} finally {
if (archive != null && destination != null) {
FileInputStream fis;
try {
fis.close();
} catch (IOException ex) {
fis = new FileInputStream(archive);
} catch (FileNotFoundException ex) {
LOGGER.debug("", ex);
throw new AnalysisException("Archive file was not found.", ex);
}
final String archiveExt = FileUtils.getFileExtension(archive.getName()).toLowerCase();
try {
if (ZIPPABLES.contains(archiveExt)) {
extractArchive(new ZipArchiveInputStream(new BufferedInputStream(fis)), destination, engine);
} else if ("tar".equals(archiveExt)) {
extractArchive(new TarArchiveInputStream(new BufferedInputStream(fis)), destination, engine);
} else if ("gz".equals(archiveExt) || "tgz".equals(archiveExt)) {
final String uncompressedName = GzipUtils.getUncompressedFilename(archive.getName());
final File f = new File(destination, uncompressedName);
if (engine.accept(f)) {
decompressFile(new GzipCompressorInputStream(new BufferedInputStream(fis)), f);
}
} else if ("bz2".equals(archiveExt) || "tbz2".equals(archiveExt)) {
final String uncompressedName = BZip2Utils.getUncompressedFilename(archive.getName());
final File f = new File(destination, uncompressedName);
if (engine.accept(f)) {
decompressFile(new BZip2CompressorInputStream(new BufferedInputStream(fis)), f);
}
}
} catch (ArchiveExtractionException ex) {
LOGGER.warn("Exception extracting archive '{}'.", archive.getName());
LOGGER.debug("", ex);
} catch (IOException ex) {
LOGGER.warn("Exception reading archive '{}'.", archive.getName());
LOGGER.debug("", ex);
} finally {
close(fis);
}
}
}
@@ -351,84 +359,63 @@ public class ArchiveAnalyzer extends AbstractFileTypeAnalyzer {
/**
* Extracts files from an archive.
*
* @param input the archive to extract files from
* @param input the archive to extract files from
* @param destination the location to write the files too
* @param engine the dependency-check engine
* @param engine the dependency-check engine
* @throws ArchiveExtractionException thrown if there is an exception extracting files from the archive
*/
private void extractArchive(ArchiveInputStream input, File destination, Engine engine) throws ArchiveExtractionException {
ArchiveEntry entry;
try {
while ((entry = input.getNextEntry()) != null) {
final File file = new File(destination, entry.getName());
if (entry.isDirectory()) {
final File d = new File(destination, entry.getName());
if (!d.exists()) {
if (!d.mkdirs()) {
final String msg = String.format("Unable to create directory '%s'.", d.getAbsolutePath());
throw new AnalysisException(msg);
}
}
} else {
final File file = new File(destination, entry.getName());
if (engine.accept(file)) {
LOGGER.debug("Extracting '{}'", file.getPath());
BufferedOutputStream bos = null;
FileOutputStream fos = null;
try {
final File parent = file.getParentFile();
if (!parent.isDirectory()) {
if (!parent.mkdirs()) {
final String msg = String.format("Unable to build directory '%s'.", parent.getAbsolutePath());
throw new AnalysisException(msg);
}
}
fos = new FileOutputStream(file);
bos = new BufferedOutputStream(fos, BUFFER_SIZE);
int count;
final byte[] data = new byte[BUFFER_SIZE];
while ((count = input.read(data, 0, BUFFER_SIZE)) != -1) {
bos.write(data, 0, count);
}
bos.flush();
} catch (FileNotFoundException ex) {
LOGGER.debug("", ex);
final String msg = String.format("Unable to find file '%s'.", file.getName());
throw new AnalysisException(msg, ex);
} catch (IOException ex) {
LOGGER.debug("", ex);
final String msg = String.format("IO Exception while parsing file '%s'.", file.getName());
throw new AnalysisException(msg, ex);
} finally {
if (bos != null) {
try {
bos.close();
} catch (IOException ex) {
LOGGER.trace("", ex);
}
}
if (fos != null) {
try {
fos.close();
} catch (IOException ex) {
LOGGER.trace("", ex);
}
}
}
if (!file.exists() && !file.mkdirs()) {
final String msg = String.format("Unable to create directory '%s'.", file.getAbsolutePath());
throw new AnalysisException(msg);
}
} else if (engine.accept(file)) {
extractAcceptedFile(input, file);
}
}
} catch (IOException ex) {
throw new ArchiveExtractionException(ex);
} catch (Throwable ex) {
throw new ArchiveExtractionException(ex);
} finally {
if (input != null) {
try {
input.close();
} catch (IOException ex) {
LOGGER.trace("", ex);
close(input);
}
}
private static void extractAcceptedFile(ArchiveInputStream input, File file) throws AnalysisException {
LOGGER.debug("Extracting '{}'", file.getPath());
BufferedOutputStream bos = null;
FileOutputStream fos = null;
try {
final File parent = file.getParentFile();
if (!parent.isDirectory()) {
if (!parent.mkdirs()) {
final String msg = String.format("Unable to build directory '%s'.", parent.getAbsolutePath());
throw new AnalysisException(msg);
}
}
fos = new FileOutputStream(file);
bos = new BufferedOutputStream(fos, BUFFER_SIZE);
int count;
final byte[] data = new byte[BUFFER_SIZE];
while ((count = input.read(data, 0, BUFFER_SIZE)) != -1) {
bos.write(data, 0, count);
}
bos.flush();
} catch (FileNotFoundException ex) {
LOGGER.debug("", ex);
final String msg = String.format("Unable to find file '%s'.", file.getName());
throw new AnalysisException(msg, ex);
} catch (IOException ex) {
LOGGER.debug("", ex);
final String msg = String.format("IO Exception while parsing file '%s'.", file.getName());
throw new AnalysisException(msg, ex);
} finally {
close(bos);
close(fos);
}
}
@@ -436,7 +423,7 @@ public class ArchiveAnalyzer extends AbstractFileTypeAnalyzer {
* Decompresses a file.
*
* @param inputStream the compressed file
* @param outputFile the location to write the decompressed file
* @param outputFile the location to write the decompressed file
* @throws ArchiveExtractionException thrown if there is an exception decompressing the file
*/
private void decompressFile(CompressorInputStream inputStream, File outputFile) throws ArchiveExtractionException {
@@ -445,7 +432,7 @@ public class ArchiveAnalyzer extends AbstractFileTypeAnalyzer {
try {
out = new FileOutputStream(outputFile);
final byte[] buffer = new byte[BUFFER_SIZE];
int n = 0;
int n; // = 0
while (-1 != (n = inputStream.read(buffer))) {
out.write(buffer, 0, n);
}
@@ -456,12 +443,21 @@ public class ArchiveAnalyzer extends AbstractFileTypeAnalyzer {
LOGGER.debug("", ex);
throw new ArchiveExtractionException(ex);
} finally {
if (out != null) {
try {
out.close();
} catch (IOException ex) {
LOGGER.trace("", ex);
}
close(out);
}
}
/**
* Close the given {@link Closeable} instance, ignoring nulls, and logging any thrown {@link IOException}.
*
* @param closeable to be closed
*/
private static void close(Closeable closeable){
if (null != closeable) {
try {
closeable.close();
} catch (IOException ex) {
LOGGER.trace("", ex);
}
}
}

74
dependency-check-core/src/test/java/org/owasp/dependencycheck/analyzer/ArchiveAnalyzerIntegrationTest.java
View File

@@ -51,6 +51,8 @@ public class ArchiveAnalyzerIntegrationTest extends AbstractDatabaseTestCase {
expResult.add("tar");
expResult.add("gz");
expResult.add("tgz");
expResult.add("bz2");
expResult.add("tbz2");
for (String ext : expResult) {
assertTrue(ext, instance.accept(new File("test." + ext)));
}
@@ -197,28 +199,31 @@ public class ArchiveAnalyzerIntegrationTest extends AbstractDatabaseTestCase {
}
}
// /**
// * Test of analyze method, of class ArchiveAnalyzer.
// */
// @Test
// public void testNestedZipFolder() throws Exception {
// ArchiveAnalyzer instance = new ArchiveAnalyzer();
// try {
// instance.initialize();
//
// File file = new File(this.getClass().getClassLoader().getResource("nested.zip").getPath());
// Settings.setBoolean(Settings.KEYS.AUTO_UPDATE, false);
// Settings.setBoolean(Settings.KEYS.ANALYZER_NEXUS_ENABLED, false);
// Settings.setBoolean(Settings.KEYS.ANALYZER_CENTRAL_ENABLED, false);
// Engine engine = new Engine();
//
// engine.scan(file);
// engine.analyzeDependencies();
//
// } finally {
// instance.close();
// }
// }
/**
* Test of analyze method, of class ArchiveAnalyzer.
*/
@Test
public void testAnalyzeTarBz2() throws Exception {
ArchiveAnalyzer instance = new ArchiveAnalyzer();
instance.accept(new File("zip")); //ensure analyzer is "enabled"
try {
instance.initialize();
File file = BaseTest.getResourceAsFile(this, "file.tar.bz2");
Settings.setBoolean(Settings.KEYS.AUTO_UPDATE, false);
Settings.setBoolean(Settings.KEYS.ANALYZER_NEXUS_ENABLED, false);
Settings.setBoolean(Settings.KEYS.ANALYZER_CENTRAL_ENABLED, false);
Engine engine = new Engine();
int initial_size = engine.getDependencies().size();
engine.scan(file);
engine.analyzeDependencies();
int ending_size = engine.getDependencies().size();
engine.cleanup();
assertTrue(initial_size < ending_size);
} finally {
instance.close();
}
}
/**
* Test of analyze method, of class ArchiveAnalyzer.
*/
@@ -248,6 +253,31 @@ public class ArchiveAnalyzerIntegrationTest extends AbstractDatabaseTestCase {
}
}
/**
* Test of analyze method, of class ArchiveAnalyzer.
*/
@Test
public void testAnalyzeTbz2() throws Exception {
ArchiveAnalyzer instance = new ArchiveAnalyzer();
instance.accept(new File("zip")); //ensure analyzer is "enabled"
try {
instance.initialize();
File file = BaseTest.getResourceAsFile(this, "file.tbz2");
Settings.setBoolean(Settings.KEYS.AUTO_UPDATE, false);
Settings.setBoolean(Settings.KEYS.ANALYZER_NEXUS_ENABLED, false);
Settings.setBoolean(Settings.KEYS.ANALYZER_CENTRAL_ENABLED, false);
Engine engine = new Engine();
int initial_size = engine.getDependencies().size();
engine.scan(file);
engine.analyzeDependencies();
int ending_size = engine.getDependencies().size();
engine.cleanup();
assertTrue(initial_size < ending_size);
} finally {
instance.close();
}
}
/**
* Test of analyze method, of class ArchiveAnalyzer.
*/

BIN
dependency-check-core/src/test/resources/file.tar.bz2 Normal file
View File

Binary file not shown.

BIN
dependency-check-core/src/test/resources/file.tbz2 Normal file
View File

Binary file not shown.