From ee969a5ed9676926e270ea2b02ba2aad6df6fe63 Mon Sep 17 00:00:00 2001 From: Jeremy Long Date: Fri, 24 Jul 2015 05:36:32 -0400 Subject: [PATCH] manually merged PR #278 Former-commit-id: 7811e5832e680589e5b1734e775ee8cab92dd30f --- .../main/resources/dependencycheck.properties | 5 +- .../test/resources/dependencycheck.properties | 3 + dependency-check-gradle/README.md | 12 ++-- .../dependencycheck/utils/Downloader.java | 60 +++++++++++++------ .../owasp/dependencycheck/utils/Settings.java | 5 ++ .../test/resources/dependencycheck.properties | 3 + 6 files changed, 64 insertions(+), 24 deletions(-) diff --git a/dependency-check-core/src/main/resources/dependencycheck.properties b/dependency-check-core/src/main/resources/dependencycheck.properties index 89f4c5e83..ab178a0f3 100644 --- a/dependency-check-core/src/main/resources/dependencycheck.properties +++ b/dependency-check-core/src/main/resources/dependencycheck.properties @@ -74,4 +74,7 @@ analyzer.central.enabled=true analyzer.central.url=http://search.maven.org/solrsearch/select # the number of nested archives that will be searched. -archive.scan.depth=3 \ No newline at end of file +archive.scan.depth=3 + +# use HEAD (default) or GET as HTTP request method for query timestamp +downloader.quick.query.timestamp=true diff --git a/dependency-check-core/src/test/resources/dependencycheck.properties b/dependency-check-core/src/test/resources/dependencycheck.properties index bac90d541..b7cdffb69 100644 --- a/dependency-check-core/src/test/resources/dependencycheck.properties +++ b/dependency-check-core/src/test/resources/dependencycheck.properties @@ -73,3 +73,6 @@ analyzer.nexus.proxy=true # the URL for searching search.maven.org for SHA-1 and whether it's enabled analyzer.central.enabled=true analyzer.central.url=http://search.maven.org/solrsearch/select + +# use HEAD (default) or GET as HTTP request method for query timestamp +downloader.quick.query.timestamp=true diff --git a/dependency-check-gradle/README.md b/dependency-check-gradle/README.md index 242f75a07..ead2a50af 100644 --- a/dependency-check-gradle/README.md +++ b/dependency-check-gradle/README.md @@ -23,7 +23,7 @@ buildscript { mavenCentral() } dependencies { - classpath 'com.thoughtworks.tools:dependency-check:0.0.4' + classpath 'com.thoughtworks.tools:dependency-check:0.0.5' } } ``` @@ -38,7 +38,7 @@ apply plugin: 'dependency.check' ```groovy plugins { - id "dependency.check" version "0.0.4" + id "dependency.check" version "0.0.5" } ``` @@ -52,7 +52,7 @@ buildscript { } } dependencies { - classpath "gradle.plugin.com.tools.security:dependency-check:0.0.4" + classpath "gradle.plugin.com.tools.security:dependency-check:0.0.5" } } @@ -73,7 +73,7 @@ buildscript { } dependencies { classpath( - 'com.tools.security:dependency-check:0.0.4' + 'com.tools.security:dependency-check:0.0.5' ) } } @@ -127,7 +127,7 @@ buildscript { mavenCentral() } dependencies { - classpath "gradle.plugin.com.tools.security:dependency-check:0.0.4" + classpath "gradle.plugin.com.tools.security:dependency-check:0.0.5" } } @@ -144,7 +144,7 @@ buildscript { mavenCentral() } dependencies { - classpath "gradle.plugin.com.tools.security:dependency-check:0.0.4" + classpath "gradle.plugin.com.tools.security:dependency-check:0.0.5" } } diff --git a/dependency-check-utils/src/main/java/org/owasp/dependencycheck/utils/Downloader.java b/dependency-check-utils/src/main/java/org/owasp/dependencycheck/utils/Downloader.java index 345d28adf..90859d4f1 100644 --- a/dependency-check-utils/src/main/java/org/owasp/dependencycheck/utils/Downloader.java +++ b/dependency-check-utils/src/main/java/org/owasp/dependencycheck/utils/Downloader.java @@ -20,11 +20,7 @@ package org.owasp.dependencycheck.utils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.BufferedOutputStream; -import java.io.File; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.InputStream; +import java.io.*; import java.net.HttpURLConnection; import java.net.URISyntaxException; import java.net.URL; @@ -32,6 +28,10 @@ import java.security.InvalidAlgorithmParameterException; import java.util.zip.GZIPInputStream; import java.util.zip.InflaterInputStream; +import static java.lang.String.format; +import static org.owasp.dependencycheck.utils.Settings.KEYS.DOWNLOADER_QUICK_QUERY_TIMESTAMP; +import static org.owasp.dependencycheck.utils.Settings.getBoolean; + /** * A utility to download files from the Internet. * @@ -48,6 +48,16 @@ public final class Downloader { */ private static final int MAX_REDIRECT_ATTEMPTS = 5; + /** + * The default HTTP request method for query timestamp + */ + private static final String HEAD = "HEAD"; + + /** + * The HTTP request method which can be used by query timestamp + */ + private static final String GET = "GET"; + /** * Private constructor for utility class. */ @@ -79,18 +89,18 @@ public final class Downloader { try { file = new File(url.toURI()); } catch (URISyntaxException ex) { - final String msg = String.format("Download failed, unable to locate '%s'", url.toString()); + final String msg = format("Download failed, unable to locate '%s'", url.toString()); throw new DownloadFailedException(msg); } if (file.exists()) { try { org.apache.commons.io.FileUtils.copyFile(file, outputPath); } catch (IOException ex) { - final String msg = String.format("Download failed, unable to copy '%s' to '%s'", url.toString(), outputPath.getAbsolutePath()); + final String msg = format("Download failed, unable to copy '%s' to '%s'", url.toString(), outputPath.getAbsolutePath()); throw new DownloadFailedException(msg); } } else { - final String msg = String.format("Download failed, file ('%s') does not exist", url.toString()); + final String msg = format("Download failed, file ('%s') does not exist", url.toString()); throw new DownloadFailedException(msg); } } else { @@ -124,7 +134,7 @@ public final class Downloader { } finally { conn = null; } - final String msg = String.format("Error downloading file %s; received response code %s.", url.toString(), status); + final String msg = format("Error downloading file %s; received response code %s.", url.toString(), status); throw new DownloadFailedException(msg); } @@ -136,7 +146,7 @@ public final class Downloader { } finally { conn = null; } - final String msg = String.format("Error downloading file %s; unable to connect.", url.toString()); + final String msg = format("Error downloading file %s; unable to connect.", url.toString()); throw new DownloadFailedException(msg, ex); } @@ -161,11 +171,11 @@ public final class Downloader { LOGGER.debug("Download of {} complete", url.toString()); } catch (IOException ex) { analyzeException(ex); - final String msg = String.format("Error saving '%s' to file '%s'%nConnection Timeout: %d%nEncoding: %s%n", + final String msg = format("Error saving '%s' to file '%s'%nConnection Timeout: %d%nEncoding: %s%n", url.toString(), outputPath.getAbsolutePath(), conn.getConnectTimeout(), encoding); throw new DownloadFailedException(msg, ex); } catch (Throwable ex) { - final String msg = String.format("Unexpected exception saving '%s' to file '%s'%nConnection Timeout: %d%nEncoding: %s%n", + final String msg = format("Unexpected exception saving '%s' to file '%s'%nConnection Timeout: %d%nEncoding: %s%n", url.toString(), outputPath.getAbsolutePath(), conn.getConnectTimeout(), encoding); throw new DownloadFailedException(msg, ex); } finally { @@ -208,27 +218,28 @@ public final class Downloader { try { lastModifiedFile = new File(url.toURI()); } catch (URISyntaxException ex) { - final String msg = String.format("Unable to locate '%s'", url.toString()); + final String msg = format("Unable to locate '%s'", url.toString()); throw new DownloadFailedException(msg); } timestamp = lastModifiedFile.lastModified(); } else { + String httpMethod = determineHttpMethod(); HttpURLConnection conn = null; try { conn = URLConnectionFactory.createHttpURLConnection(url); - conn.setRequestMethod("HEAD"); + conn.setRequestMethod(httpMethod); conn.connect(); final int t = conn.getResponseCode(); if (t >= 200 && t < 300) { timestamp = conn.getLastModified(); } else { - throw new DownloadFailedException("HEAD request returned a non-200 status code"); + throw new DownloadFailedException(format("%s request returned a non-200 status code", httpMethod)); } } catch (URLConnectionFailureException ex) { - throw new DownloadFailedException("Error creating URL Connection for HTTP HEAD request.", ex); + throw new DownloadFailedException(format("Error creating URL Connection for HTTP %s request.", httpMethod), ex); } catch (IOException ex) { analyzeException(ex); - throw new DownloadFailedException("Error making HTTP HEAD request.", ex); + throw new DownloadFailedException(format("Error making HTTP %s request.", httpMethod), ex); } finally { if (conn != null) { try { @@ -266,4 +277,19 @@ public final class Downloader { cause = cause.getCause(); } } + + private static String determineHttpMethod() { + return isQuickQuery() ? HEAD : GET; + } + + private static boolean isQuickQuery() { + boolean quickQuery; + + try { + quickQuery = getBoolean(DOWNLOADER_QUICK_QUERY_TIMESTAMP, true); + } catch (InvalidSettingException e) { + quickQuery = true; + } + return quickQuery; + } } diff --git a/dependency-check-utils/src/main/java/org/owasp/dependencycheck/utils/Settings.java b/dependency-check-utils/src/main/java/org/owasp/dependencycheck/utils/Settings.java index 9e308aca5..45271ef59 100644 --- a/dependency-check-utils/src/main/java/org/owasp/dependencycheck/utils/Settings.java +++ b/dependency-check-utils/src/main/java/org/owasp/dependencycheck/utils/Settings.java @@ -275,6 +275,11 @@ public final class Settings { * The key to obtain the VFeed update status. */ public static final String VFEED_UPDATE_STATUS = "vfeed.update_status"; + + /** + * The HTTP request method for query last modified date + */ + public static final String DOWNLOADER_QUICK_QUERY_TIMESTAMP = "downloader.quick.query.timestamp"; } // diff --git a/dependency-check-utils/src/test/resources/dependencycheck.properties b/dependency-check-utils/src/test/resources/dependencycheck.properties index 3e8480667..6c156df32 100644 --- a/dependency-check-utils/src/test/resources/dependencycheck.properties +++ b/dependency-check-utils/src/test/resources/dependencycheck.properties @@ -64,3 +64,6 @@ analyzer.nexus.url=https://repository.sonatype.org/service/local/ # If set to true, the proxy will still ONLY be used if the proxy properties (proxy.url, proxy.port) # are configured analyzer.nexus.proxy=true + +# use HEAD (default) or GET as HTTP request method for query timestamp +downloader.quick.query.timestamp=true \ No newline at end of file