Added the Accept-Encoding header to speed up downloads

Former-commit-id: 669fed48dcf120c5a1bef0e6073e0107afd0db4a
This commit is contained in:
Jeremy Long
2012-10-29 21:55:05 -04:00
parent 96bdd8a41e
commit 8315d43f54
2 changed files with 5 additions and 2 deletions

View File

@@ -31,6 +31,7 @@ import java.net.URL;
import java.util.logging.Level;
import java.util.logging.Logger;
import java.util.zip.GZIPInputStream;
import java.util.zip.InflaterInputStream;
/**
* A utility to download files from the Internet.
@@ -129,8 +130,10 @@ public class Downloader {
BufferedOutputStream writer = null;
try {
InputStream reader;
if (unzip) {
if (unzip || (encoding != null && "gzip".equalsIgnoreCase(encoding))) {
reader = new GZIPInputStream(conn.getInputStream());
} else if (encoding != null && "deflate".equalsIgnoreCase(encoding)) {
reader = new InflaterInputStream(conn.getInputStream());
} else {
reader = conn.getInputStream();
}

View File

@@ -52,7 +52,7 @@ public class DownloaderIntegrationTest {
// Settings.setString(Settings.KEYS.PROXY_URL, "127.0.0.1");
//URL url = new URL(Settings.getString(Settings.KEYS.CPE_URL));
URL url = new URL("http://static.nvd.nist.gov/feeds/xml/cve/nvdcve-2.0-modified.xml");
URL url = new URL("http://static.nvd.nist.gov/feeds/xml/cve/nvdcve-2.0-2010.xml");
String outputPath = "target\\downloaded_cpe.xml";
Downloader.fetchFile(url, outputPath);