github-mirrors/DependencyCheck
1
0
Fork 0
mirror of https://github.com/ysoftdevs/DependencyCheck.git synced 2026-01-15 08:13:43 +01:00
Code Issues Packages Projects Releases Wiki Activity

improved downloading of CPE data

Browse Source
This commit is contained in:
Jeremy Long
2012-09-14 06:14:54 -04:00
parent 46f0f891f3
commit c089ac330a
9 changed files with 247 additions and 94 deletions
Download Patch File Download Diff File Expand all files Collapse all files

114
src/main/java/org/codesecure/dependencycheck/data/cpe/Index.java
View File

@@ -23,6 +23,7 @@ import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.io.OutputStreamWriter;
import java.net.MalformedURLException;
@@ -47,8 +48,6 @@ import org.codesecure.dependencycheck.utils.Downloader;
import org.codesecure.dependencycheck.utils.Settings;
import org.codesecure.dependencycheck.data.cpe.xml.Importer;
import org.codesecure.dependencycheck.utils.DownloadFailedException;
import org.joda.time.DateTime;
import org.joda.time.Days;
import org.xml.sax.SAXException;
/**
@@ -140,23 +139,24 @@ public class Index {
* @throws IOException is thrown if a temporary file could not be created.
*/
public void updateIndexFromWeb() throws MalformedURLException, ParserConfigurationException, SAXException, IOException {
if (updateNeeded()) {
long timeStamp = updateNeeded();
if (timeStamp > 0) {
URL url = new URL(Settings.getString(Settings.KEYS.CPE_URL));
File outputPath = null;
try {
outputPath = File.createTempFile("cpe", ".xml");
Downloader.fetchFile(url, outputPath);
Downloader.fetchFile(url, outputPath, true);
Importer.importXML(outputPath.toString());
writeLastUpdatedPropertyFile();
writeLastUpdatedPropertyFile(timeStamp);
} catch (DownloadFailedException ex) {
Logger.getLogger(Index.class.getName()).log(Level.SEVERE, null, ex);
} finally {
boolean deleted = false;
try {
deleted = outputPath.delete();
if (outputPath != null && outputPath.exists()) {
outputPath.delete();
}
} finally {
if (!deleted) {
if (outputPath != null && outputPath.exists()) {
outputPath.deleteOnExit();
}
}
@@ -164,12 +164,15 @@ public class Index {
}
}
private void writeLastUpdatedPropertyFile() {
DateTime now = new DateTime();
/**
* Writes a properties file containing the last updated date to the CPE directory.
* @param timeStamp the timestamp to write.
*/
private void writeLastUpdatedPropertyFile(long timeStamp) {
String dir = Settings.getString(Settings.KEYS.CPE_INDEX);
File cpeProp = new File(dir + File.separatorChar + UPDATE_PROPERTIES_FILE);
Properties prop = new Properties();
prop.put(this.LAST_UPDATED, String.valueOf(now.getMillis()));
prop.put(Index.LAST_UPDATED, String.valueOf(timeStamp));
OutputStream os = null;
try {
os = new FileOutputStream(cpeProp);
@@ -194,48 +197,91 @@ public class Index {
}
/**
* Determines if the index needs to be updated.
* Determines if the index needs to be updated. This is done by fetching the
* cpe.meta data and checking the lastModifiedDate. If the CPE data needs to
* be refreshed this method will return the timestamp of the new CPE. If an
* update is not required this function will return 0.
*
* @return whether or not the CPE Index needs to be updated.
* @return the timestamp of the currently published CPE.xml if the index needs to be updated, otherwise returns 0..
* @throws MalformedURLException is thrown if the URL for the CPE Meta data is incorrect.
* @throws DownloadFailedException is thrown if there is an error downloading the cpe.meta data file.
*/
public boolean updateNeeded() {
boolean needed = false;
String lastUpdated = null;
public long updateNeeded() throws MalformedURLException, DownloadFailedException {
long retVal = 0;
long lastUpdated = 0;
long currentlyPublishedDate = retrieveCurrentCPETimestampFromWeb();
if (currentlyPublishedDate == 0) {
throw new DownloadFailedException("Unable to retrieve valid timestamp from cpe.meta file");
}
String dir = Settings.getString(Settings.KEYS.CPE_INDEX);
File f = new File(dir);
if (!f.exists()) {
needed = true;
retVal = currentlyPublishedDate;
} else {
File cpeProp = new File(dir + File.separatorChar + UPDATE_PROPERTIES_FILE);
if (!cpeProp.exists()) {
needed = true;
retVal = currentlyPublishedDate;
} else {
Properties prop = new Properties();
FileInputStream is = null;
InputStream is = null;
try {
is = new FileInputStream(cpeProp);
prop.load(is);
lastUpdated = prop.getProperty(this.LAST_UPDATED);
lastUpdated = Long.parseLong(prop.getProperty(Index.LAST_UPDATED));
} catch (FileNotFoundException ex) {
Logger.getLogger(Index.class.getName()).log(Level.FINEST, null, ex);
} catch (IOException ex) {
Logger.getLogger(Index.class.getName()).log(Level.FINEST, null, ex);
}
try {
long lastupdate = Long.parseLong(lastUpdated);
DateTime last = new DateTime(lastupdate);
DateTime now = new DateTime();
Days d = Days.daysBetween(last, now);
int days = d.getDays();
int freq = Settings.getInt(Settings.KEYS.CPE_DOWNLOAD_FREQUENCY);
if (days >= freq) {
needed = true;
}
} catch (NumberFormatException ex) {
needed = true;
Logger.getLogger(Index.class.getName()).log(Level.FINEST, null, ex);
}
if (currentlyPublishedDate > lastUpdated) {
retVal = currentlyPublishedDate;
}
}
}
return needed;
return retVal;
}
/**
* Retrieves the timestamp from the CPE meta data file.
* @return the timestamp from the currently published cpe.meta.
* @throws MalformedURLException is thrown if the URL for the CPE Meta data is incorrect.
* @throws DownloadFailedException is thrown if there is an error downloading the cpe.meta data file.
*/
private long retrieveCurrentCPETimestampFromWeb() throws MalformedURLException, DownloadFailedException {
long timestamp = 0;
File tmp = null;
InputStream is = null;
try {
tmp = File.createTempFile("cpe", "meta");
URL url = new URL(Settings.getString(Settings.KEYS.CPE_META_URL));
Downloader.fetchFile(url, tmp);
Properties prop = new Properties();
is = new FileInputStream(tmp);
prop.load(is);
timestamp = Long.parseLong(prop.getProperty("lastModifiedDate"));
} catch (IOException ex) {
throw new DownloadFailedException("Unable to create temporary file for CPE Meta File download.", ex);
} finally {
try {
if (is != null) {
try {
is.close();
} catch (IOException ex) {
Logger.getLogger(Index.class.getName()).log(Level.FINEST, null, ex);
}
}
if (tmp != null && tmp.exists()) {
tmp.delete();
}
} finally {
if (tmp != null && tmp.exists()) {
tmp.deleteOnExit();
}
}
}
return timestamp;
}
}

32
src/main/java/org/codesecure/dependencycheck/utils/Downloader.java
View File

@@ -30,6 +30,7 @@ import java.net.SocketAddress;
import java.net.URL;
import java.util.logging.Level;
import java.util.logging.Logger;
import java.util.zip.GZIPInputStream;
/**
* A utility to download files from the Internet.
@@ -51,8 +52,19 @@ public class Downloader {
* @throws DownloadFailedException is thrown if there is an error downloading the file.
*/
public static void fetchFile(URL url, String outputPath) throws DownloadFailedException {
fetchFile(url, outputPath, false);
}
/**
* Retrieves a file from a given URL and saves it to the outputPath.
* @param url the URL of the file to download.
* @param outputPath the path to the save the file to.
* @param unzip true/false indicating that the file being retrieved is gzipped and if true, should be uncompressed before writting to the file.
* @throws DownloadFailedException is thrown if there is an error downloading the file.
*/
public static void fetchFile(URL url, String outputPath, boolean unzip) throws DownloadFailedException {
File f = new File(outputPath);
fetchFile(url, f);
fetchFile(url, f, unzip);
}
/**
@@ -62,6 +74,17 @@ public class Downloader {
* @throws DownloadFailedException is thrown if there is an error downloading the file.
*/
public static void fetchFile(URL url, File outputPath) throws DownloadFailedException {
fetchFile(url, outputPath, false);
}
/**
* Retrieves a file from a given URL and saves it to the outputPath.
* @param url the URL of the file to download.
* @param outputPath the path to the save the file to.
* @param unzip true/false indicating that the file being retrieved is gzipped and if true, should be uncompressed before writting to the file.
* @throws DownloadFailedException is thrown if there is an error downloading the file.
*/
public static void fetchFile(URL url, File outputPath, boolean unzip) throws DownloadFailedException {
HttpURLConnection conn = null;
Proxy proxy = null;
String proxyUrl = Settings.getString(Settings.KEYS.PROXY_URL);
@@ -96,7 +119,12 @@ public class Downloader {
try {
//the following times out on some systems because the CPE is big.
//InputStream reader = url.openStream();
InputStream reader = conn.getInputStream();
InputStream reader;
if (unzip) {
reader = new GZIPInputStream(conn.getInputStream());
} else {
reader = conn.getInputStream();
}
writer = new BufferedOutputStream(new FileOutputStream(outputPath));
byte[] buffer = new byte[4096];

14
src/main/java/org/codesecure/dependencycheck/utils/Settings.java
View File

@@ -47,7 +47,7 @@ public class Settings {
/**
* The properties key for the URL to the CPE.
*/
public static final String CPE_DOWNLOAD_FREQUENCY = "cpe.downloadfrequency";
public static final String CPE_META_URL = "cpe.meta.url";
/**
* The properties key for the path where the CCE Lucene Index will be stored.
*/
@@ -69,8 +69,8 @@ public class Settings {
*/
public static final String CONNECTION_TIMEOUT = "connection.timeout";
}
private static final String PROPERTIES_FILE = "dependencycheck.properties";
private static Settings instance = new Settings();
private static final String PROPERTIES_FILE = "META-INF/dependencycheck.properties";
private static final Settings INSTANCE = new Settings();
private Properties props = null;
/**
@@ -97,7 +97,7 @@ public class Settings {
* @return the property from the properties file.
*/
public static String getString(String key, String defaultValue) {
String str = System.getProperty(key, instance.props.getProperty(key));
String str = System.getProperty(key, INSTANCE.props.getProperty(key));
if (str == null) {
str = defaultValue;
}
@@ -110,7 +110,7 @@ public class Settings {
* @param value the value for the property.
*/
public static void setString(String key, String value) {
instance.props.setProperty(key, value);
INSTANCE.props.setProperty(key, value);
}
/**
@@ -123,7 +123,7 @@ public class Settings {
* @return the property from the properties file.
*/
public static String getString(String key) {
return System.getProperty(key, instance.props.getProperty(key));
return System.getProperty(key, INSTANCE.props.getProperty(key));
}
/**
@@ -151,4 +151,4 @@ public class Settings {
public static boolean getBoolean(String key) {
return Boolean.parseBoolean(Settings.getString(key));
}
}
}