From 8f22740e073ddd6b278c8bd17330b629c072e5de Mon Sep 17 00:00:00 2001 From: Jeremy Long Date: Sun, 17 Nov 2013 22:30:31 -0500 Subject: [PATCH] updated to make downloading of the NVD CVE a multi-threaded operations Former-commit-id: 28e7467020db617007e89018d2d9fed8de335181 --- .../data/update/StandardUpdateTask.java | 219 +++++++++++++----- 1 file changed, 165 insertions(+), 54 deletions(-) diff --git a/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/StandardUpdateTask.java b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/StandardUpdateTask.java index 5c50ee9a5..d963a1837 100644 --- a/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/StandardUpdateTask.java +++ b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/StandardUpdateTask.java @@ -29,6 +29,14 @@ import java.net.URL; import java.sql.SQLException; import java.util.Calendar; import java.util.Date; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.concurrent.Callable; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; import java.util.logging.Level; import java.util.logging.Logger; import org.owasp.dependencycheck.data.UpdateException; @@ -70,13 +78,16 @@ public class StandardUpdateTask extends AbstractUpdateTask { */ @Override public void update() throws UpdateException { + int maxUpdates = 0; try { - int maxUpdates = 0; for (NvdCveInfo cve : getUpdateable()) { if (cve.getNeedsUpdate()) { maxUpdates += 1; } } + if (maxUpdates <= 0) { + return; + } if (maxUpdates > 3) { Logger.getLogger(StandardUpdateTask.class.getName()).log(Level.INFO, "NVD CVE requires several updates; this could take a couple of minutes."); @@ -85,35 +96,34 @@ public class StandardUpdateTask extends AbstractUpdateTask { openDataStores(); } - int count = 0; + int numThreads = (3 > maxUpdates) ? 3 : maxUpdates; + final ExecutorService executorService = Executors.newFixedThreadPool(numThreads); + Set> futures = new HashSet>(maxUpdates); + for (NvdCveInfo cve : getUpdateable()) { if (cve.getNeedsUpdate()) { - count += 1; - Logger.getLogger(StandardUpdateTask.class.getName()).log(Level.INFO, - "Updating NVD CVE ({0} of {1})", new Object[]{count, maxUpdates}); - URL url = new URL(cve.getUrl()); - File outputPath = null; - File outputPath12 = null; + final File file_1; + final File file_2; try { - Logger.getLogger(StandardUpdateTask.class.getName()).log(Level.INFO, - "Downloading {0}", cve.getUrl()); - outputPath = File.createTempFile("cve" + cve.getId() + "_", ".xml"); - Downloader.fetchFile(url, outputPath); - - url = new URL(cve.getOldSchemaVersionUrl()); - outputPath12 = File.createTempFile("cve_1_2_" + cve.getId() + "_", ".xml"); - Downloader.fetchFile(url, outputPath12); - - Logger.getLogger(StandardUpdateTask.class.getName()).log(Level.INFO, - "Processing {0}", cve.getUrl()); - - importXML(outputPath, outputPath12); + file_1 = File.createTempFile("cve" + cve.getId() + "_", ".xml"); + file_2 = File.createTempFile("cve_1_2_" + cve.getId() + "_", ".xml"); + } catch (IOException ex) { + throw new UpdateException(ex); + } + final CallableDownloadTask call = new CallableDownloadTask(cve, file_1, file_2); + futures.add(executorService.submit(call)); + } + } + try { + for (Future future : futures) { + CallableDownloadTask filePair = future.get(); + String msg = String.format("Processing Started for NVD CVE - %s", filePair.getNvdCveInfo().getId()); + Logger.getLogger(StandardUpdateTask.class.getName()).log(Level.INFO, msg); + try { + importXML(filePair.getFirst(), filePair.getSecond()); getCveDB().commit(); - getProperties().save(cve); - - Logger.getLogger(StandardUpdateTask.class.getName()).log(Level.INFO, - "Completed update {0} of {1}", new Object[]{count, maxUpdates}); + getProperties().save(filePair.getNvdCveInfo()); } catch (FileNotFoundException ex) { throw new UpdateException(ex); } catch (ParserConfigurationException ex) { @@ -129,40 +139,132 @@ public class StandardUpdateTask extends AbstractUpdateTask { } catch (ClassNotFoundException ex) { throw new UpdateException(ex); } finally { - boolean deleted = false; - try { - if (outputPath != null && outputPath.exists()) { - deleted = outputPath.delete(); - } - } finally { - if (outputPath != null && (outputPath.exists() || !deleted)) { - outputPath.deleteOnExit(); - } - } - try { - deleted = false; - if (outputPath12 != null && outputPath12.exists()) { - deleted = outputPath12.delete(); - } - } finally { - if (outputPath12 != null && (outputPath12.exists() || !deleted)) { - outputPath12.deleteOnExit(); - } - } + filePair.cleanup(); } + msg = String.format("Processing Complete for NVD CVE - %s", filePair.getNvdCveInfo().getId()); + Logger.getLogger(StandardUpdateTask.class.getName()).log(Level.INFO, msg); } + } catch (InterruptedException ex) { + executorService.shutdownNow(); + Logger.getLogger(StandardUpdateTask.class.getName()).log(Level.FINE, "Thread was interupted", ex); + throw new UpdateException(ex); + } catch (ExecutionException ex) { + executorService.shutdownNow(); + Logger.getLogger(StandardUpdateTask.class.getName()).log(Level.SEVERE, null, ex); + throw new UpdateException(ex); + } finally { + //yes, this should likely not be in the finally because of the shutdownNow above. + executorService.shutdown(); } - if (maxUpdates >= 1) { //ensure the modified file date gets written + + if (maxUpdates + >= 1) { //ensure the modified file date gets written getProperties().save(getUpdateable().get(MODIFIED)); getCveDB().cleanupDatabase(); } - } catch (MalformedURLException ex) { - throw new UpdateException(ex); } finally { closeDataStores(); } } + // + /* + * TODO - remove this + public void update() throws UpdateException { + try { + int maxUpdates = 0; + for (NvdCveInfo cve : getUpdateable()) { + if (cve.getNeedsUpdate()) { + maxUpdates += 1; + } + } + if (maxUpdates > 3) { + Logger.getLogger(StandardUpdateTask.class.getName()).log(Level.INFO, + "NVD CVE requires several updates; this could take a couple of minutes."); + } + if (maxUpdates > 0) { + openDataStores(); + } + + int count = 0; + for (NvdCveInfo cve : getUpdateable()) { + if (cve.getNeedsUpdate()) { + count += 1; + Logger.getLogger(StandardUpdateTask.class.getName()).log(Level.INFO, + "Updating NVD CVE ({0} of {1})", new Object[]{count, maxUpdates}); + URL url = new URL(cve.getUrl()); + File outputPath = null; + File outputPath12 = null; + try { + Logger.getLogger(StandardUpdateTask.class.getName()).log(Level.INFO, + "Downloading {0}", cve.getUrl()); + outputPath = File.createTempFile("cve" + cve.getId() + "_", ".xml"); + Downloader.fetchFile(url, outputPath); + + url = new URL(cve.getOldSchemaVersionUrl()); + outputPath12 = File.createTempFile("cve_1_2_" + cve.getId() + "_", ".xml"); + Downloader.fetchFile(url, outputPath12); + + Logger.getLogger(StandardUpdateTask.class.getName()).log(Level.INFO, + "Processing {0}", cve.getUrl()); + + importXML(outputPath, outputPath12); + + getCveDB().commit(); + getProperties().save(cve); + + Logger.getLogger(StandardUpdateTask.class.getName()).log(Level.INFO, + "Completed update {0} of {1}", new Object[]{count, maxUpdates}); + } catch (FileNotFoundException ex) { + throw new UpdateException(ex); + } catch (ParserConfigurationException ex) { + throw new UpdateException(ex); + } catch (SAXException ex) { + throw new UpdateException(ex); + } catch (IOException ex) { + throw new UpdateException(ex); + } catch (SQLException ex) { + throw new UpdateException(ex); + } catch (DatabaseException ex) { + throw new UpdateException(ex); + } catch (ClassNotFoundException ex) { + throw new UpdateException(ex); + } finally { + boolean deleted = false; + try { + if (outputPath != null && outputPath.exists()) { + deleted = outputPath.delete(); + } + } finally { + if (outputPath != null && (outputPath.exists() || !deleted)) { + outputPath.deleteOnExit(); + } + } + try { + deleted = false; + if (outputPath12 != null && outputPath12.exists()) { + deleted = outputPath12.delete(); + } + } finally { + if (outputPath12 != null && (outputPath12.exists() || !deleted)) { + outputPath12.deleteOnExit(); + } + } + } + } + } + if (maxUpdates >= 1) { //ensure the modified file date gets written + getProperties().save(getUpdateable().get(MODIFIED)); + getCveDB().cleanupDatabase(); + } + } catch (MalformedURLException ex) { + throw new UpdateException(ex); + } finally { + closeDataStores(); + } + } + */ + // /** * Determines if the index needs to be updated. This is done by fetching the * NVD CVE meta data and checking the last update date. If the data needs to @@ -184,11 +286,15 @@ public class StandardUpdateTask extends AbstractUpdateTask { updates = retrieveCurrentTimestampsFromWeb(); } catch (InvalidDataException ex) { final String msg = "Unable to retrieve valid timestamp from nvd cve downloads page"; - Logger.getLogger(StandardUpdateTask.class.getName()).log(Level.FINE, msg, ex); + Logger + .getLogger(StandardUpdateTask.class + .getName()).log(Level.FINE, msg, ex); throw new DownloadFailedException(msg, ex); } catch (InvalidSettingException ex) { - Logger.getLogger(StandardUpdateTask.class.getName()).log(Level.FINE, "Invalid setting found when retrieving timestamps", ex); - throw new DownloadFailedException("Invalid settings", ex); + Logger.getLogger(StandardUpdateTask.class + .getName()).log(Level.FINE, "Invalid setting found when retrieving timestamps", ex); + throw new DownloadFailedException( + "Invalid settings", ex); } if (updates == null) { @@ -241,7 +347,9 @@ public class StandardUpdateTask extends AbstractUpdateTask { } catch (NumberFormatException ex) { final String msg = String.format("Error parsing '%s' '%s' from nvdcve.lastupdated", DataStoreMetaInfo.LAST_UPDATED_BASE, entry.getId()); - Logger.getLogger(StandardUpdateTask.class.getName()).log(Level.FINE, msg, ex); + Logger + .getLogger(StandardUpdateTask.class + .getName()).log(Level.FINE, msg, ex); } if (currentTimestamp == entry.getTimestamp()) { entry.setNeedsUpdate(false); @@ -251,8 +359,11 @@ public class StandardUpdateTask extends AbstractUpdateTask { } } catch (NumberFormatException ex) { final String msg = "An invalid schema version or timestamp exists in the data.properties file."; - Logger.getLogger(StandardUpdateTask.class.getName()).log(Level.WARNING, msg); - Logger.getLogger(StandardUpdateTask.class.getName()).log(Level.FINE, null, ex); + Logger + .getLogger(StandardUpdateTask.class + .getName()).log(Level.WARNING, msg); + Logger.getLogger(StandardUpdateTask.class + .getName()).log(Level.FINE, null, ex); } } return updates;