github-mirrors/DependencyCheck
1
0
Fork 0
mirror of https://github.com/ysoftdevs/DependencyCheck.git synced 2026-03-25 10:32:00 +01:00
Code Issues Packages Projects Releases Wiki Activity

improved multi-threaded processing and renamed things for clarity

Browse Source 
Former-commit-id: 5ae87f85859bb2b473a49ac6fa17035b26167fe0
This commit is contained in:
Jeremy Long
2013-12-02 21:49:55 -05:00
parent 8502c0f048
commit ef6035b5be
4 changed files with 68 additions and 61 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/DatabaseUpdater.java
View File

@@ -46,7 +46,7 @@ public class DatabaseUpdater implements CachedWebDataSource {
@Override @Override
public void update() throws UpdateException { public void update() throws UpdateException {
try { try {
final StandardUpdateTask task = new StandardUpdateTask(); final StandardUpdate task = new StandardUpdate();
if (task.isUpdateNeeded()) { if (task.isUpdateNeeded()) {
if (task.shouldDeleteAndRecreate()) { if (task.shouldDeleteAndRecreate()) {
try { try {

4
dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/ProcessTask.java
View File

@@ -120,7 +120,7 @@ public class ProcessTask implements Callable<ProcessTask> {
private void processFiles() throws UpdateException { private void processFiles() throws UpdateException {
String msg = String.format("Processing Started for NVD CVE - %s", filePair.getNvdCveInfo().getId()); String msg = String.format("Processing Started for NVD CVE - %s", filePair.getNvdCveInfo().getId());
Logger.getLogger(StandardUpdateTask.class.getName()).log(Level.INFO, msg); Logger.getLogger(StandardUpdate.class.getName()).log(Level.INFO, msg);
try { try {
importXML(filePair.getFirst(), filePair.getSecond()); importXML(filePair.getFirst(), filePair.getSecond());
cveDB.commit(); cveDB.commit();
@@ -143,6 +143,6 @@ public class ProcessTask implements Callable<ProcessTask> {
filePair.cleanup(); filePair.cleanup();
} }
msg = String.format("Processing Complete for NVD CVE - %s", filePair.getNvdCveInfo().getId()); msg = String.format("Processing Complete for NVD CVE - %s", filePair.getNvdCveInfo().getId());
Logger.getLogger(StandardUpdateTask.class.getName()).log(Level.INFO, msg); Logger.getLogger(StandardUpdate.class.getName()).log(Level.INFO, msg);
} }
} }

91
dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/StandardUpdateTask.java → dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/StandardUpdate.java
View File

@@ -44,11 +44,11 @@ import static org.owasp.dependencycheck.data.update.DataStoreMetaInfo.MODIFIED;
import org.owasp.dependencycheck.utils.FileUtils; import org.owasp.dependencycheck.utils.FileUtils;
/** /**
* Class responsible for updating the CPE and NVDCVE data stores. * Class responsible for updating the NVDCVE data store.
* *
* @author Jeremy Long (jeremy.long@owasp.org) * @author Jeremy Long (jeremy.long@owasp.org)
*/ */
public class StandardUpdateTask { public class StandardUpdate {
/** /**
* The max thread pool size to use when downloading files. * The max thread pool size to use when downloading files.
@@ -109,7 +109,7 @@ public class StandardUpdateTask {
* @throws UpdateException thrown if there is an exception generating the * @throws UpdateException thrown if there is an exception generating the
* update task * update task
*/ */
public StandardUpdateTask() throws MalformedURLException, DownloadFailedException, UpdateException { public StandardUpdate() throws MalformedURLException, DownloadFailedException, UpdateException {
properties = new DataStoreMetaInfo(); properties = new DataStoreMetaInfo();
updateable = updatesNeeded(); updateable = updatesNeeded();
} }
@@ -133,7 +133,7 @@ public class StandardUpdateTask {
return; return;
} }
if (maxUpdates > 3) { if (maxUpdates > 3) {
Logger.getLogger(StandardUpdateTask.class.getName()).log(Level.INFO, Logger.getLogger(StandardUpdate.class.getName()).log(Level.INFO,
"NVD CVE requires several updates; this could take a couple of minutes."); "NVD CVE requires several updates; this could take a couple of minutes.");
} }
if (maxUpdates > 0) { if (maxUpdates > 0) {
@@ -159,37 +159,33 @@ public class StandardUpdateTask {
} }
final CallableDownloadTask call = new CallableDownloadTask(cve, file1, file2); final CallableDownloadTask call = new CallableDownloadTask(cve, file1, file2);
downloadFutures.add(downloadExecutor.submit(call)); downloadFutures.add(downloadExecutor.submit(call));
if (ctr == 3) {
ctr = 0;
final Iterator<Future<CallableDownloadTask>> itr = downloadFutures.iterator(); boolean waitForFuture = ctr % 2 == 0;
while (itr.hasNext()) {
final Future<CallableDownloadTask> future = itr.next();
while (!future.isDone()) {
try {
Thread.sleep(1000);
} catch (InterruptedException ex) {
Logger.getLogger(StandardUpdateTask.class.getName()).log(Level.FINE, null, ex);
}
}
final CallableDownloadTask filePair; final Iterator<Future<CallableDownloadTask>> itr = downloadFutures.iterator();
while (itr.hasNext()) {
final Future<CallableDownloadTask> future = itr.next();
if (waitForFuture) { //only allow two NVD/CVE files to be downloaded at a time
spinWaitForFuture(future);
}
if (future.isDone()) { //if we find something complete, add it to the process queue
try { try {
filePair = future.get(); final CallableDownloadTask filePair = future.get();
itr.remove();
final ProcessTask task = new ProcessTask(cveDB, properties, filePair);
processFutures.add(processExecutor.submit(task));
} catch (InterruptedException ex) { } catch (InterruptedException ex) {
downloadExecutor.shutdownNow(); downloadExecutor.shutdownNow();
Logger.getLogger(StandardUpdateTask.class.getName()).log(Level.FINE, "Thread was interupted", ex); Logger.getLogger(StandardUpdate.class.getName()).log(Level.FINE, "Thread was interupted", ex);
throw new UpdateException(ex); throw new UpdateException(ex);
} catch (ExecutionException ex) { } catch (ExecutionException ex) {
downloadExecutor.shutdownNow(); downloadExecutor.shutdownNow();
Logger.getLogger(StandardUpdateTask.class.getName()).log(Level.SEVERE, null, ex); Logger.getLogger(StandardUpdate.class.getName()).log(Level.SEVERE, null, ex);
throw new UpdateException(ex); throw new UpdateException(ex);
} }
itr.remove();
final ProcessTask task = new ProcessTask(cveDB, properties, filePair);
processFutures.add(processExecutor.submit(task));
} }
} }
} }
} }
@@ -203,11 +199,11 @@ public class StandardUpdateTask {
} }
} catch (InterruptedException ex) { } catch (InterruptedException ex) {
downloadExecutor.shutdownNow(); downloadExecutor.shutdownNow();
Logger.getLogger(StandardUpdateTask.class.getName()).log(Level.FINE, "Thread was interupted during download", ex); Logger.getLogger(StandardUpdate.class.getName()).log(Level.FINE, "Thread was interupted during download", ex);
throw new UpdateException(ex); throw new UpdateException(ex);
} catch (ExecutionException ex) { } catch (ExecutionException ex) {
downloadExecutor.shutdownNow(); downloadExecutor.shutdownNow();
Logger.getLogger(StandardUpdateTask.class.getName()).log(Level.FINE, "Execution Exception during download", ex); Logger.getLogger(StandardUpdate.class.getName()).log(Level.FINE, "Execution Exception during download", ex);
throw new UpdateException(ex); throw new UpdateException(ex);
} finally { } finally {
downloadExecutor.shutdown(); downloadExecutor.shutdown();
@@ -221,11 +217,11 @@ public class StandardUpdateTask {
} }
} catch (InterruptedException ex) { } catch (InterruptedException ex) {
processExecutor.shutdownNow(); processExecutor.shutdownNow();
Logger.getLogger(StandardUpdateTask.class.getName()).log(Level.FINE, "Thread was interupted during processing", ex); Logger.getLogger(StandardUpdate.class.getName()).log(Level.FINE, "Thread was interupted during processing", ex);
throw new UpdateException(ex); throw new UpdateException(ex);
} catch (ExecutionException ex) { } catch (ExecutionException ex) {
processExecutor.shutdownNow(); processExecutor.shutdownNow();
Logger.getLogger(StandardUpdateTask.class.getName()).log(Level.FINE, "Execution Exception during process", ex); Logger.getLogger(StandardUpdate.class.getName()).log(Level.FINE, "Execution Exception during process", ex);
throw new UpdateException(ex); throw new UpdateException(ex);
} finally { } finally {
processExecutor.shutdown(); processExecutor.shutdown();
@@ -253,7 +249,7 @@ public class StandardUpdateTask {
} }
} }
if (maxUpdates > 3) { if (maxUpdates > 3) {
Logger.getLogger(StandardUpdateTask.class.getName()).log(Level.INFO, Logger.getLogger(StandardUpdate.class.getName()).log(Level.INFO,
"NVD CVE requires several updates; this could take a couple of minutes."); "NVD CVE requires several updates; this could take a couple of minutes.");
} }
if (maxUpdates > 0) { if (maxUpdates > 0) {
@@ -264,13 +260,13 @@ public class StandardUpdateTask {
for (NvdCveInfo cve : getUpdateable()) { for (NvdCveInfo cve : getUpdateable()) {
if (cve.getNeedsUpdate()) { if (cve.getNeedsUpdate()) {
count += 1; count += 1;
Logger.getLogger(StandardUpdateTask.class.getName()).log(Level.INFO, Logger.getLogger(StandardUpdate.class.getName()).log(Level.INFO,
"Updating NVD CVE ({0} of {1})", new Object[]{count, maxUpdates}); "Updating NVD CVE ({0} of {1})", new Object[]{count, maxUpdates});
URL url = new URL(cve.getUrl()); URL url = new URL(cve.getUrl());
File outputPath = null; File outputPath = null;
File outputPath12 = null; File outputPath12 = null;
try { try {
Logger.getLogger(StandardUpdateTask.class.getName()).log(Level.INFO, Logger.getLogger(StandardUpdate.class.getName()).log(Level.INFO,
"Downloading {0}", cve.getUrl()); "Downloading {0}", cve.getUrl());
outputPath = File.createTempFile("cve" + cve.getId() + "_", ".xml"); outputPath = File.createTempFile("cve" + cve.getId() + "_", ".xml");
Downloader.fetchFile(url, outputPath); Downloader.fetchFile(url, outputPath);
@@ -279,7 +275,7 @@ public class StandardUpdateTask {
outputPath12 = File.createTempFile("cve_1_2_" + cve.getId() + "_", ".xml"); outputPath12 = File.createTempFile("cve_1_2_" + cve.getId() + "_", ".xml");
Downloader.fetchFile(url, outputPath12); Downloader.fetchFile(url, outputPath12);
Logger.getLogger(StandardUpdateTask.class.getName()).log(Level.INFO, Logger.getLogger(StandardUpdate.class.getName()).log(Level.INFO,
"Processing {0}", cve.getUrl()); "Processing {0}", cve.getUrl());
importXML(outputPath, outputPath12); importXML(outputPath, outputPath12);
@@ -287,7 +283,7 @@ public class StandardUpdateTask {
getCveDB().commit(); getCveDB().commit();
getProperties().save(cve); getProperties().save(cve);
Logger.getLogger(StandardUpdateTask.class.getName()).log(Level.INFO, Logger.getLogger(StandardUpdate.class.getName()).log(Level.INFO,
"Completed update {0} of {1}", new Object[]{count, maxUpdates}); "Completed update {0} of {1}", new Object[]{count, maxUpdates});
} catch (FileNotFoundException ex) { } catch (FileNotFoundException ex) {
throw new UpdateException(ex); throw new UpdateException(ex);
@@ -360,11 +356,11 @@ public class StandardUpdateTask {
} catch (InvalidDataException ex) { } catch (InvalidDataException ex) {
final String msg = "Unable to retrieve valid timestamp from nvd cve downloads page"; final String msg = "Unable to retrieve valid timestamp from nvd cve downloads page";
Logger Logger
.getLogger(StandardUpdateTask.class .getLogger(StandardUpdate.class
.getName()).log(Level.FINE, msg, ex); .getName()).log(Level.FINE, msg, ex);
throw new DownloadFailedException(msg, ex); throw new DownloadFailedException(msg, ex);
} catch (InvalidSettingException ex) { } catch (InvalidSettingException ex) {
Logger.getLogger(StandardUpdateTask.class Logger.getLogger(StandardUpdate.class
.getName()).log(Level.FINE, "Invalid setting found when retrieving timestamps", ex); .getName()).log(Level.FINE, "Invalid setting found when retrieving timestamps", ex);
throw new DownloadFailedException( throw new DownloadFailedException(
"Invalid settings", ex); "Invalid settings", ex);
@@ -420,7 +416,7 @@ public class StandardUpdateTask {
final String msg = String.format("Error parsing '%s' '%s' from nvdcve.lastupdated", final String msg = String.format("Error parsing '%s' '%s' from nvdcve.lastupdated",
DataStoreMetaInfo.LAST_UPDATED_BASE, entry.getId()); DataStoreMetaInfo.LAST_UPDATED_BASE, entry.getId());
Logger Logger
.getLogger(StandardUpdateTask.class .getLogger(StandardUpdate.class
.getName()).log(Level.FINE, msg, ex); .getName()).log(Level.FINE, msg, ex);
} }
if (currentTimestamp == entry.getTimestamp()) { if (currentTimestamp == entry.getTimestamp()) {
@@ -432,9 +428,9 @@ public class StandardUpdateTask {
} catch (NumberFormatException ex) { } catch (NumberFormatException ex) {
final String msg = "An invalid schema version or timestamp exists in the data.properties file."; final String msg = "An invalid schema version or timestamp exists in the data.properties file.";
Logger Logger
.getLogger(StandardUpdateTask.class .getLogger(StandardUpdate.class
.getName()).log(Level.WARNING, msg); .getName()).log(Level.WARNING, msg);
Logger.getLogger(StandardUpdateTask.class Logger.getLogger(StandardUpdate.class
.getName()).log(Level.FINE, null, ex); .getName()).log(Level.FINE, null, ex);
} }
} }
@@ -498,7 +494,7 @@ public class StandardUpdateTask {
try { try {
cveDB.close(); cveDB.close();
} catch (Exception ignore) { } catch (Exception ignore) {
Logger.getLogger(StandardUpdateTask.class.getName()).log(Level.FINEST, "Error closing the cveDB", ignore); Logger.getLogger(StandardUpdate.class.getName()).log(Level.FINEST, "Error closing the cveDB", ignore);
} }
} }
} }
@@ -515,19 +511,19 @@ public class StandardUpdateTask {
cveDB.open(); cveDB.open();
} catch (IOException ex) { } catch (IOException ex) {
closeDataStores(); closeDataStores();
Logger.getLogger(StandardUpdateTask.class.getName()).log(Level.FINE, "IO Error opening databases", ex); Logger.getLogger(StandardUpdate.class.getName()).log(Level.FINE, "IO Error opening databases", ex);
throw new UpdateException("Error updating the CPE/CVE data, please see the log file for more details."); throw new UpdateException("Error updating the CPE/CVE data, please see the log file for more details.");
} catch (SQLException ex) { } catch (SQLException ex) {
closeDataStores(); closeDataStores();
Logger.getLogger(StandardUpdateTask.class.getName()).log(Level.FINE, "SQL Exception opening databases", ex); Logger.getLogger(StandardUpdate.class.getName()).log(Level.FINE, "SQL Exception opening databases", ex);
throw new UpdateException("Error updating the CPE/CVE data, please see the log file for more details."); throw new UpdateException("Error updating the CPE/CVE data, please see the log file for more details.");
} catch (DatabaseException ex) { } catch (DatabaseException ex) {
closeDataStores(); closeDataStores();
Logger.getLogger(StandardUpdateTask.class.getName()).log(Level.FINE, "Database Exception opening databases", ex); Logger.getLogger(StandardUpdate.class.getName()).log(Level.FINE, "Database Exception opening databases", ex);
throw new UpdateException("Error updating the CPE/CVE data, please see the log file for more details."); throw new UpdateException("Error updating the CPE/CVE data, please see the log file for more details.");
} catch (ClassNotFoundException ex) { } catch (ClassNotFoundException ex) {
closeDataStores(); closeDataStores();
Logger.getLogger(StandardUpdateTask.class.getName()).log(Level.FINE, "Class not found exception opening databases", ex); Logger.getLogger(StandardUpdate.class.getName()).log(Level.FINE, "Class not found exception opening databases", ex);
throw new UpdateException("Error updating the CPE/CVE data, please see the log file for more details."); throw new UpdateException("Error updating the CPE/CVE data, please see the log file for more details.");
} }
} }
@@ -547,4 +543,15 @@ public class StandardUpdateTask {
final double differenceInDays = (compareTo - date) / 1000.0 / 60.0 / 60.0 / 24.0; final double differenceInDays = (compareTo - date) / 1000.0 / 60.0 / 60.0 / 24.0;
return differenceInDays < range; return differenceInDays < range;
} }
private void spinWaitForFuture(final Future<CallableDownloadTask> future) {
//then wait for downloads to finish
while (!future.isDone()) {
try {
Thread.sleep(1000);
} catch (InterruptedException ex) {
Logger.getLogger(StandardUpdate.class.getName()).log(Level.FINE, null, ex);
}
}
}
} }

32
dependency-check-core/src/test/java/org/owasp/dependencycheck/data/update/StandardUpdateTaskIntegrationTest.java → dependency-check-core/src/test/java/org/owasp/dependencycheck/data/update/StandardUpdateIntegrationTest.java
View File

@@ -34,9 +34,9 @@ import org.owasp.dependencycheck.utils.DownloadFailedException;
* *
* @author Jeremy Long (jeremy.long@owasp.org) * @author Jeremy Long (jeremy.long@owasp.org)
*/ */
public class StandardUpdateTaskIntegrationTest { public class StandardUpdateIntegrationTest {
public StandardUpdateTaskIntegrationTest() { public StandardUpdateIntegrationTest() {
} }
@BeforeClass @BeforeClass
@@ -55,30 +55,30 @@ public class StandardUpdateTaskIntegrationTest {
public void tearDown() { public void tearDown() {
} }
public StandardUpdateTask getStandardUpdateTask() throws MalformedURLException, DownloadFailedException, UpdateException { public StandardUpdate getStandardUpdateTask() throws MalformedURLException, DownloadFailedException, UpdateException {
StandardUpdateTask instance = new StandardUpdateTask(); StandardUpdate instance = new StandardUpdate();
return instance; return instance;
} }
/** /**
* Test of setDeleteAndRecreate method, of class StandardUpdateTask. * Test of setDeleteAndRecreate method, of class StandardUpdate.
*/ */
@Test @Test
public void testSetDeleteAndRecreate() throws Exception { public void testSetDeleteAndRecreate() throws Exception {
boolean deleteAndRecreate = false; boolean deleteAndRecreate = false;
boolean expResult = false; boolean expResult = false;
StandardUpdateTask instance = getStandardUpdateTask(); StandardUpdate instance = getStandardUpdateTask();
instance.setDeleteAndRecreate(deleteAndRecreate); instance.setDeleteAndRecreate(deleteAndRecreate);
boolean result = instance.shouldDeleteAndRecreate(); boolean result = instance.shouldDeleteAndRecreate();
assertEquals(expResult, result); assertEquals(expResult, result);
} }
/** /**
* Test of deleteExistingData method, of class StandardUpdateTask. * Test of deleteExistingData method, of class StandardUpdate.
*/ */
@Test @Test
public void testDeleteExistingData() throws Exception { public void testDeleteExistingData() throws Exception {
StandardUpdateTask instance = getStandardUpdateTask(); StandardUpdate instance = getStandardUpdateTask();
Exception result = null; Exception result = null;
try { try {
instance.deleteExistingData(); instance.deleteExistingData();
@@ -89,17 +89,17 @@ public class StandardUpdateTaskIntegrationTest {
} }
/** /**
* Test of openDataStores method, of class StandardUpdateTask. * Test of openDataStores method, of class StandardUpdate.
*/ */
@Test @Test
public void testOpenDataStores() throws Exception { public void testOpenDataStores() throws Exception {
StandardUpdateTask instance = getStandardUpdateTask(); StandardUpdate instance = getStandardUpdateTask();
instance.openDataStores(); instance.openDataStores();
instance.closeDataStores(); instance.closeDataStores();
} }
/** /**
* Test of withinRange method, of class StandardUpdateTask. * Test of withinRange method, of class StandardUpdate.
*/ */
@Test @Test
public void testWithinRange() throws Exception { public void testWithinRange() throws Exception {
@@ -108,7 +108,7 @@ public class StandardUpdateTaskIntegrationTest {
long current = c.getTimeInMillis(); long current = c.getTimeInMillis();
long lastRun = c.getTimeInMillis() - (3 * (1000 * 60 * 60 * 24)); long lastRun = c.getTimeInMillis() - (3 * (1000 * 60 * 60 * 24));
int range = 7; // 7 days int range = 7; // 7 days
StandardUpdateTask instance = getStandardUpdateTask(); StandardUpdate instance = getStandardUpdateTask();
boolean expResult = true; boolean expResult = true;
boolean result = instance.withinRange(lastRun, current, range); boolean result = instance.withinRange(lastRun, current, range);
assertEquals(expResult, result); assertEquals(expResult, result);
@@ -120,21 +120,21 @@ public class StandardUpdateTaskIntegrationTest {
} }
/** /**
* Test of update method, of class StandardUpdateTask. * Test of update method, of class StandardUpdate.
*/ */
@Test @Test
public void testUpdate() throws Exception { public void testUpdate() throws Exception {
StandardUpdateTask instance = getStandardUpdateTask(); StandardUpdate instance = getStandardUpdateTask();
instance.update(); instance.update();
//TODO make this an actual test //TODO make this an actual test
} }
/** /**
* Test of updatesNeeded method, of class StandardUpdateTask. * Test of updatesNeeded method, of class StandardUpdate.
*/ */
@Test @Test
public void testUpdatesNeeded() throws Exception { public void testUpdatesNeeded() throws Exception {
StandardUpdateTask instance = getStandardUpdateTask(); StandardUpdate instance = getStandardUpdateTask();
Updateable result = instance.updatesNeeded(); Updateable result = instance.updatesNeeded();
assertNotNull(result); assertNotNull(result);
} }