diff --git a/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/ProcessTask.java b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/ProcessTask.java new file mode 100644 index 000000000..ff7914b69 --- /dev/null +++ b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/ProcessTask.java @@ -0,0 +1,130 @@ +/* + * To change this template, choose Tools | Templates + * and open the template in the editor. + */ +package org.owasp.dependencycheck.data.update; + +import java.io.File; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.sql.SQLException; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import java.util.concurrent.Callable; +import java.util.logging.Level; +import java.util.logging.Logger; +import javax.xml.parsers.ParserConfigurationException; +import javax.xml.parsers.SAXParser; +import javax.xml.parsers.SAXParserFactory; +import org.owasp.dependencycheck.data.UpdateException; +import org.owasp.dependencycheck.data.nvdcve.CveDB; +import org.owasp.dependencycheck.data.nvdcve.DatabaseException; +import org.owasp.dependencycheck.data.nvdcve.NvdCve12Handler; +import org.owasp.dependencycheck.data.nvdcve.NvdCve20Handler; +import org.owasp.dependencycheck.dependency.VulnerableSoftware; +import org.xml.sax.SAXException; + +/** + * + * @author Jeremy Long (jeremy.long@owasp.org) + */ +public class ProcessTask implements Callable { + + private UpdateException exception = null; + + /** + * Get the value of exception + * + * @return the value of exception + */ + public UpdateException getException() { + return exception; + } + + /** + * Set the value of exception + * + * @param exception new value of exception + */ + public void setException(UpdateException exception) { + this.exception = exception; + } + private final CveDB cveDB; + private final CallableDownloadTask filePair; + private final DataStoreMetaInfo properties; + + public ProcessTask(final CveDB cveDB, final DataStoreMetaInfo properties, final CallableDownloadTask filePair) { + this.cveDB = cveDB; + this.filePair = filePair; + this.properties = properties; + } + + @Override + public ProcessTask call() throws Exception { + try { + processFiles(); + } catch (UpdateException ex) { + this.exception = ex; + } + return this; + } + + /** + * Imports the NVD CVE XML File into the Lucene Index. + * + * @param file the file containing the NVD CVE XML + * @param oldVersion contains the file containing the NVD CVE XML 1.2 + * @throws ParserConfigurationException is thrown if there is a parser + * configuration exception + * @throws SAXException is thrown if there is a SAXException + * @throws IOException is thrown if there is a IO Exception + * @throws SQLException is thrown if there is a SQL exception + * @throws DatabaseException is thrown if there is a database exception + * @throws ClassNotFoundException thrown if the h2 database driver cannot be + * loaded + */ + protected void importXML(File file, File oldVersion) throws ParserConfigurationException, + SAXException, IOException, SQLException, DatabaseException, ClassNotFoundException { + + final SAXParserFactory factory = SAXParserFactory.newInstance(); + final SAXParser saxParser = factory.newSAXParser(); + + final NvdCve12Handler cve12Handler = new NvdCve12Handler(); + saxParser.parse(oldVersion, cve12Handler); + final Map> prevVersionVulnMap = cve12Handler.getVulnerabilities(); + + final NvdCve20Handler cve20Handler = new NvdCve20Handler(); + cve20Handler.setCveDB(cveDB); + cve20Handler.setPrevVersionVulnMap(prevVersionVulnMap); + saxParser.parse(file, cve20Handler); + } + + private void processFiles() throws UpdateException { + String msg = String.format("Processing Started for NVD CVE - %s", filePair.getNvdCveInfo().getId()); + Logger.getLogger(StandardUpdateTask.class.getName()).log(Level.INFO, msg); + try { + importXML(filePair.getFirst(), filePair.getSecond()); + cveDB.commit(); + properties.save(filePair.getNvdCveInfo()); + } catch (FileNotFoundException ex) { + throw new UpdateException(ex); + } catch (ParserConfigurationException ex) { + throw new UpdateException(ex); + } catch (SAXException ex) { + throw new UpdateException(ex); + } catch (IOException ex) { + throw new UpdateException(ex); + } catch (SQLException ex) { + throw new UpdateException(ex); + } catch (DatabaseException ex) { + throw new UpdateException(ex); + } catch (ClassNotFoundException ex) { + throw new UpdateException(ex); + } finally { + filePair.cleanup(); + } + msg = String.format("Processing Complete for NVD CVE - %s", filePair.getNvdCveInfo().getId()); + Logger.getLogger(StandardUpdateTask.class.getName()).log(Level.INFO, msg); + } +} diff --git a/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/StandardUpdateTask.java b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/StandardUpdateTask.java index 5c09fcd02..9f7e32e03 100644 --- a/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/StandardUpdateTask.java +++ b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/StandardUpdateTask.java @@ -29,6 +29,7 @@ import java.sql.SQLException; import java.util.Calendar; import java.util.Date; import java.util.HashSet; +import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Set; @@ -151,8 +152,10 @@ public class StandardUpdateTask { } final int poolSize = (MAX_THREAD_POOL_SIZE > maxUpdates) ? MAX_THREAD_POOL_SIZE : maxUpdates; - final ExecutorService executorService = Executors.newFixedThreadPool(poolSize); - final Set> futures = new HashSet>(maxUpdates); + final ExecutorService downloadExecutor = Executors.newFixedThreadPool(poolSize); + final ExecutorService processExecutor = Executors.newSingleThreadExecutor(); + final Set> downloadFutures = new HashSet>(maxUpdates); + final Set> processFutures = new HashSet>(maxUpdates); int ctr = 0; for (NvdCveInfo cve : updateable) { if (cve.getNeedsUpdate()) { @@ -166,11 +169,13 @@ public class StandardUpdateTask { throw new UpdateException(ex); } final CallableDownloadTask call = new CallableDownloadTask(cve, file1, file2); - futures.add(executorService.submit(call)); + downloadFutures.add(downloadExecutor.submit(call)); if (ctr == 3) { ctr = 0; - for (Future future : futures) { + Iterator> itr = downloadFutures.iterator(); + while (itr.hasNext()) { + final Future future = itr.next(); while (!future.isDone()) { try { Thread.sleep(1000); @@ -178,52 +183,64 @@ public class StandardUpdateTask { Logger.getLogger(StandardUpdateTask.class.getName()).log(Level.FINE, null, ex); } } - } + final CallableDownloadTask filePair; + try { + filePair = future.get(); + } catch (InterruptedException ex) { + downloadExecutor.shutdownNow(); + Logger.getLogger(StandardUpdateTask.class.getName()).log(Level.FINE, "Thread was interupted", ex); + throw new UpdateException(ex); + } catch (ExecutionException ex) { + downloadExecutor.shutdownNow(); + Logger.getLogger(StandardUpdateTask.class.getName()).log(Level.SEVERE, null, ex); + throw new UpdateException(ex); + } + itr.remove(); + final ProcessTask task = new ProcessTask(cveDB, properties, filePair); + processFutures.add(processExecutor.submit(task)); + } } } } try { - for (Future future : futures) { + Iterator> itr = downloadFutures.iterator(); + while (itr.hasNext()) { + final Future future = itr.next(); final CallableDownloadTask filePair = future.get(); - String msg = String.format("Processing Started for NVD CVE - %s", filePair.getNvdCveInfo().getId()); - Logger.getLogger(StandardUpdateTask.class.getName()).log(Level.INFO, msg); - try { - importXML(filePair.getFirst(), filePair.getSecond()); - cveDB.commit(); - properties.save(filePair.getNvdCveInfo()); - } catch (FileNotFoundException ex) { - throw new UpdateException(ex); - } catch (ParserConfigurationException ex) { - throw new UpdateException(ex); - } catch (SAXException ex) { - throw new UpdateException(ex); - } catch (IOException ex) { - throw new UpdateException(ex); - } catch (SQLException ex) { - throw new UpdateException(ex); - } catch (DatabaseException ex) { - throw new UpdateException(ex); - } catch (ClassNotFoundException ex) { - throw new UpdateException(ex); - } finally { - filePair.cleanup(); - } - msg = String.format("Processing Complete for NVD CVE - %s", filePair.getNvdCveInfo().getId()); - Logger.getLogger(StandardUpdateTask.class.getName()).log(Level.INFO, msg); + final ProcessTask task = new ProcessTask(cveDB, properties, filePair); + processFutures.add(processExecutor.submit(task)); } } catch (InterruptedException ex) { - executorService.shutdownNow(); - Logger.getLogger(StandardUpdateTask.class.getName()).log(Level.FINE, "Thread was interupted", ex); + downloadExecutor.shutdownNow(); + Logger.getLogger(StandardUpdateTask.class.getName()).log(Level.FINE, "Thread was interupted during download", ex); throw new UpdateException(ex); } catch (ExecutionException ex) { - executorService.shutdownNow(); - Logger.getLogger(StandardUpdateTask.class.getName()).log(Level.SEVERE, null, ex); + downloadExecutor.shutdownNow(); + Logger.getLogger(StandardUpdateTask.class.getName()).log(Level.FINE, "Execution Exception during download", ex); throw new UpdateException(ex); } finally { - //yes, this should likely not be in the finally because of the shutdownNow above. - executorService.shutdown(); + downloadExecutor.shutdown(); + } + + for (Future future : processFutures) { + try { + ProcessTask task = future.get(); + if (task.getException() != null) { + throw task.getException(); + } + } catch (InterruptedException ex) { + processExecutor.shutdownNow(); + Logger.getLogger(StandardUpdateTask.class.getName()).log(Level.FINE, "Thread was interupted during processing", ex); + throw new UpdateException(ex); + } catch (ExecutionException ex) { + processExecutor.shutdownNow(); + Logger.getLogger(StandardUpdateTask.class.getName()).log(Level.FINE, "Execution Exception during process", ex); + throw new UpdateException(ex); + } finally { + processExecutor.shutdown(); + } } if (maxUpdates >= 1) { //ensure the modified file date gets written @@ -347,7 +364,7 @@ public class StandardUpdateTask { * @throws UpdateException Is thrown if there is an issue with the last * updated properties file */ - protected Updateable updatesNeeded() throws MalformedURLException, DownloadFailedException, UpdateException { + final protected Updateable updatesNeeded() throws MalformedURLException, DownloadFailedException, UpdateException { Updateable updates = null; try { updates = retrieveCurrentTimestampsFromWeb(); @@ -541,34 +558,4 @@ public class StandardUpdateTask { final double differenceInDays = (compareTo - date) / 1000.0 / 60.0 / 60.0 / 24.0; return differenceInDays < range; } - - /** - * Imports the NVD CVE XML File into the Lucene Index. - * - * @param file the file containing the NVD CVE XML - * @param oldVersion contains the file containing the NVD CVE XML 1.2 - * @throws ParserConfigurationException is thrown if there is a parser - * configuration exception - * @throws SAXException is thrown if there is a SAXException - * @throws IOException is thrown if there is a IO Exception - * @throws SQLException is thrown if there is a SQL exception - * @throws DatabaseException is thrown if there is a database exception - * @throws ClassNotFoundException thrown if the h2 database driver cannot be - * loaded - */ - protected void importXML(File file, File oldVersion) throws ParserConfigurationException, - SAXException, IOException, SQLException, DatabaseException, ClassNotFoundException { - - final SAXParserFactory factory = SAXParserFactory.newInstance(); - final SAXParser saxParser = factory.newSAXParser(); - - final NvdCve12Handler cve12Handler = new NvdCve12Handler(); - saxParser.parse(oldVersion, cve12Handler); - final Map> prevVersionVulnMap = cve12Handler.getVulnerabilities(); - - final NvdCve20Handler cve20Handler = new NvdCve20Handler(); - cve20Handler.setCveDB(cveDB); - cve20Handler.setPrevVersionVulnMap(prevVersionVulnMap); - saxParser.parse(file, cve20Handler); - } }