From 70103dcca3f3bbcfe680049c434a9ab4e3487c43 Mon Sep 17 00:00:00 2001 From: Jeremy Long Date: Tue, 3 Dec 2013 06:05:31 -0500 Subject: [PATCH] version 1.0.7 --- .../data/update/ProcessTask.html | 365 +++++++ .../data/update/StandardUpdate.html | 519 ++++++++++ .../data/update/class-use/ProcessTask.html | 185 ++++ .../data/update/class-use/StandardUpdate.html | 145 +++ ...pendencycheck.data.update.ProcessTask.html | 271 +++++ ...dencycheck.data.update.StandardUpdate.html | 941 ++++++++++++++++++ .../update/StandardUpdateIntegrationTest.html | 155 +++ .../data/update/ProcessTask.html | 162 +++ .../data/update/StandardUpdate.html | 571 +++++++++++ 9 files changed, 3314 insertions(+) create mode 100644 dependency-check-core/apidocs/org/owasp/dependencycheck/data/update/ProcessTask.html create mode 100644 dependency-check-core/apidocs/org/owasp/dependencycheck/data/update/StandardUpdate.html create mode 100644 dependency-check-core/apidocs/org/owasp/dependencycheck/data/update/class-use/ProcessTask.html create mode 100644 dependency-check-core/apidocs/org/owasp/dependencycheck/data/update/class-use/StandardUpdate.html create mode 100644 dependency-check-core/cobertura/org.owasp.dependencycheck.data.update.ProcessTask.html create mode 100644 dependency-check-core/cobertura/org.owasp.dependencycheck.data.update.StandardUpdate.html create mode 100644 dependency-check-core/xref-test/org/owasp/dependencycheck/data/update/StandardUpdateIntegrationTest.html create mode 100644 dependency-check-core/xref/org/owasp/dependencycheck/data/update/ProcessTask.html create mode 100644 dependency-check-core/xref/org/owasp/dependencycheck/data/update/StandardUpdate.html diff --git a/dependency-check-core/apidocs/org/owasp/dependencycheck/data/update/ProcessTask.html b/dependency-check-core/apidocs/org/owasp/dependencycheck/data/update/ProcessTask.html new file mode 100644 index 000000000..e02ed370a --- /dev/null +++ b/dependency-check-core/apidocs/org/owasp/dependencycheck/data/update/ProcessTask.html @@ -0,0 +1,365 @@ + + + + + + + +ProcessTask (Dependency-Check Core 1.0.7 API) + + + + + + + + + + + + +
+ + + + + + + + + + + + + + + + + + + +
+ + + + + + + + + + + + +
Overview  Package   Class  Use  Tree  Deprecated  Index  Help 
+		 + +
+ PREV CLASS  + NEXT CLASS + FRAMES   + NO FRAMES   +  + + + +
+ SUMMARY: NESTED | FIELD | CONSTR | METHOD +DETAIL: FIELD | CONSTR | METHOD
+ + + +
+ +

+ +org.owasp.dependencycheck.data.update +
+Class ProcessTask

+
+java.lang.Object
+  extended by org.owasp.dependencycheck.data.update.ProcessTask
+
+
+
All Implemented Interfaces:
Callable<ProcessTask>
+
+
+
+
public class ProcessTask
extends Object
implements Callable<ProcessTask>
+ + +

+A callable task that will process a given set of NVD CVE xml files and update + the Cve Database accordingly. +

+ +

+

+
Author:
+
Jeremy Long (jeremy.long@owasp.org)
+
+
+ +

+ + + + + + + + + + + +
+Constructor Summary
ProcessTask(CveDB cveDB, + DataStoreMetaInfo properties, + CallableDownloadTask filePair) + +
+           
+  + + + + + + + + + + + + + + + + + + + + + + + +
+Method Summary
+ ProcessTaskcall() + +
+           
+ UpdateExceptiongetException() + +
+          Get the value of exception.
+protected  voidimportXML(File file, + File oldVersion) + +
+          Imports the NVD CVE XML File into the Lucene Index.
+ voidsetException(UpdateException exception) + +
+          Set the value of exception.
+  + + + + + + + +
Methods inherited from class java.lang.Object
clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait
+  +

+ + + + + + + + +
+Constructor Detail
+ +

+ProcessTask

+
+public ProcessTask(CveDB cveDB,
+                   DataStoreMetaInfo properties,
+                   CallableDownloadTask filePair)
+
+
+ + + + + + + + +
+Method Detail
+ +

+getException

+
+public UpdateException getException()
+
+
Get the value of exception. +

+

+
+
+
+ +
Returns:
the value of exception
+
+
+
+ +

+setException

+
+public void setException(UpdateException exception)
+
+
Set the value of exception. +

+

+
+
+
+
Parameters:
exception - new value of exception
+
+
+
+ +

+call

+
+public ProcessTask call()
+                 throws Exception
+
+
+
Specified by:
call in interface Callable<ProcessTask>
+
+
+ +
Throws: +
Exception
+
+
+
+ +

+importXML

+
+protected void importXML(File file,
+                         File oldVersion)
+                  throws ParserConfigurationException,
+                         SAXException,
+                         IOException,
+                         SQLException,
+                         DatabaseException,
+                         ClassNotFoundException
+
+
Imports the NVD CVE XML File into the Lucene Index. +

+

+
+
+
+
Parameters:
file - the file containing the NVD CVE XML
oldVersion - contains the file containing the NVD CVE XML 1.2 +
Throws: +
ParserConfigurationException - is thrown if there is a parser + configuration exception +
SAXException - is thrown if there is a SAXException +
IOException - is thrown if there is a IO Exception +
SQLException - is thrown if there is a SQL exception +
DatabaseException - is thrown if there is a database exception +
ClassNotFoundException - thrown if the h2 database driver cannot be + loaded
+
+
+ +
+ + + + + + + + + + + + + + + + + + + +
+ + + + + + + + + + + + +
Overview  Package   Class  Use  Tree  Deprecated  Index  Help 
+		 + +
+ PREV CLASS  + NEXT CLASS + FRAMES   + NO FRAMES   +  + + + +
+ SUMMARY: NESTED | FIELD | CONSTR | METHOD +DETAIL: FIELD | CONSTR | METHOD
+ + + +
+Copyright © 2012-2013 OWASP. All Rights Reserved. + + diff --git a/dependency-check-core/apidocs/org/owasp/dependencycheck/data/update/StandardUpdate.html b/dependency-check-core/apidocs/org/owasp/dependencycheck/data/update/StandardUpdate.html new file mode 100644 index 000000000..0eac58b41 --- /dev/null +++ b/dependency-check-core/apidocs/org/owasp/dependencycheck/data/update/StandardUpdate.html @@ -0,0 +1,519 @@ + + + + + + + +StandardUpdate (Dependency-Check Core 1.0.7 API) + + + + + + + + + + + + +
+ + + + + + + + + + + + + + + + + + + +
+ + + + + + + + + + + + +
Overview  Package   Class  Use  Tree  Deprecated  Index  Help 
+		 + +
+ PREV CLASS  + NEXT CLASS + FRAMES   + NO FRAMES   +  + + + +
+ SUMMARY: NESTED | FIELD | CONSTR | METHOD +DETAIL: FIELD | CONSTR | METHOD
+ + + +
+ +

+ +org.owasp.dependencycheck.data.update +
+Class StandardUpdate

+
+java.lang.Object
+  extended by org.owasp.dependencycheck.data.update.StandardUpdate
+
+
+
+
public class StandardUpdate
extends Object
+ + +

+Class responsible for updating the NVDCVE data store. +

+ +

+

+
Author:
+
Jeremy Long (jeremy.long@owasp.org)
+
+
+ +

+ + + + + + + + + + + +
+Field Summary
+static intMAX_THREAD_POOL_SIZE + +
+          The max thread pool size to use when downloading files.
+  + + + + + + + + + + +
+Constructor Summary
StandardUpdate() + +
+          Constructs a new Standard Update Task.
+  + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+Method Summary
+protected  voidcloseDataStores() + +
+          Closes the CVE and CPE data stores.
+protected  voiddeleteExistingData() + +
+          Deletes the existing data directories.
+ booleanisUpdateNeeded() + +
+          Gets whether or not an update is needed.
+protected  voidopenDataStores() + +
+          Opens the CVE and CPE data stores.
+protected  voidsetDeleteAndRecreate(boolean deleteAndRecreate) + +
+          Set the value of deleteAndRecreate.
+ booleanshouldDeleteAndRecreate() + +
+          Get the value of deleteAndRecreate.
+ voidupdate() + +
+          Downloads the latest NVD CVE XML file from the web and imports it into + the current CVE Database.
+protected  UpdateableupdatesNeeded() + +
+          Determines if the index needs to be updated.
+protected  booleanwithinRange(long date, + long compareTo, + int range) + +
+          Determines if the epoch date is within the range specified of the + compareTo epoch time.
+  + + + + + + + +
Methods inherited from class java.lang.Object
clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait
+  +

+ + + + + + + + +
+Field Detail
+ +

+MAX_THREAD_POOL_SIZE

+
+public static final int MAX_THREAD_POOL_SIZE
+
+
The max thread pool size to use when downloading files. +

+

+
+
+ + + + + + + + +
+Constructor Detail
+ +

+StandardUpdate

+
+public StandardUpdate()
+               throws MalformedURLException,
+                      DownloadFailedException,
+                      UpdateException
+
+
Constructs a new Standard Update Task. +

+

+ +
Throws: +
MalformedURLException - thrown if a configured URL is malformed +
DownloadFailedException - thrown if a timestamp cannot be checked + on a configured URL +
UpdateException - thrown if there is an exception generating the + update task
+
+ + + + + + + + +
+Method Detail
+ +

+isUpdateNeeded

+
+public boolean isUpdateNeeded()
+
+
Gets whether or not an update is needed. +

+

+ +
Returns:
true or false depending on whether an update is needed
+
+
+
+ +

+setDeleteAndRecreate

+
+protected void setDeleteAndRecreate(boolean deleteAndRecreate)
+
+
Set the value of deleteAndRecreate. +

+

+
Parameters:
deleteAndRecreate - new value of deleteAndRecreate
+
+
+
+ +

+shouldDeleteAndRecreate

+
+public boolean shouldDeleteAndRecreate()
+
+
Get the value of deleteAndRecreate. +

+

+ +
Returns:
the value of deleteAndRecreate
+
+
+
+ +

+update

+
+public void update()
+            throws UpdateException
+
+

Downloads the latest NVD CVE XML file from the web and imports it into + the current CVE Database.

+

+

+ +
Throws: +
UpdateException - is thrown if there is an error updating the + database
+
+
+
+ +

+updatesNeeded

+
+protected final Updateable updatesNeeded()
+                                  throws MalformedURLException,
+                                         DownloadFailedException,
+                                         UpdateException
+
+
Determines if the index needs to be updated. This is done by fetching the + NVD CVE meta data and checking the last update date. If the data needs to + be refreshed this method will return the NvdCveUrl for the files that + need to be updated. +

+

+ +
Returns:
the collection of files that need to be updated +
Throws: +
MalformedURLException - is thrown if the URL for the NVD CVE Meta + data is incorrect +
DownloadFailedException - is thrown if there is an error. + downloading the NVD CVE download data file +
UpdateException - Is thrown if there is an issue with the last + updated properties file
+
+
+
+ +

+deleteExistingData

+
+protected void deleteExistingData()
+                           throws IOException
+
+
Deletes the existing data directories. +

+

+ +
Throws: +
IOException - thrown if the directory cannot be deleted
+
+
+
+ +

+closeDataStores

+
+protected void closeDataStores()
+
+
Closes the CVE and CPE data stores. +

+

+
+
+
+
+ +

+openDataStores

+
+protected void openDataStores()
+                       throws UpdateException
+
+
Opens the CVE and CPE data stores. +

+

+ +
Throws: +
UpdateException - thrown if a data store cannot be opened
+
+
+
+ +

+withinRange

+
+protected boolean withinRange(long date,
+                              long compareTo,
+                              int range)
+
+
Determines if the epoch date is within the range specified of the + compareTo epoch time. This takes the (compareTo-date)/1000/60/60/24 to + get the number of days. If the calculated days is less then the range the + date is considered valid. +

+

+
Parameters:
date - the date to be checked.
compareTo - the date to compare to.
range - the range in days to be considered valid. +
Returns:
whether or not the date is within the range.
+
+
+ +
+ + + + + + + + + + + + + + + + + + + +
+ + + + + + + + + + + + +
Overview  Package   Class  Use  Tree  Deprecated  Index  Help 
+		 + +
+ PREV CLASS  + NEXT CLASS + FRAMES   + NO FRAMES   +  + + + +
+ SUMMARY: NESTED | FIELD | CONSTR | METHOD +DETAIL: FIELD | CONSTR | METHOD
+ + + +
+Copyright © 2012-2013 OWASP. All Rights Reserved. + + diff --git a/dependency-check-core/apidocs/org/owasp/dependencycheck/data/update/class-use/ProcessTask.html b/dependency-check-core/apidocs/org/owasp/dependencycheck/data/update/class-use/ProcessTask.html new file mode 100644 index 000000000..763c4e097 --- /dev/null +++ b/dependency-check-core/apidocs/org/owasp/dependencycheck/data/update/class-use/ProcessTask.html @@ -0,0 +1,185 @@ + + + + + + + +Uses of Class org.owasp.dependencycheck.data.update.ProcessTask (Dependency-Check Core 1.0.7 API) + + + + + + + + + + + + +
+ + + + + + + + + + + + + + + +
+ + + + + + + + + + + + +
Overview  Package  Class   Use  Tree  Deprecated  Index  Help 
+		 + +
+ PREV  + NEXT + FRAMES   + NO FRAMES   +  + + + +
+ + + +
+
+

+Uses of Class
org.owasp.dependencycheck.data.update.ProcessTask

+
+ + + + + + + + + +
+Packages that use ProcessTask
org.owasp.dependencycheck.data.update + + org.owasp.dependencycheck.data.nvdcve.xml + +  
+  +

+ + + + + +
+Uses of ProcessTask in org.owasp.dependencycheck.data.update
+  +

+ + + + + + + + + +
Methods in org.owasp.dependencycheck.data.update that return ProcessTask
+ ProcessTaskProcessTask.call() + +
+           
+  +

+

+ + + + + + + + + + + + + + + +
+ + + + + + + + + + + + +
Overview  Package  Class   Use  Tree  Deprecated  Index  Help 
+		 + +
+ PREV  + NEXT + FRAMES   + NO FRAMES   +  + + + +
+ + + +
+Copyright © 2012-2013 OWASP. All Rights Reserved. + + diff --git a/dependency-check-core/apidocs/org/owasp/dependencycheck/data/update/class-use/StandardUpdate.html b/dependency-check-core/apidocs/org/owasp/dependencycheck/data/update/class-use/StandardUpdate.html new file mode 100644 index 000000000..4069cfbbb --- /dev/null +++ b/dependency-check-core/apidocs/org/owasp/dependencycheck/data/update/class-use/StandardUpdate.html @@ -0,0 +1,145 @@ + + + + + + + +Uses of Class org.owasp.dependencycheck.data.update.StandardUpdate (Dependency-Check Core 1.0.7 API) + + + + + + + + + + + + +
+ + + + + + + + + + + + + + + +
+ + + + + + + + + + + + +
Overview  Package  Class   Use  Tree  Deprecated  Index  Help 
+		 + +
+ PREV  + NEXT + FRAMES   + NO FRAMES   +  + + + +
+ + + +
+
+

+Uses of Class
org.owasp.dependencycheck.data.update.StandardUpdate

+
+No usage of org.owasp.dependencycheck.data.update.StandardUpdate +

+

+ + + + + + + + + + + + + + + +
+ + + + + + + + + + + + +
Overview  Package  Class   Use  Tree  Deprecated  Index  Help 
+		 + +
+ PREV  + NEXT + FRAMES   + NO FRAMES   +  + + + +
+ + + +
+Copyright © 2012-2013 OWASP. All Rights Reserved. + + diff --git a/dependency-check-core/cobertura/org.owasp.dependencycheck.data.update.ProcessTask.html b/dependency-check-core/cobertura/org.owasp.dependencycheck.data.update.ProcessTask.html new file mode 100644 index 000000000..547c3e594 --- /dev/null +++ b/dependency-check-core/cobertura/org.owasp.dependencycheck.data.update.ProcessTask.html @@ -0,0 +1,271 @@ + + + + +Coverage Report + + + + +
Coverage Report - org.owasp.dependencycheck.data.update.ProcessTask
+
 
+ + + + +
Classes in this File Line Coverage Branch Coverage Complexity
ProcessTask
0%
0/49
N/A
N/A
3.5
+
 
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
 1  
 /*
 2  
  * This file is part of dependency-check-core.
 3  
  *
 4  
  * Dependency-check-core is free software: you can redistribute it and/or modify it
 5  
  * under the terms of the GNU General Public License as published by the Free
 6  
  * Software Foundation, either version 3 of the License, or (at your option) any
 7  
  * later version.
 8  
  *
 9  
  * Dependency-check-core is distributed in the hope that it will be useful, but
 10  
  * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 11  
  * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
 12  
  * details.
 13  
  *
 14  
  * You should have received a copy of the GNU General Public License along with
 15  
  * dependency-check-core. If not, see http://www.gnu.org/licenses/.
 16  
  *
 17  
  * Copyright (c) 2013 Jeremy Long. All Rights Reserved.
 18  
  */
 19  
 package org.owasp.dependencycheck.data.update;
 20  
 
 21  
 import java.io.File;
 22  
 import java.io.FileNotFoundException;
 23  
 import java.io.IOException;
 24  
 import java.sql.SQLException;
 25  
 import java.util.List;
 26  
 import java.util.Map;
 27  
 import java.util.concurrent.Callable;
 28  
 import java.util.logging.Level;
 29  
 import java.util.logging.Logger;
 30  
 import javax.xml.parsers.ParserConfigurationException;
 31  
 import javax.xml.parsers.SAXParser;
 32  
 import javax.xml.parsers.SAXParserFactory;
 33  
 import org.owasp.dependencycheck.data.UpdateException;
 34  
 import org.owasp.dependencycheck.data.nvdcve.CveDB;
 35  
 import org.owasp.dependencycheck.data.nvdcve.DatabaseException;
 36  
 import org.owasp.dependencycheck.data.nvdcve.NvdCve12Handler;
 37  
 import org.owasp.dependencycheck.data.nvdcve.NvdCve20Handler;
 38  
 import org.owasp.dependencycheck.dependency.VulnerableSoftware;
 39  
 import org.xml.sax.SAXException;
 40  
 
 41  
 /**
 42  
  * A callable task that will process a given set of NVD CVE xml files and update
 43  
  * the Cve Database accordingly.
 44  
  *
 45  
  * @author Jeremy Long (jeremy.long@owasp.org)
 46  
  */
 47  0 
 public class ProcessTask implements Callable<ProcessTask> {
 48  
 
 49  
     /**
 50  
      * A field to store any update exceptions that occur during the "call".
 51  
      */
 52  0 
     private UpdateException exception = null;
 53  
 
 54  
     /**
 55  
      * Get the value of exception.
 56  
      *
 57  
      * @return the value of exception
 58  
      */
 59  
     public UpdateException getException() {
 60  0 
         return exception;
 61  
     }
 62  
 
 63  
     /**
 64  
      * Set the value of exception.
 65  
      *
 66  
      * @param exception new value of exception
 67  
      */
 68  
     public void setException(UpdateException exception) {
 69  0 
         this.exception = exception;
 70  0 
     }
 71  
     private final CveDB cveDB;
 72  
     private final CallableDownloadTask filePair;
 73  
     private final DataStoreMetaInfo properties;
 74  
 
 75  0 
     public ProcessTask(final CveDB cveDB, final DataStoreMetaInfo properties, final CallableDownloadTask filePair) {
 76  0 
         this.cveDB = cveDB;
 77  0 
         this.filePair = filePair;
 78  0 
         this.properties = properties;
 79  0 
     }
 80  
 
 81  
     @Override
 82  
     public ProcessTask call() throws Exception {
 83  
         try {
 84  0 
             processFiles();
 85  0 
         } catch (UpdateException ex) {
 86  0 
             this.exception = ex;
 87  0 
         }
 88  0 
         return this;
 89  
     }
 90  
 
 91  
     /**
 92  
      * Imports the NVD CVE XML File into the Lucene Index.
 93  
      *
 94  
      * @param file the file containing the NVD CVE XML
 95  
      * @param oldVersion contains the file containing the NVD CVE XML 1.2
 96  
      * @throws ParserConfigurationException is thrown if there is a parser
 97  
      * configuration exception
 98  
      * @throws SAXException is thrown if there is a SAXException
 99  
      * @throws IOException is thrown if there is a IO Exception
 100  
      * @throws SQLException is thrown if there is a SQL exception
 101  
      * @throws DatabaseException is thrown if there is a database exception
 102  
      * @throws ClassNotFoundException thrown if the h2 database driver cannot be
 103  
      * loaded
 104  
      */
 105  
     protected void importXML(File file, File oldVersion) throws ParserConfigurationException,
 106  
             SAXException, IOException, SQLException, DatabaseException, ClassNotFoundException {
 107  
 
 108  0 
         final SAXParserFactory factory = SAXParserFactory.newInstance();
 109  0 
         final SAXParser saxParser = factory.newSAXParser();
 110  
 
 111  0 
         final NvdCve12Handler cve12Handler = new NvdCve12Handler();
 112  0 
         saxParser.parse(oldVersion, cve12Handler);
 113  0 
         final Map<String, List<VulnerableSoftware>> prevVersionVulnMap = cve12Handler.getVulnerabilities();
 114  
 
 115  0 
         final NvdCve20Handler cve20Handler = new NvdCve20Handler();
 116  0 
         cve20Handler.setCveDB(cveDB);
 117  0 
         cve20Handler.setPrevVersionVulnMap(prevVersionVulnMap);
 118  0 
         saxParser.parse(file, cve20Handler);
 119  0 
     }
 120  
 
 121  
     private void processFiles() throws UpdateException {
 122  0 
         String msg = String.format("Processing Started for NVD CVE - %s", filePair.getNvdCveInfo().getId());
 123  0 
         Logger.getLogger(StandardUpdate.class.getName()).log(Level.INFO, msg);
 124  
         try {
 125  0 
             importXML(filePair.getFirst(), filePair.getSecond());
 126  0 
             cveDB.commit();
 127  0 
             properties.save(filePair.getNvdCveInfo());
 128  0 
         } catch (FileNotFoundException ex) {
 129  0 
             throw new UpdateException(ex);
 130  0 
         } catch (ParserConfigurationException ex) {
 131  0 
             throw new UpdateException(ex);
 132  0 
         } catch (SAXException ex) {
 133  0 
             throw new UpdateException(ex);
 134  0 
         } catch (IOException ex) {
 135  0 
             throw new UpdateException(ex);
 136  0 
         } catch (SQLException ex) {
 137  0 
             throw new UpdateException(ex);
 138  0 
         } catch (DatabaseException ex) {
 139  0 
             throw new UpdateException(ex);
 140  0 
         } catch (ClassNotFoundException ex) {
 141  0 
             throw new UpdateException(ex);
 142  
         } finally {
 143  0 
             filePair.cleanup();
 144  0 
         }
 145  0 
         msg = String.format("Processing Complete for NVD CVE - %s", filePair.getNvdCveInfo().getId());
 146  0 
         Logger.getLogger(StandardUpdate.class.getName()).log(Level.INFO, msg);
 147  0 
     }
 148  
 }
+ +
Report generated by Cobertura 1.9.4.1 on 12/2/13 10:29 PM.
+ + diff --git a/dependency-check-core/cobertura/org.owasp.dependencycheck.data.update.StandardUpdate.html b/dependency-check-core/cobertura/org.owasp.dependencycheck.data.update.StandardUpdate.html new file mode 100644 index 000000000..d6442a69e --- /dev/null +++ b/dependency-check-core/cobertura/org.owasp.dependencycheck.data.update.StandardUpdate.html @@ -0,0 +1,941 @@ + + + + +Coverage Report + + + + +
Coverage Report - org.owasp.dependencycheck.data.update.StandardUpdate
+
 
+ + + + +
Classes in this File Line Coverage Branch Coverage Complexity
StandardUpdate
0%
0/197
0%
0/68
6.583
+
 
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
 1  
 /*
 2  
  * This file is part of dependency-check-core.
 3  
  *
 4  
  * Dependency-check-core is free software: you can redistribute it and/or modify it
 5  
  * under the terms of the GNU General Public License as published by the Free
 6  
  * Software Foundation, either version 3 of the License, or (at your option) any
 7  
  * later version.
 8  
  *
 9  
  * Dependency-check-core is distributed in the hope that it will be useful, but
 10  
  * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 11  
  * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
 12  
  * details.
 13  
  *
 14  
  * You should have received a copy of the GNU General Public License along with
 15  
  * dependency-check-core. If not, see http://www.gnu.org/licenses/.
 16  
  *
 17  
  * Copyright (c) 2012 Jeremy Long. All Rights Reserved.
 18  
  */
 19  
 package org.owasp.dependencycheck.data.update;
 20  
 
 21  
 import org.owasp.dependencycheck.data.nvdcve.InvalidDataException;
 22  
 import java.io.File;
 23  
 import java.io.IOException;
 24  
 import java.net.MalformedURLException;
 25  
 import java.sql.SQLException;
 26  
 import java.util.Calendar;
 27  
 import java.util.Date;
 28  
 import java.util.HashSet;
 29  
 import java.util.Iterator;
 30  
 import java.util.Set;
 31  
 import java.util.concurrent.ExecutionException;
 32  
 import java.util.concurrent.ExecutorService;
 33  
 import java.util.concurrent.Executors;
 34  
 import java.util.concurrent.Future;
 35  
 import java.util.logging.Level;
 36  
 import java.util.logging.Logger;
 37  
 import org.owasp.dependencycheck.data.UpdateException;
 38  
 import org.owasp.dependencycheck.data.nvdcve.CveDB;
 39  
 import org.owasp.dependencycheck.utils.DownloadFailedException;
 40  
 import org.owasp.dependencycheck.utils.Settings;
 41  
 import org.owasp.dependencycheck.data.nvdcve.DatabaseException;
 42  
 import org.owasp.dependencycheck.utils.InvalidSettingException;
 43  
 import static org.owasp.dependencycheck.data.update.DataStoreMetaInfo.MODIFIED;
 44  
 import org.owasp.dependencycheck.utils.FileUtils;
 45  
 
 46  
 /**
 47  
  * Class responsible for updating the NVDCVE data store.
 48  
  *
 49  
  * @author Jeremy Long (jeremy.long@owasp.org)
 50  
  */
 51  
 public class StandardUpdate {
 52  
 
 53  
     /**
 54  
      * The max thread pool size to use when downloading files.
 55  
      */
 56  0 
     public static final int MAX_THREAD_POOL_SIZE = Settings.getInt(Settings.KEYS.MAX_DOWNLOAD_THREAD_POOL_SIZE, 3);
 57  
     /**
 58  
      * Information about the timestamps and URLs for data that needs to be
 59  
      * updated.
 60  
      */
 61  
     private DataStoreMetaInfo properties;
 62  
     /**
 63  
      * A collection of updateable NVD CVE items.
 64  
      */
 65  
     private Updateable updateable;
 66  
     /**
 67  
      * A flag indicating whether or not the current data store should be
 68  
      * deleted.
 69  
      */
 70  0 
     private boolean deleteAndRecreate = false;
 71  
     /**
 72  
      * Reference to the Cve Database.
 73  
      */
 74  0 
     private CveDB cveDB = null;
 75  
 
 76  
     /**
 77  
      * Gets whether or not an update is needed.
 78  
      *
 79  
      * @return true or false depending on whether an update is needed
 80  
      */
 81  
     public boolean isUpdateNeeded() {
 82  0 
         return updateable.isUpdateNeeded();
 83  
     }
 84  
 
 85  
     /**
 86  
      * Set the value of deleteAndRecreate.
 87  
      *
 88  
      * @param deleteAndRecreate new value of deleteAndRecreate
 89  
      */
 90  
     protected void setDeleteAndRecreate(boolean deleteAndRecreate) {
 91  0 
         this.deleteAndRecreate = deleteAndRecreate;
 92  0 
     }
 93  
 
 94  
     /**
 95  
      * Get the value of deleteAndRecreate.
 96  
      *
 97  
      * @return the value of deleteAndRecreate
 98  
      */
 99  
     public boolean shouldDeleteAndRecreate() {
 100  0 
         return deleteAndRecreate;
 101  
     }
 102  
 
 103  
     /**
 104  
      * Constructs a new Standard Update Task.
 105  
      *
 106  
      * @throws MalformedURLException thrown if a configured URL is malformed
 107  
      * @throws DownloadFailedException thrown if a timestamp cannot be checked
 108  
      * on a configured URL
 109  
      * @throws UpdateException thrown if there is an exception generating the
 110  
      * update task
 111  
      */
 112  0 
     public StandardUpdate() throws MalformedURLException, DownloadFailedException, UpdateException {
 113  0 
         properties = new DataStoreMetaInfo();
 114  0 
         updateable = updatesNeeded();
 115  0 
     }
 116  
 
 117  
     /**
 118  
      * <p>Downloads the latest NVD CVE XML file from the web and imports it into
 119  
      * the current CVE Database.</p>
 120  
      *
 121  
      * @throws UpdateException is thrown if there is an error updating the
 122  
      * database
 123  
      */
 124  
     public void update() throws UpdateException {
 125  0 
         int maxUpdates = 0;
 126  
         try {
 127  0 
             for (NvdCveInfo cve : updateable) {
 128  0 
                 if (cve.getNeedsUpdate()) {
 129  0 
                     maxUpdates += 1;
 130  
                 }
 131  
             }
 132  0 
             if (maxUpdates <= 0) {
 133  
                 return;
 134  
             }
 135  0 
             if (maxUpdates > 3) {
 136  0 
                 Logger.getLogger(StandardUpdate.class.getName()).log(Level.INFO,
 137  
                         "NVD CVE requires several updates; this could take a couple of minutes.");
 138  
             }
 139  0 
             if (maxUpdates > 0) {
 140  0 
                 openDataStores();
 141  
             }
 142  
 
 143  0 
             final int poolSize = (MAX_THREAD_POOL_SIZE > maxUpdates) ? MAX_THREAD_POOL_SIZE : maxUpdates;
 144  0 
             final ExecutorService downloadExecutor = Executors.newFixedThreadPool(poolSize);
 145  0 
             final ExecutorService processExecutor = Executors.newSingleThreadExecutor();
 146  0 
             final Set<Future<CallableDownloadTask>> downloadFutures = new HashSet<Future<CallableDownloadTask>>(maxUpdates);
 147  0 
             final Set<Future<ProcessTask>> processFutures = new HashSet<Future<ProcessTask>>(maxUpdates);
 148  0 
             int ctr = 0;
 149  0 
             for (NvdCveInfo cve : updateable) {
 150  0 
                 if (cve.getNeedsUpdate()) {
 151  0 
                     ctr += 1;
 152  
                     final File file1;
 153  
                     final File file2;
 154  
                     try {
 155  0 
                         file1 = File.createTempFile("cve" + cve.getId() + "_", ".xml");
 156  0 
                         file2 = File.createTempFile("cve_1_2_" + cve.getId() + "_", ".xml");
 157  0 
                     } catch (IOException ex) {
 158  0 
                         throw new UpdateException(ex);
 159  0 
                     }
 160  0 
                     final CallableDownloadTask call = new CallableDownloadTask(cve, file1, file2);
 161  0 
                     downloadFutures.add(downloadExecutor.submit(call));
 162  
 
 163  0 
                     boolean waitForFuture = ctr % 2 == 0;
 164  
 
 165  0 
                     final Iterator<Future<CallableDownloadTask>> itr = downloadFutures.iterator();
 166  0 
                     while (itr.hasNext()) {
 167  0 
                         final Future<CallableDownloadTask> future = itr.next();
 168  0 
                         if (waitForFuture) { //only allow two NVD/CVE files to be downloaded at a time
 169  0 
                             spinWaitForFuture(future);
 170  
                         }
 171  0 
                         if (future.isDone()) { //if we find something complete, add it to the process queue
 172  
                             try {
 173  0 
                                 final CallableDownloadTask filePair = future.get();
 174  0 
                                 itr.remove();
 175  0 
                                 final ProcessTask task = new ProcessTask(cveDB, properties, filePair);
 176  0 
                                 processFutures.add(processExecutor.submit(task));
 177  0 
                             } catch (InterruptedException ex) {
 178  0 
                                 downloadExecutor.shutdownNow();
 179  0 
                                 Logger.getLogger(StandardUpdate.class.getName()).log(Level.FINE, "Thread was interupted", ex);
 180  0 
                                 throw new UpdateException(ex);
 181  0 
                             } catch (ExecutionException ex) {
 182  0 
                                 downloadExecutor.shutdownNow();
 183  0 
                                 Logger.getLogger(StandardUpdate.class.getName()).log(Level.SEVERE, null, ex);
 184  0 
                                 throw new UpdateException(ex);
 185  0 
                             }
 186  
                         }
 187  0 
                     }
 188  
 
 189  0 
                 }
 190  
             }
 191  
 
 192  
             try {
 193  0 
                 final Iterator<Future<CallableDownloadTask>> itr = downloadFutures.iterator();
 194  0 
                 while (itr.hasNext()) {
 195  0 
                     final Future<CallableDownloadTask> future = itr.next();
 196  0 
                     final CallableDownloadTask filePair = future.get();
 197  0 
                     final ProcessTask task = new ProcessTask(cveDB, properties, filePair);
 198  0 
                     processFutures.add(processExecutor.submit(task));
 199  0 
                 }
 200  0 
             } catch (InterruptedException ex) {
 201  0 
                 downloadExecutor.shutdownNow();
 202  0 
                 Logger.getLogger(StandardUpdate.class.getName()).log(Level.FINE, "Thread was interupted during download", ex);
 203  0 
                 throw new UpdateException(ex);
 204  0 
             } catch (ExecutionException ex) {
 205  0 
                 downloadExecutor.shutdownNow();
 206  0 
                 Logger.getLogger(StandardUpdate.class.getName()).log(Level.FINE, "Execution Exception during download", ex);
 207  0 
                 throw new UpdateException(ex);
 208  
             } finally {
 209  0 
                 downloadExecutor.shutdown();
 210  0 
             }
 211  
 
 212  0 
             for (Future<ProcessTask> future : processFutures) {
 213  
                 try {
 214  0 
                     final ProcessTask task = future.get();
 215  0 
                     if (task.getException() != null) {
 216  0 
                         throw task.getException();
 217  
                     }
 218  0 
                 } catch (InterruptedException ex) {
 219  0 
                     processExecutor.shutdownNow();
 220  0 
                     Logger.getLogger(StandardUpdate.class.getName()).log(Level.FINE, "Thread was interupted during processing", ex);
 221  0 
                     throw new UpdateException(ex);
 222  0 
                 } catch (ExecutionException ex) {
 223  0 
                     processExecutor.shutdownNow();
 224  0 
                     Logger.getLogger(StandardUpdate.class.getName()).log(Level.FINE, "Execution Exception during process", ex);
 225  0 
                     throw new UpdateException(ex);
 226  
                 } finally {
 227  0 
                     processExecutor.shutdown();
 228  0 
                 }
 229  
             }
 230  
 
 231  0 
             if (maxUpdates >= 1) { //ensure the modified file date gets written
 232  0 
                 properties.save(updateable.get(MODIFIED));
 233  0 
                 cveDB.cleanupDatabase();
 234  
             }
 235  
         } finally {
 236  0 
             closeDataStores();
 237  0 
         }
 238  0 
     }
 239  
 
 240  
     //<editor-fold defaultstate="collapsed" desc="OLD version of update() - not multithreaded">
 241  
     /*
 242  
      * TODO - remove this
 243  
      public void update() throws UpdateException {
 244  
      try {
 245  
      int maxUpdates = 0;
 246  
      for (NvdCveInfo cve : getUpdateable()) {
 247  
      if (cve.getNeedsUpdate()) {
 248  
      maxUpdates += 1;
 249  
      }
 250  
      }
 251  
      if (maxUpdates > 3) {
 252  
      Logger.getLogger(StandardUpdate.class.getName()).log(Level.INFO,
 253  
      "NVD CVE requires several updates; this could take a couple of minutes.");
 254  
      }
 255  
      if (maxUpdates > 0) {
 256  
      openDataStores();
 257  
      }
 258  
 
 259  
      int count = 0;
 260  
      for (NvdCveInfo cve : getUpdateable()) {
 261  
      if (cve.getNeedsUpdate()) {
 262  
      count += 1;
 263  
      Logger.getLogger(StandardUpdate.class.getName()).log(Level.INFO,
 264  
      "Updating NVD CVE ({0} of {1})", new Object[]{count, maxUpdates});
 265  
      URL url = new URL(cve.getUrl());
 266  
      File outputPath = null;
 267  
      File outputPath12 = null;
 268  
      try {
 269  
      Logger.getLogger(StandardUpdate.class.getName()).log(Level.INFO,
 270  
      "Downloading {0}", cve.getUrl());
 271  
      outputPath = File.createTempFile("cve" + cve.getId() + "_", ".xml");
 272  
      Downloader.fetchFile(url, outputPath);
 273  
 
 274  
      url = new URL(cve.getOldSchemaVersionUrl());
 275  
      outputPath12 = File.createTempFile("cve_1_2_" + cve.getId() + "_", ".xml");
 276  
      Downloader.fetchFile(url, outputPath12);
 277  
 
 278  
      Logger.getLogger(StandardUpdate.class.getName()).log(Level.INFO,
 279  
      "Processing {0}", cve.getUrl());
 280  
 
 281  
      importXML(outputPath, outputPath12);
 282  
 
 283  
      getCveDB().commit();
 284  
      getProperties().save(cve);
 285  
 
 286  
      Logger.getLogger(StandardUpdate.class.getName()).log(Level.INFO,
 287  
      "Completed update {0} of {1}", new Object[]{count, maxUpdates});
 288  
      } catch (FileNotFoundException ex) {
 289  
      throw new UpdateException(ex);
 290  
      } catch (ParserConfigurationException ex) {
 291  
      throw new UpdateException(ex);
 292  
      } catch (SAXException ex) {
 293  
      throw new UpdateException(ex);
 294  
      } catch (IOException ex) {
 295  
      throw new UpdateException(ex);
 296  
      } catch (SQLException ex) {
 297  
      throw new UpdateException(ex);
 298  
      } catch (DatabaseException ex) {
 299  
      throw new UpdateException(ex);
 300  
      } catch (ClassNotFoundException ex) {
 301  
      throw new UpdateException(ex);
 302  
      } finally {
 303  
      boolean deleted = false;
 304  
      try {
 305  
      if (outputPath != null && outputPath.exists()) {
 306  
      deleted = outputPath.delete();
 307  
      }
 308  
      } finally {
 309  
      if (outputPath != null && (outputPath.exists() || !deleted)) {
 310  
      outputPath.deleteOnExit();
 311  
      }
 312  
      }
 313  
      try {
 314  
      deleted = false;
 315  
      if (outputPath12 != null && outputPath12.exists()) {
 316  
      deleted = outputPath12.delete();
 317  
      }
 318  
      } finally {
 319  
      if (outputPath12 != null && (outputPath12.exists() || !deleted)) {
 320  
      outputPath12.deleteOnExit();
 321  
      }
 322  
      }
 323  
      }
 324  
      }
 325  
      }
 326  
      if (maxUpdates >= 1) { //ensure the modified file date gets written
 327  
      getProperties().save(getUpdateable().get(MODIFIED));
 328  
      getCveDB().cleanupDatabase();
 329  
      }
 330  
      } catch (MalformedURLException ex) {
 331  
      throw new UpdateException(ex);
 332  
      } finally {
 333  
      closeDataStores();
 334  
      }
 335  
      }
 336  
      */
 337  
     //</editor-fold>
 338  
     /**
 339  
      * Determines if the index needs to be updated. This is done by fetching the
 340  
      * NVD CVE meta data and checking the last update date. If the data needs to
 341  
      * be refreshed this method will return the NvdCveUrl for the files that
 342  
      * need to be updated.
 343  
      *
 344  
      * @return the collection of files that need to be updated
 345  
      * @throws MalformedURLException is thrown if the URL for the NVD CVE Meta
 346  
      * data is incorrect
 347  
      * @throws DownloadFailedException is thrown if there is an error.
 348  
      * downloading the NVD CVE download data file
 349  
      * @throws UpdateException Is thrown if there is an issue with the last
 350  
      * updated properties file
 351  
      */
 352  
     protected final Updateable updatesNeeded() throws MalformedURLException, DownloadFailedException, UpdateException {
 353  0 
         Updateable updates = null;
 354  
         try {
 355  0 
             updates = retrieveCurrentTimestampsFromWeb();
 356  0 
         } catch (InvalidDataException ex) {
 357  0 
             final String msg = "Unable to retrieve valid timestamp from nvd cve downloads page";
 358  0 
             Logger
 359  
                     .getLogger(StandardUpdate.class
 360  
                     .getName()).log(Level.FINE, msg, ex);
 361  0 
             throw new DownloadFailedException(msg, ex);
 362  0 
         } catch (InvalidSettingException ex) {
 363  0 
             Logger.getLogger(StandardUpdate.class
 364  
                     .getName()).log(Level.FINE, "Invalid setting found when retrieving timestamps", ex);
 365  0 
             throw new DownloadFailedException(
 366  
                     "Invalid settings", ex);
 367  0 
         }
 368  
 
 369  0 
         if (updates == null) {
 370  0 
             throw new DownloadFailedException("Unable to retrieve the timestamps of the currently published NVD CVE data");
 371  
         }
 372  0 
         if (!properties.isEmpty()) {
 373  
             try {
 374  
                 float version;
 375  
 
 376  0 
                 if (properties.getProperty("version") == null) {
 377  0 
                     deleteAndRecreate = true;
 378  
                 } else {
 379  
                     try {
 380  0 
                         version = Float.parseFloat(properties.getProperty("version"));
 381  0 
                         final float currentVersion = Float.parseFloat(CveDB.DB_SCHEMA_VERSION);
 382  0 
                         if (currentVersion > version) {
 383  0 
                             deleteAndRecreate = true;
 384  
                         }
 385  0 
                     } catch (NumberFormatException ex) {
 386  0 
                         deleteAndRecreate = true;
 387  0 
                     }
 388  
                 }
 389  
 
 390  0 
                 if (deleteAndRecreate) {
 391  0 
                     return updates;
 392  
                 }
 393  
 
 394  0 
                 final long lastUpdated = Long.parseLong(properties.getProperty(DataStoreMetaInfo.LAST_UPDATED, "0"));
 395  0 
                 final Date now = new Date();
 396  0 
                 final int days = Settings.getInt(Settings.KEYS.CVE_MODIFIED_VALID_FOR_DAYS, 7);
 397  0 
                 if (lastUpdated == updates.getTimeStamp(MODIFIED)) {
 398  0 
                     updates.clear(); //we don't need to update anything.
 399  0 
                 } else if (withinRange(lastUpdated, now.getTime(), days)) {
 400  0 
                     for (NvdCveInfo entry : updates) {
 401  0 
                         if (MODIFIED.equals(entry.getId())) {
 402  0 
                             entry.setNeedsUpdate(true);
 403  
                         } else {
 404  0 
                             entry.setNeedsUpdate(false);
 405  
                         }
 406  
                     }
 407  
                 } else { //we figure out which of the several XML files need to be downloaded.
 408  0 
                     for (NvdCveInfo entry : updates) {
 409  0 
                         if (MODIFIED.equals(entry.getId())) {
 410  0 
                             entry.setNeedsUpdate(true);
 411  
                         } else {
 412  0 
                             long currentTimestamp = 0;
 413  
                             try {
 414  0 
                                 currentTimestamp = Long.parseLong(properties.getProperty(DataStoreMetaInfo.LAST_UPDATED_BASE + entry.getId(), "0"));
 415  0 
                             } catch (NumberFormatException ex) {
 416  0 
                                 final String msg = String.format("Error parsing '%s' '%s' from nvdcve.lastupdated",
 417  
                                         DataStoreMetaInfo.LAST_UPDATED_BASE, entry.getId());
 418  0 
                                 Logger
 419  
                                         .getLogger(StandardUpdate.class
 420  
                                         .getName()).log(Level.FINE, msg, ex);
 421  0 
                             }
 422  0 
                             if (currentTimestamp == entry.getTimestamp()) {
 423  0 
                                 entry.setNeedsUpdate(false);
 424  
                             }
 425  0 
                         }
 426  
                     }
 427  
                 }
 428  0 
             } catch (NumberFormatException ex) {
 429  0 
                 final String msg = "An invalid schema version or timestamp exists in the data.properties file.";
 430  0 
                 Logger
 431  
                         .getLogger(StandardUpdate.class
 432  
                         .getName()).log(Level.WARNING, msg);
 433  0 
                 Logger.getLogger(StandardUpdate.class
 434  
                         .getName()).log(Level.FINE, null, ex);
 435  0 
             }
 436  
         }
 437  0 
         return updates;
 438  
     }
 439  
 
 440  
     /**
 441  
      * Retrieves the timestamps from the NVD CVE meta data file.
 442  
      *
 443  
      * @return the timestamp from the currently published nvdcve downloads page
 444  
      * @throws MalformedURLException thrown if the URL for the NVD CCE Meta data
 445  
      * is incorrect.
 446  
      * @throws DownloadFailedException thrown if there is an error downloading
 447  
      * the nvd cve meta data file
 448  
      * @throws InvalidDataException thrown if there is an exception parsing the
 449  
      * timestamps
 450  
      * @throws InvalidSettingException thrown if the settings are invalid
 451  
      */
 452  
     private Updateable retrieveCurrentTimestampsFromWeb()
 453  
             throws MalformedURLException, DownloadFailedException, InvalidDataException, InvalidSettingException {
 454  
 
 455  0 
         final Updateable updates = new Updateable();
 456  0 
         updates.add(MODIFIED, Settings.getString(Settings.KEYS.CVE_MODIFIED_20_URL),
 457  
                 Settings.getString(Settings.KEYS.CVE_MODIFIED_12_URL),
 458  
                 false);
 459  
 
 460  0 
         final int start = Settings.getInt(Settings.KEYS.CVE_START_YEAR);
 461  0 
         final int end = Calendar.getInstance().get(Calendar.YEAR);
 462  0 
         final String baseUrl20 = Settings.getString(Settings.KEYS.CVE_SCHEMA_2_0);
 463  0 
         final String baseUrl12 = Settings.getString(Settings.KEYS.CVE_SCHEMA_1_2);
 464  0 
         for (int i = start; i <= end; i++) {
 465  0 
             updates.add(Integer.toString(i), String.format(baseUrl20, i),
 466  
                     String.format(baseUrl12, i),
 467  
                     true);
 468  
         }
 469  
 
 470  0 
         return updates;
 471  
     }
 472  
 
 473  
     /**
 474  
      * Deletes the existing data directories.
 475  
      *
 476  
      * @throws IOException thrown if the directory cannot be deleted
 477  
      */
 478  
     protected void deleteExistingData() throws IOException {
 479  0 
         File data = Settings.getDataFile(Settings.KEYS.CVE_DATA_DIRECTORY);
 480  0 
         if (data.exists()) {
 481  0 
             FileUtils.delete(data);
 482  
         }
 483  0 
         data = DataStoreMetaInfo.getPropertiesFile();
 484  0 
         if (data.exists()) {
 485  0 
             FileUtils.delete(data);
 486  
         }
 487  0 
     }
 488  
 
 489  
     /**
 490  
      * Closes the CVE and CPE data stores.
 491  
      */
 492  
     protected void closeDataStores() {
 493  0 
         if (cveDB != null) {
 494  
             try {
 495  0 
                 cveDB.close();
 496  0 
             } catch (Exception ignore) {
 497  0 
                 Logger.getLogger(StandardUpdate.class.getName()).log(Level.FINEST, "Error closing the cveDB", ignore);
 498  0 
             }
 499  
         }
 500  0 
     }
 501  
 
 502  
     /**
 503  
      * Opens the CVE and CPE data stores.
 504  
      *
 505  
      * @throws UpdateException thrown if a data store cannot be opened
 506  
      */
 507  
     protected void openDataStores() throws UpdateException {
 508  
         //open the cve and cpe data stores
 509  
         try {
 510  0 
             cveDB = new CveDB();
 511  0 
             cveDB.open();
 512  0 
         } catch (IOException ex) {
 513  0 
             closeDataStores();
 514  0 
             Logger.getLogger(StandardUpdate.class.getName()).log(Level.FINE, "IO Error opening databases", ex);
 515  0 
             throw new UpdateException("Error updating the CPE/CVE data, please see the log file for more details.");
 516  0 
         } catch (SQLException ex) {
 517  0 
             closeDataStores();
 518  0 
             Logger.getLogger(StandardUpdate.class.getName()).log(Level.FINE, "SQL Exception opening databases", ex);
 519  0 
             throw new UpdateException("Error updating the CPE/CVE data, please see the log file for more details.");
 520  0 
         } catch (DatabaseException ex) {
 521  0 
             closeDataStores();
 522  0 
             Logger.getLogger(StandardUpdate.class.getName()).log(Level.FINE, "Database Exception opening databases", ex);
 523  0 
             throw new UpdateException("Error updating the CPE/CVE data, please see the log file for more details.");
 524  0 
         } catch (ClassNotFoundException ex) {
 525  0 
             closeDataStores();
 526  0 
             Logger.getLogger(StandardUpdate.class.getName()).log(Level.FINE, "Class not found exception opening databases", ex);
 527  0 
             throw new UpdateException("Error updating the CPE/CVE data, please see the log file for more details.");
 528  0 
         }
 529  0 
     }
 530  
 
 531  
     /**
 532  
      * Determines if the epoch date is within the range specified of the
 533  
      * compareTo epoch time. This takes the (compareTo-date)/1000/60/60/24 to
 534  
      * get the number of days. If the calculated days is less then the range the
 535  
      * date is considered valid.
 536  
      *
 537  
      * @param date the date to be checked.
 538  
      * @param compareTo the date to compare to.
 539  
      * @param range the range in days to be considered valid.
 540  
      * @return whether or not the date is within the range.
 541  
      */
 542  
     protected boolean withinRange(long date, long compareTo, int range) {
 543  0 
         final double differenceInDays = (compareTo - date) / 1000.0 / 60.0 / 60.0 / 24.0;
 544  0 
         return differenceInDays < range;
 545  
     }
 546  
 
 547  
     private void spinWaitForFuture(final Future<CallableDownloadTask> future) {
 548  
         //then wait for downloads to finish
 549  0 
         while (!future.isDone()) {
 550  
             try {
 551  0 
                 Thread.sleep(1000);
 552  0 
             } catch (InterruptedException ex) {
 553  0 
                 Logger.getLogger(StandardUpdate.class.getName()).log(Level.FINE, null, ex);
 554  0 
             }
 555  
         }
 556  0 
     }
 557  
 }
+ +
Report generated by Cobertura 1.9.4.1 on 12/2/13 10:29 PM.
+ + diff --git a/dependency-check-core/xref-test/org/owasp/dependencycheck/data/update/StandardUpdateIntegrationTest.html b/dependency-check-core/xref-test/org/owasp/dependencycheck/data/update/StandardUpdateIntegrationTest.html new file mode 100644 index 000000000..33458c468 --- /dev/null +++ b/dependency-check-core/xref-test/org/owasp/dependencycheck/data/update/StandardUpdateIntegrationTest.html @@ -0,0 +1,155 @@ + + + + +StandardUpdateIntegrationTest xref + + + +
View Javadoc
+
+1   /*
+2    * This file is part of dependency-check-core.
+3    *
+4    * Dependency-check-core is free software: you can redistribute it and/or modify it
+5    * under the terms of the GNU General Public License as published by the Free
+6    * Software Foundation, either version 3 of the License, or (at your option) any
+7    * later version.
+8    *
+9    * Dependency-check-core is distributed in the hope that it will be useful, but
+10   * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+11   * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+12   * details.
+13   *
+14   * You should have received a copy of the GNU General Public License along with
+15   * dependency-check-core. If not, see http://www.gnu.org/licenses/.
+16   *
+17   * Copyright (c) 2013 Jeremy Long. All Rights Reserved.
+18   */
+19  package org.owasp.dependencycheck.data.update;
+20  
+21  import java.io.IOException;
+22  import java.net.MalformedURLException;
+23  import java.util.Calendar;
+24  import org.junit.After;
+25  import org.junit.AfterClass;
+26  import org.junit.Before;
+27  import org.junit.BeforeClass;
+28  import org.junit.Test;
+29  import static org.junit.Assert.*;
+30  import org.owasp.dependencycheck.data.UpdateException;
+31  import org.owasp.dependencycheck.utils.DownloadFailedException;
+32  
+33  /**
+34   *
+35   * @author Jeremy Long (jeremy.long@owasp.org)
+36   */
+37  public class StandardUpdateIntegrationTest {
+38  
+39      public StandardUpdateIntegrationTest() {
+40      }
+41  
+42      @BeforeClass
+43      public static void setUpClass() {
+44      }
+45  
+46      @AfterClass
+47      public static void tearDownClass() {
+48      }
+49  
+50      @Before
+51      public void setUp() {
+52      }
+53  
+54      @After
+55      public void tearDown() {
+56      }
+57  
+58      public StandardUpdate getStandardUpdateTask() throws MalformedURLException, DownloadFailedException, UpdateException {
+59          StandardUpdate instance = new StandardUpdate();
+60          return instance;
+61      }
+62  
+63      /**
+64       * Test of setDeleteAndRecreate method, of class StandardUpdate.
+65       */
+66      @Test
+67      public void testSetDeleteAndRecreate() throws Exception {
+68          boolean deleteAndRecreate = false;
+69          boolean expResult = false;
+70          StandardUpdate instance = getStandardUpdateTask();
+71          instance.setDeleteAndRecreate(deleteAndRecreate);
+72          boolean result = instance.shouldDeleteAndRecreate();
+73          assertEquals(expResult, result);
+74      }
+75  
+76      /**
+77       * Test of deleteExistingData method, of class StandardUpdate.
+78       */
+79      @Test
+80      public void testDeleteExistingData() throws Exception {
+81          StandardUpdate instance = getStandardUpdateTask();
+82          Exception result = null;
+83          try {
+84              instance.deleteExistingData();
+85          } catch (IOException ex) {
+86              result = ex;
+87          }
+88          assertNull(result);
+89      }
+90  
+91      /**
+92       * Test of openDataStores method, of class StandardUpdate.
+93       */
+94      @Test
+95      public void testOpenDataStores() throws Exception {
+96          StandardUpdate instance = getStandardUpdateTask();
+97          instance.openDataStores();
+98          instance.closeDataStores();
+99      }
+100 
+101     /**
+102      * Test of withinRange method, of class StandardUpdate.
+103      */
+104     @Test
+105     public void testWithinRange() throws Exception {
+106         Calendar c = Calendar.getInstance();
+107 
+108         long current = c.getTimeInMillis();
+109         long lastRun = c.getTimeInMillis() - (3 * (1000 * 60 * 60 * 24));
+110         int range = 7; // 7 days
+111         StandardUpdate instance = getStandardUpdateTask();
+112         boolean expResult = true;
+113         boolean result = instance.withinRange(lastRun, current, range);
+114         assertEquals(expResult, result);
+115 
+116         lastRun = c.getTimeInMillis() - (8 * (1000 * 60 * 60 * 24));
+117         expResult = false;
+118         result = instance.withinRange(lastRun, current, range);
+119         assertEquals(expResult, result);
+120     }
+121 
+122     /**
+123      * Test of update method, of class StandardUpdate.
+124      */
+125     @Test
+126     public void testUpdate() throws Exception {
+127         StandardUpdate instance = getStandardUpdateTask();
+128         instance.update();
+129         //TODO make this an actual test
+130     }
+131 
+132     /**
+133      * Test of updatesNeeded method, of class StandardUpdate.
+134      */
+135     @Test
+136     public void testUpdatesNeeded() throws Exception {
+137         StandardUpdate instance = getStandardUpdateTask();
+138         Updateable result = instance.updatesNeeded();
+139         assertNotNull(result);
+140     }
+141 }
+
+
+ + diff --git a/dependency-check-core/xref/org/owasp/dependencycheck/data/update/ProcessTask.html b/dependency-check-core/xref/org/owasp/dependencycheck/data/update/ProcessTask.html new file mode 100644 index 000000000..c75d278de --- /dev/null +++ b/dependency-check-core/xref/org/owasp/dependencycheck/data/update/ProcessTask.html @@ -0,0 +1,162 @@ + + + + +ProcessTask xref + + + +
View Javadoc
+
+1   /*
+2    * This file is part of dependency-check-core.
+3    *
+4    * Dependency-check-core is free software: you can redistribute it and/or modify it
+5    * under the terms of the GNU General Public License as published by the Free
+6    * Software Foundation, either version 3 of the License, or (at your option) any
+7    * later version.
+8    *
+9    * Dependency-check-core is distributed in the hope that it will be useful, but
+10   * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+11   * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+12   * details.
+13   *
+14   * You should have received a copy of the GNU General Public License along with
+15   * dependency-check-core. If not, see http://www.gnu.org/licenses/.
+16   *
+17   * Copyright (c) 2013 Jeremy Long. All Rights Reserved.
+18   */
+19  package org.owasp.dependencycheck.data.update;
+20  
+21  import java.io.File;
+22  import java.io.FileNotFoundException;
+23  import java.io.IOException;
+24  import java.sql.SQLException;
+25  import java.util.List;
+26  import java.util.Map;
+27  import java.util.concurrent.Callable;
+28  import java.util.logging.Level;
+29  import java.util.logging.Logger;
+30  import javax.xml.parsers.ParserConfigurationException;
+31  import javax.xml.parsers.SAXParser;
+32  import javax.xml.parsers.SAXParserFactory;
+33  import org.owasp.dependencycheck.data.UpdateException;
+34  import org.owasp.dependencycheck.data.nvdcve.CveDB;
+35  import org.owasp.dependencycheck.data.nvdcve.DatabaseException;
+36  import org.owasp.dependencycheck.data.nvdcve.NvdCve12Handler;
+37  import org.owasp.dependencycheck.data.nvdcve.NvdCve20Handler;
+38  import org.owasp.dependencycheck.dependency.VulnerableSoftware;
+39  import org.xml.sax.SAXException;
+40  
+41  /**
+42   * A callable task that will process a given set of NVD CVE xml files and update
+43   * the Cve Database accordingly.
+44   *
+45   * @author Jeremy Long (jeremy.long@owasp.org)
+46   */
+47  public class ProcessTask implements Callable<ProcessTask> {
+48  
+49      /**
+50       * A field to store any update exceptions that occur during the "call".
+51       */
+52      private UpdateException exception = null;
+53  
+54      /**
+55       * Get the value of exception.
+56       *
+57       * @return the value of exception
+58       */
+59      public UpdateException getException() {
+60          return exception;
+61      }
+62  
+63      /**
+64       * Set the value of exception.
+65       *
+66       * @param exception new value of exception
+67       */
+68      public void setException(UpdateException exception) {
+69          this.exception = exception;
+70      }
+71      private final CveDB cveDB;
+72      private final CallableDownloadTask filePair;
+73      private final DataStoreMetaInfo properties;
+74  
+75      public ProcessTask(final CveDB cveDB, final DataStoreMetaInfo properties, final CallableDownloadTask filePair) {
+76          this.cveDB = cveDB;
+77          this.filePair = filePair;
+78          this.properties = properties;
+79      }
+80  
+81      @Override
+82      public ProcessTask call() throws Exception {
+83          try {
+84              processFiles();
+85          } catch (UpdateException ex) {
+86              this.exception = ex;
+87          }
+88          return this;
+89      }
+90  
+91      /**
+92       * Imports the NVD CVE XML File into the Lucene Index.
+93       *
+94       * @param file the file containing the NVD CVE XML
+95       * @param oldVersion contains the file containing the NVD CVE XML 1.2
+96       * @throws ParserConfigurationException is thrown if there is a parser
+97       * configuration exception
+98       * @throws SAXException is thrown if there is a SAXException
+99       * @throws IOException is thrown if there is a IO Exception
+100      * @throws SQLException is thrown if there is a SQL exception
+101      * @throws DatabaseException is thrown if there is a database exception
+102      * @throws ClassNotFoundException thrown if the h2 database driver cannot be
+103      * loaded
+104      */
+105     protected void importXML(File file, File oldVersion) throws ParserConfigurationException,
+106             SAXException, IOException, SQLException, DatabaseException, ClassNotFoundException {
+107 
+108         final SAXParserFactory factory = SAXParserFactory.newInstance();
+109         final SAXParser saxParser = factory.newSAXParser();
+110 
+111         final NvdCve12Handler cve12Handler = new NvdCve12Handler();
+112         saxParser.parse(oldVersion, cve12Handler);
+113         final Map<String, List<VulnerableSoftware>> prevVersionVulnMap = cve12Handler.getVulnerabilities();
+114 
+115         final NvdCve20Handler cve20Handler = new NvdCve20Handler();
+116         cve20Handler.setCveDB(cveDB);
+117         cve20Handler.setPrevVersionVulnMap(prevVersionVulnMap);
+118         saxParser.parse(file, cve20Handler);
+119     }
+120 
+121     private void processFiles() throws UpdateException {
+122         String msg = String.format("Processing Started for NVD CVE - %s", filePair.getNvdCveInfo().getId());
+123         Logger.getLogger(StandardUpdate.class.getName()).log(Level.INFO, msg);
+124         try {
+125             importXML(filePair.getFirst(), filePair.getSecond());
+126             cveDB.commit();
+127             properties.save(filePair.getNvdCveInfo());
+128         } catch (FileNotFoundException ex) {
+129             throw new UpdateException(ex);
+130         } catch (ParserConfigurationException ex) {
+131             throw new UpdateException(ex);
+132         } catch (SAXException ex) {
+133             throw new UpdateException(ex);
+134         } catch (IOException ex) {
+135             throw new UpdateException(ex);
+136         } catch (SQLException ex) {
+137             throw new UpdateException(ex);
+138         } catch (DatabaseException ex) {
+139             throw new UpdateException(ex);
+140         } catch (ClassNotFoundException ex) {
+141             throw new UpdateException(ex);
+142         } finally {
+143             filePair.cleanup();
+144         }
+145         msg = String.format("Processing Complete for NVD CVE - %s", filePair.getNvdCveInfo().getId());
+146         Logger.getLogger(StandardUpdate.class.getName()).log(Level.INFO, msg);
+147     }
+148 }
+
+
+ + diff --git a/dependency-check-core/xref/org/owasp/dependencycheck/data/update/StandardUpdate.html b/dependency-check-core/xref/org/owasp/dependencycheck/data/update/StandardUpdate.html new file mode 100644 index 000000000..2733bffda --- /dev/null +++ b/dependency-check-core/xref/org/owasp/dependencycheck/data/update/StandardUpdate.html @@ -0,0 +1,571 @@ + + + + +StandardUpdate xref + + + +
View Javadoc
+
+1   /*
+2    * This file is part of dependency-check-core.
+3    *
+4    * Dependency-check-core is free software: you can redistribute it and/or modify it
+5    * under the terms of the GNU General Public License as published by the Free
+6    * Software Foundation, either version 3 of the License, or (at your option) any
+7    * later version.
+8    *
+9    * Dependency-check-core is distributed in the hope that it will be useful, but
+10   * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+11   * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+12   * details.
+13   *
+14   * You should have received a copy of the GNU General Public License along with
+15   * dependency-check-core. If not, see http://www.gnu.org/licenses/.
+16   *
+17   * Copyright (c) 2012 Jeremy Long. All Rights Reserved.
+18   */
+19  package org.owasp.dependencycheck.data.update;
+20  
+21  import org.owasp.dependencycheck.data.nvdcve.InvalidDataException;
+22  import java.io.File;
+23  import java.io.IOException;
+24  import java.net.MalformedURLException;
+25  import java.sql.SQLException;
+26  import java.util.Calendar;
+27  import java.util.Date;
+28  import java.util.HashSet;
+29  import java.util.Iterator;
+30  import java.util.Set;
+31  import java.util.concurrent.ExecutionException;
+32  import java.util.concurrent.ExecutorService;
+33  import java.util.concurrent.Executors;
+34  import java.util.concurrent.Future;
+35  import java.util.logging.Level;
+36  import java.util.logging.Logger;
+37  import org.owasp.dependencycheck.data.UpdateException;
+38  import org.owasp.dependencycheck.data.nvdcve.CveDB;
+39  import org.owasp.dependencycheck.utils.DownloadFailedException;
+40  import org.owasp.dependencycheck.utils.Settings;
+41  import org.owasp.dependencycheck.data.nvdcve.DatabaseException;
+42  import org.owasp.dependencycheck.utils.InvalidSettingException;
+43  import static org.owasp.dependencycheck.data.update.DataStoreMetaInfo.MODIFIED;
+44  import org.owasp.dependencycheck.utils.FileUtils;
+45  
+46  /**
+47   * Class responsible for updating the NVDCVE data store.
+48   *
+49   * @author Jeremy Long (jeremy.long@owasp.org)
+50   */
+51  public class StandardUpdate {
+52  
+53      /**
+54       * The max thread pool size to use when downloading files.
+55       */
+56      public static final int MAX_THREAD_POOL_SIZE = Settings.getInt(Settings.KEYS.MAX_DOWNLOAD_THREAD_POOL_SIZE, 3);
+57      /**
+58       * Information about the timestamps and URLs for data that needs to be
+59       * updated.
+60       */
+61      private DataStoreMetaInfo properties;
+62      /**
+63       * A collection of updateable NVD CVE items.
+64       */
+65      private Updateable updateable;
+66      /**
+67       * A flag indicating whether or not the current data store should be
+68       * deleted.
+69       */
+70      private boolean deleteAndRecreate = false;
+71      /**
+72       * Reference to the Cve Database.
+73       */
+74      private CveDB cveDB = null;
+75  
+76      /**
+77       * Gets whether or not an update is needed.
+78       *
+79       * @return true or false depending on whether an update is needed
+80       */
+81      public boolean isUpdateNeeded() {
+82          return updateable.isUpdateNeeded();
+83      }
+84  
+85      /**
+86       * Set the value of deleteAndRecreate.
+87       *
+88       * @param deleteAndRecreate new value of deleteAndRecreate
+89       */
+90      protected void setDeleteAndRecreate(boolean deleteAndRecreate) {
+91          this.deleteAndRecreate = deleteAndRecreate;
+92      }
+93  
+94      /**
+95       * Get the value of deleteAndRecreate.
+96       *
+97       * @return the value of deleteAndRecreate
+98       */
+99      public boolean shouldDeleteAndRecreate() {
+100         return deleteAndRecreate;
+101     }
+102 
+103     /**
+104      * Constructs a new Standard Update Task.
+105      *
+106      * @throws MalformedURLException thrown if a configured URL is malformed
+107      * @throws DownloadFailedException thrown if a timestamp cannot be checked
+108      * on a configured URL
+109      * @throws UpdateException thrown if there is an exception generating the
+110      * update task
+111      */
+112     public StandardUpdate() throws MalformedURLException, DownloadFailedException, UpdateException {
+113         properties = new DataStoreMetaInfo();
+114         updateable = updatesNeeded();
+115     }
+116 
+117     /**
+118      * <p>Downloads the latest NVD CVE XML file from the web and imports it into
+119      * the current CVE Database.</p>
+120      *
+121      * @throws UpdateException is thrown if there is an error updating the
+122      * database
+123      */
+124     public void update() throws UpdateException {
+125         int maxUpdates = 0;
+126         try {
+127             for (NvdCveInfo cve : updateable) {
+128                 if (cve.getNeedsUpdate()) {
+129                     maxUpdates += 1;
+130                 }
+131             }
+132             if (maxUpdates <= 0) {
+133                 return;
+134             }
+135             if (maxUpdates > 3) {
+136                 Logger.getLogger(StandardUpdate.class.getName()).log(Level.INFO,
+137                         "NVD CVE requires several updates; this could take a couple of minutes.");
+138             }
+139             if (maxUpdates > 0) {
+140                 openDataStores();
+141             }
+142 
+143             final int poolSize = (MAX_THREAD_POOL_SIZE > maxUpdates) ? MAX_THREAD_POOL_SIZE : maxUpdates;
+144             final ExecutorService downloadExecutor = Executors.newFixedThreadPool(poolSize);
+145             final ExecutorService processExecutor = Executors.newSingleThreadExecutor();
+146             final Set<Future<CallableDownloadTask>> downloadFutures = new HashSet<Future<CallableDownloadTask>>(maxUpdates);
+147             final Set<Future<ProcessTask>> processFutures = new HashSet<Future<ProcessTask>>(maxUpdates);
+148             int ctr = 0;
+149             for (NvdCveInfo cve : updateable) {
+150                 if (cve.getNeedsUpdate()) {
+151                     ctr += 1;
+152                     final File file1;
+153                     final File file2;
+154                     try {
+155                         file1 = File.createTempFile("cve" + cve.getId() + "_", ".xml");
+156                         file2 = File.createTempFile("cve_1_2_" + cve.getId() + "_", ".xml");
+157                     } catch (IOException ex) {
+158                         throw new UpdateException(ex);
+159                     }
+160                     final CallableDownloadTask call = new CallableDownloadTask(cve, file1, file2);
+161                     downloadFutures.add(downloadExecutor.submit(call));
+162 
+163                     boolean waitForFuture = ctr % 2 == 0;
+164 
+165                     final Iterator<Future<CallableDownloadTask>> itr = downloadFutures.iterator();
+166                     while (itr.hasNext()) {
+167                         final Future<CallableDownloadTask> future = itr.next();
+168                         if (waitForFuture) { //only allow two NVD/CVE files to be downloaded at a time
+169                             spinWaitForFuture(future);
+170                         }
+171                         if (future.isDone()) { //if we find something complete, add it to the process queue
+172                             try {
+173                                 final CallableDownloadTask filePair = future.get();
+174                                 itr.remove();
+175                                 final ProcessTask task = new ProcessTask(cveDB, properties, filePair);
+176                                 processFutures.add(processExecutor.submit(task));
+177                             } catch (InterruptedException ex) {
+178                                 downloadExecutor.shutdownNow();
+179                                 Logger.getLogger(StandardUpdate.class.getName()).log(Level.FINE, "Thread was interupted", ex);
+180                                 throw new UpdateException(ex);
+181                             } catch (ExecutionException ex) {
+182                                 downloadExecutor.shutdownNow();
+183                                 Logger.getLogger(StandardUpdate.class.getName()).log(Level.SEVERE, null, ex);
+184                                 throw new UpdateException(ex);
+185                             }
+186                         }
+187                     }
+188 
+189                 }
+190             }
+191 
+192             try {
+193                 final Iterator<Future<CallableDownloadTask>> itr = downloadFutures.iterator();
+194                 while (itr.hasNext()) {
+195                     final Future<CallableDownloadTask> future = itr.next();
+196                     final CallableDownloadTask filePair = future.get();
+197                     final ProcessTask task = new ProcessTask(cveDB, properties, filePair);
+198                     processFutures.add(processExecutor.submit(task));
+199                 }
+200             } catch (InterruptedException ex) {
+201                 downloadExecutor.shutdownNow();
+202                 Logger.getLogger(StandardUpdate.class.getName()).log(Level.FINE, "Thread was interupted during download", ex);
+203                 throw new UpdateException(ex);
+204             } catch (ExecutionException ex) {
+205                 downloadExecutor.shutdownNow();
+206                 Logger.getLogger(StandardUpdate.class.getName()).log(Level.FINE, "Execution Exception during download", ex);
+207                 throw new UpdateException(ex);
+208             } finally {
+209                 downloadExecutor.shutdown();
+210             }
+211 
+212             for (Future<ProcessTask> future : processFutures) {
+213                 try {
+214                     final ProcessTask task = future.get();
+215                     if (task.getException() != null) {
+216                         throw task.getException();
+217                     }
+218                 } catch (InterruptedException ex) {
+219                     processExecutor.shutdownNow();
+220                     Logger.getLogger(StandardUpdate.class.getName()).log(Level.FINE, "Thread was interupted during processing", ex);
+221                     throw new UpdateException(ex);
+222                 } catch (ExecutionException ex) {
+223                     processExecutor.shutdownNow();
+224                     Logger.getLogger(StandardUpdate.class.getName()).log(Level.FINE, "Execution Exception during process", ex);
+225                     throw new UpdateException(ex);
+226                 } finally {
+227                     processExecutor.shutdown();
+228                 }
+229             }
+230 
+231             if (maxUpdates >= 1) { //ensure the modified file date gets written
+232                 properties.save(updateable.get(MODIFIED));
+233                 cveDB.cleanupDatabase();
+234             }
+235         } finally {
+236             closeDataStores();
+237         }
+238     }
+239 
+240     //<editor-fold defaultstate="collapsed" desc="OLD version of update() - not multithreaded">
+241     /*
+242      * TODO - remove this
+243      public void update() throws UpdateException {
+244      try {
+245      int maxUpdates = 0;
+246      for (NvdCveInfo cve : getUpdateable()) {
+247      if (cve.getNeedsUpdate()) {
+248      maxUpdates += 1;
+249      }
+250      }
+251      if (maxUpdates > 3) {
+252      Logger.getLogger(StandardUpdate.class.getName()).log(Level.INFO,
+253      "NVD CVE requires several updates; this could take a couple of minutes.");
+254      }
+255      if (maxUpdates > 0) {
+256      openDataStores();
+257      }
+258 
+259      int count = 0;
+260      for (NvdCveInfo cve : getUpdateable()) {
+261      if (cve.getNeedsUpdate()) {
+262      count += 1;
+263      Logger.getLogger(StandardUpdate.class.getName()).log(Level.INFO,
+264      "Updating NVD CVE ({0} of {1})", new Object[]{count, maxUpdates});
+265      URL url = new URL(cve.getUrl());
+266      File outputPath = null;
+267      File outputPath12 = null;
+268      try {
+269      Logger.getLogger(StandardUpdate.class.getName()).log(Level.INFO,
+270      "Downloading {0}", cve.getUrl());
+271      outputPath = File.createTempFile("cve" + cve.getId() + "_", ".xml");
+272      Downloader.fetchFile(url, outputPath);
+273 
+274      url = new URL(cve.getOldSchemaVersionUrl());
+275      outputPath12 = File.createTempFile("cve_1_2_" + cve.getId() + "_", ".xml");
+276      Downloader.fetchFile(url, outputPath12);
+277 
+278      Logger.getLogger(StandardUpdate.class.getName()).log(Level.INFO,
+279      "Processing {0}", cve.getUrl());
+280 
+281      importXML(outputPath, outputPath12);
+282 
+283      getCveDB().commit();
+284      getProperties().save(cve);
+285 
+286      Logger.getLogger(StandardUpdate.class.getName()).log(Level.INFO,
+287      "Completed update {0} of {1}", new Object[]{count, maxUpdates});
+288      } catch (FileNotFoundException ex) {
+289      throw new UpdateException(ex);
+290      } catch (ParserConfigurationException ex) {
+291      throw new UpdateException(ex);
+292      } catch (SAXException ex) {
+293      throw new UpdateException(ex);
+294      } catch (IOException ex) {
+295      throw new UpdateException(ex);
+296      } catch (SQLException ex) {
+297      throw new UpdateException(ex);
+298      } catch (DatabaseException ex) {
+299      throw new UpdateException(ex);
+300      } catch (ClassNotFoundException ex) {
+301      throw new UpdateException(ex);
+302      } finally {
+303      boolean deleted = false;
+304      try {
+305      if (outputPath != null && outputPath.exists()) {
+306      deleted = outputPath.delete();
+307      }
+308      } finally {
+309      if (outputPath != null && (outputPath.exists() || !deleted)) {
+310      outputPath.deleteOnExit();
+311      }
+312      }
+313      try {
+314      deleted = false;
+315      if (outputPath12 != null && outputPath12.exists()) {
+316      deleted = outputPath12.delete();
+317      }
+318      } finally {
+319      if (outputPath12 != null && (outputPath12.exists() || !deleted)) {
+320      outputPath12.deleteOnExit();
+321      }
+322      }
+323      }
+324      }
+325      }
+326      if (maxUpdates >= 1) { //ensure the modified file date gets written
+327      getProperties().save(getUpdateable().get(MODIFIED));
+328      getCveDB().cleanupDatabase();
+329      }
+330      } catch (MalformedURLException ex) {
+331      throw new UpdateException(ex);
+332      } finally {
+333      closeDataStores();
+334      }
+335      }
+336      */
+337     //</editor-fold>
+338     /**
+339      * Determines if the index needs to be updated. This is done by fetching the
+340      * NVD CVE meta data and checking the last update date. If the data needs to
+341      * be refreshed this method will return the NvdCveUrl for the files that
+342      * need to be updated.
+343      *
+344      * @return the collection of files that need to be updated
+345      * @throws MalformedURLException is thrown if the URL for the NVD CVE Meta
+346      * data is incorrect
+347      * @throws DownloadFailedException is thrown if there is an error.
+348      * downloading the NVD CVE download data file
+349      * @throws UpdateException Is thrown if there is an issue with the last
+350      * updated properties file
+351      */
+352     protected final Updateable updatesNeeded() throws MalformedURLException, DownloadFailedException, UpdateException {
+353         Updateable updates = null;
+354         try {
+355             updates = retrieveCurrentTimestampsFromWeb();
+356         } catch (InvalidDataException ex) {
+357             final String msg = "Unable to retrieve valid timestamp from nvd cve downloads page";
+358             Logger
+359                     .getLogger(StandardUpdate.class
+360                     .getName()).log(Level.FINE, msg, ex);
+361             throw new DownloadFailedException(msg, ex);
+362         } catch (InvalidSettingException ex) {
+363             Logger.getLogger(StandardUpdate.class
+364                     .getName()).log(Level.FINE, "Invalid setting found when retrieving timestamps", ex);
+365             throw new DownloadFailedException(
+366                     "Invalid settings", ex);
+367         }
+368 
+369         if (updates == null) {
+370             throw new DownloadFailedException("Unable to retrieve the timestamps of the currently published NVD CVE data");
+371         }
+372         if (!properties.isEmpty()) {
+373             try {
+374                 float version;
+375 
+376                 if (properties.getProperty("version") == null) {
+377                     deleteAndRecreate = true;
+378                 } else {
+379                     try {
+380                         version = Float.parseFloat(properties.getProperty("version"));
+381                         final float currentVersion = Float.parseFloat(CveDB.DB_SCHEMA_VERSION);
+382                         if (currentVersion > version) {
+383                             deleteAndRecreate = true;
+384                         }
+385                     } catch (NumberFormatException ex) {
+386                         deleteAndRecreate = true;
+387                     }
+388                 }
+389 
+390                 if (deleteAndRecreate) {
+391                     return updates;
+392                 }
+393 
+394                 final long lastUpdated = Long.parseLong(properties.getProperty(DataStoreMetaInfo.LAST_UPDATED, "0"));
+395                 final Date now = new Date();
+396                 final int days = Settings.getInt(Settings.KEYS.CVE_MODIFIED_VALID_FOR_DAYS, 7);
+397                 if (lastUpdated == updates.getTimeStamp(MODIFIED)) {
+398                     updates.clear(); //we don't need to update anything.
+399                 } else if (withinRange(lastUpdated, now.getTime(), days)) {
+400                     for (NvdCveInfo entry : updates) {
+401                         if (MODIFIED.equals(entry.getId())) {
+402                             entry.setNeedsUpdate(true);
+403                         } else {
+404                             entry.setNeedsUpdate(false);
+405                         }
+406                     }
+407                 } else { //we figure out which of the several XML files need to be downloaded.
+408                     for (NvdCveInfo entry : updates) {
+409                         if (MODIFIED.equals(entry.getId())) {
+410                             entry.setNeedsUpdate(true);
+411                         } else {
+412                             long currentTimestamp = 0;
+413                             try {
+414                                 currentTimestamp = Long.parseLong(properties.getProperty(DataStoreMetaInfo.LAST_UPDATED_BASE + entry.getId(), "0"));
+415                             } catch (NumberFormatException ex) {
+416                                 final String msg = String.format("Error parsing '%s' '%s' from nvdcve.lastupdated",
+417                                         DataStoreMetaInfo.LAST_UPDATED_BASE, entry.getId());
+418                                 Logger
+419                                         .getLogger(StandardUpdate.class
+420                                         .getName()).log(Level.FINE, msg, ex);
+421                             }
+422                             if (currentTimestamp == entry.getTimestamp()) {
+423                                 entry.setNeedsUpdate(false);
+424                             }
+425                         }
+426                     }
+427                 }
+428             } catch (NumberFormatException ex) {
+429                 final String msg = "An invalid schema version or timestamp exists in the data.properties file.";
+430                 Logger
+431                         .getLogger(StandardUpdate.class
+432                         .getName()).log(Level.WARNING, msg);
+433                 Logger.getLogger(StandardUpdate.class
+434                         .getName()).log(Level.FINE, null, ex);
+435             }
+436         }
+437         return updates;
+438     }
+439 
+440     /**
+441      * Retrieves the timestamps from the NVD CVE meta data file.
+442      *
+443      * @return the timestamp from the currently published nvdcve downloads page
+444      * @throws MalformedURLException thrown if the URL for the NVD CCE Meta data
+445      * is incorrect.
+446      * @throws DownloadFailedException thrown if there is an error downloading
+447      * the nvd cve meta data file
+448      * @throws InvalidDataException thrown if there is an exception parsing the
+449      * timestamps
+450      * @throws InvalidSettingException thrown if the settings are invalid
+451      */
+452     private Updateable retrieveCurrentTimestampsFromWeb()
+453             throws MalformedURLException, DownloadFailedException, InvalidDataException, InvalidSettingException {
+454 
+455         final Updateable updates = new Updateable();
+456         updates.add(MODIFIED, Settings.getString(Settings.KEYS.CVE_MODIFIED_20_URL),
+457                 Settings.getString(Settings.KEYS.CVE_MODIFIED_12_URL),
+458                 false);
+459 
+460         final int start = Settings.getInt(Settings.KEYS.CVE_START_YEAR);
+461         final int end = Calendar.getInstance().get(Calendar.YEAR);
+462         final String baseUrl20 = Settings.getString(Settings.KEYS.CVE_SCHEMA_2_0);
+463         final String baseUrl12 = Settings.getString(Settings.KEYS.CVE_SCHEMA_1_2);
+464         for (int i = start; i <= end; i++) {
+465             updates.add(Integer.toString(i), String.format(baseUrl20, i),
+466                     String.format(baseUrl12, i),
+467                     true);
+468         }
+469 
+470         return updates;
+471     }
+472 
+473     /**
+474      * Deletes the existing data directories.
+475      *
+476      * @throws IOException thrown if the directory cannot be deleted
+477      */
+478     protected void deleteExistingData() throws IOException {
+479         File data = Settings.getDataFile(Settings.KEYS.CVE_DATA_DIRECTORY);
+480         if (data.exists()) {
+481             FileUtils.delete(data);
+482         }
+483         data = DataStoreMetaInfo.getPropertiesFile();
+484         if (data.exists()) {
+485             FileUtils.delete(data);
+486         }
+487     }
+488 
+489     /**
+490      * Closes the CVE and CPE data stores.
+491      */
+492     protected void closeDataStores() {
+493         if (cveDB != null) {
+494             try {
+495                 cveDB.close();
+496             } catch (Exception ignore) {
+497                 Logger.getLogger(StandardUpdate.class.getName()).log(Level.FINEST, "Error closing the cveDB", ignore);
+498             }
+499         }
+500     }
+501 
+502     /**
+503      * Opens the CVE and CPE data stores.
+504      *
+505      * @throws UpdateException thrown if a data store cannot be opened
+506      */
+507     protected void openDataStores() throws UpdateException {
+508         //open the cve and cpe data stores
+509         try {
+510             cveDB = new CveDB();
+511             cveDB.open();
+512         } catch (IOException ex) {
+513             closeDataStores();
+514             Logger.getLogger(StandardUpdate.class.getName()).log(Level.FINE, "IO Error opening databases", ex);
+515             throw new UpdateException("Error updating the CPE/CVE data, please see the log file for more details.");
+516         } catch (SQLException ex) {
+517             closeDataStores();
+518             Logger.getLogger(StandardUpdate.class.getName()).log(Level.FINE, "SQL Exception opening databases", ex);
+519             throw new UpdateException("Error updating the CPE/CVE data, please see the log file for more details.");
+520         } catch (DatabaseException ex) {
+521             closeDataStores();
+522             Logger.getLogger(StandardUpdate.class.getName()).log(Level.FINE, "Database Exception opening databases", ex);
+523             throw new UpdateException("Error updating the CPE/CVE data, please see the log file for more details.");
+524         } catch (ClassNotFoundException ex) {
+525             closeDataStores();
+526             Logger.getLogger(StandardUpdate.class.getName()).log(Level.FINE, "Class not found exception opening databases", ex);
+527             throw new UpdateException("Error updating the CPE/CVE data, please see the log file for more details.");
+528         }
+529     }
+530 
+531     /**
+532      * Determines if the epoch date is within the range specified of the
+533      * compareTo epoch time. This takes the (compareTo-date)/1000/60/60/24 to
+534      * get the number of days. If the calculated days is less then the range the
+535      * date is considered valid.
+536      *
+537      * @param date the date to be checked.
+538      * @param compareTo the date to compare to.
+539      * @param range the range in days to be considered valid.
+540      * @return whether or not the date is within the range.
+541      */
+542     protected boolean withinRange(long date, long compareTo, int range) {
+543         final double differenceInDays = (compareTo - date) / 1000.0 / 60.0 / 60.0 / 24.0;
+544         return differenceInDays < range;
+545     }
+546 
+547     private void spinWaitForFuture(final Future<CallableDownloadTask> future) {
+548         //then wait for downloads to finish
+549         while (!future.isDone()) {
+550             try {
+551                 Thread.sleep(1000);
+552             } catch (InterruptedException ex) {
+553                 Logger.getLogger(StandardUpdate.class.getName()).log(Level.FINE, null, ex);
+554             }
+555         }
+556     }
+557 }
+
+
+ +