Coverage Report - org.owasp.dependencycheck.data.update.StandardUpdate
 
Classes in this File Line Coverage Branch Coverage Complexity
StandardUpdate
0%
0/141
0%
0/50
7.714
 
 1  
 /*
 2  
  * This file is part of dependency-check-core.
 3  
  *
 4  
  * Licensed under the Apache License, Version 2.0 (the "License");
 5  
  * you may not use this file except in compliance with the License.
 6  
  * You may obtain a copy of the License at
 7  
  *
 8  
  *     http://www.apache.org/licenses/LICENSE-2.0
 9  
  *
 10  
  * Unless required by applicable law or agreed to in writing, software
 11  
  * distributed under the License is distributed on an "AS IS" BASIS,
 12  
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 13  
  * See the License for the specific language governing permissions and
 14  
  * limitations under the License.
 15  
  *
 16  
  * Copyright (c) 2012 Jeremy Long. All Rights Reserved.
 17  
  */
 18  
 package org.owasp.dependencycheck.data.update;
 19  
 
 20  
 import java.net.MalformedURLException;
 21  
 import java.util.Calendar;
 22  
 import java.util.Date;
 23  
 import java.util.HashSet;
 24  
 import java.util.Set;
 25  
 import java.util.concurrent.ExecutionException;
 26  
 import java.util.concurrent.ExecutorService;
 27  
 import java.util.concurrent.Executors;
 28  
 import java.util.concurrent.Future;
 29  
 import java.util.logging.Level;
 30  
 import java.util.logging.Logger;
 31  
 import org.owasp.dependencycheck.data.nvdcve.CveDB;
 32  
 import org.owasp.dependencycheck.data.nvdcve.DatabaseException;
 33  
 import org.owasp.dependencycheck.data.nvdcve.DatabaseProperties;
 34  
 import static org.owasp.dependencycheck.data.nvdcve.DatabaseProperties.MODIFIED;
 35  
 import org.owasp.dependencycheck.data.update.exception.InvalidDataException;
 36  
 import org.owasp.dependencycheck.data.update.exception.UpdateException;
 37  
 import org.owasp.dependencycheck.data.update.task.DownloadTask;
 38  
 import org.owasp.dependencycheck.data.update.task.ProcessTask;
 39  
 import org.owasp.dependencycheck.utils.DateUtil;
 40  
 import org.owasp.dependencycheck.utils.DownloadFailedException;
 41  
 import org.owasp.dependencycheck.utils.InvalidSettingException;
 42  
 import org.owasp.dependencycheck.utils.Settings;
 43  
 
 44  
 /**
 45  
  * Class responsible for updating the NVDCVE data store.
 46  
  *
 47  
  * @author Jeremy Long
 48  
  */
 49  
 public class StandardUpdate {
 50  
 
 51  
     /**
 52  
      * Static logger.
 53  
      */
 54  0
     private static final Logger LOGGER = Logger.getLogger(StandardUpdate.class.getName());
 55  
     /**
 56  
      * The max thread pool size to use when downloading files.
 57  
      */
 58  0
     public static final int MAX_THREAD_POOL_SIZE = Settings.getInt(Settings.KEYS.MAX_DOWNLOAD_THREAD_POOL_SIZE, 3);
 59  
     /**
 60  
      * Information about the timestamps and URLs for data that needs to be updated.
 61  
      */
 62  
     private DatabaseProperties properties;
 63  
     /**
 64  
      * A collection of updateable NVD CVE items.
 65  
      */
 66  
     private UpdateableNvdCve updateable;
 67  
     /**
 68  
      * Reference to the Cve Database.
 69  
      */
 70  0
     private CveDB cveDB = null;
 71  
 
 72  
     /**
 73  
      * Gets whether or not an update is needed.
 74  
      *
 75  
      * @return true or false depending on whether an update is needed
 76  
      */
 77  
     public boolean isUpdateNeeded() {
 78  0
         return updateable.isUpdateNeeded();
 79  
     }
 80  
 
 81  
     /**
 82  
      * Constructs a new Standard Update Task.
 83  
      *
 84  
      * @throws MalformedURLException thrown if a configured URL is malformed
 85  
      * @throws DownloadFailedException thrown if a timestamp cannot be checked on a configured URL
 86  
      * @throws UpdateException thrown if there is an exception generating the update task
 87  
      */
 88  0
     public StandardUpdate() throws MalformedURLException, DownloadFailedException, UpdateException {
 89  0
         openDataStores();
 90  0
         properties = cveDB.getDatabaseProperties();
 91  0
         updateable = updatesNeeded();
 92  0
     }
 93  
 
 94  
     /**
 95  
      * <p>
 96  
      * Downloads the latest NVD CVE XML file from the web and imports it into the current CVE Database.</p>
 97  
      *
 98  
      * @throws UpdateException is thrown if there is an error updating the database
 99  
      */
 100  
     public void update() throws UpdateException {
 101  0
         int maxUpdates = 0;
 102  
         try {
 103  0
             for (NvdCveInfo cve : updateable) {
 104  0
                 if (cve.getNeedsUpdate()) {
 105  0
                     maxUpdates += 1;
 106  
                 }
 107  0
             }
 108  0
             if (maxUpdates <= 0) {
 109  
                 return;
 110  
             }
 111  0
             if (maxUpdates > 3) {
 112  0
                 LOGGER.log(Level.INFO,
 113  
                         "NVD CVE requires several updates; this could take a couple of minutes.");
 114  
             }
 115  0
             if (maxUpdates > 0) {
 116  0
                 openDataStores();
 117  
             }
 118  
 
 119  0
             final int poolSize = (MAX_THREAD_POOL_SIZE < maxUpdates) ? MAX_THREAD_POOL_SIZE : maxUpdates;
 120  
 
 121  0
             final ExecutorService downloadExecutors = Executors.newFixedThreadPool(poolSize);
 122  0
             final ExecutorService processExecutor = Executors.newSingleThreadExecutor();
 123  0
             final Set<Future<Future<ProcessTask>>> downloadFutures = new HashSet<Future<Future<ProcessTask>>>(maxUpdates);
 124  0
             for (NvdCveInfo cve : updateable) {
 125  0
                 if (cve.getNeedsUpdate()) {
 126  0
                     final DownloadTask call = new DownloadTask(cve, processExecutor, cveDB, Settings.getInstance());
 127  0
                     downloadFutures.add(downloadExecutors.submit(call));
 128  
                 }
 129  0
             }
 130  0
             downloadExecutors.shutdown();
 131  
 
 132  
             //next, move the future future processTasks to just future processTasks
 133  0
             final Set<Future<ProcessTask>> processFutures = new HashSet<Future<ProcessTask>>(maxUpdates);
 134  0
             for (Future<Future<ProcessTask>> future : downloadFutures) {
 135  0
                 Future<ProcessTask> task = null;
 136  
                 try {
 137  0
                     task = future.get();
 138  0
                 } catch (InterruptedException ex) {
 139  0
                     downloadExecutors.shutdownNow();
 140  0
                     processExecutor.shutdownNow();
 141  
 
 142  0
                     LOGGER.log(Level.FINE, "Thread was interrupted during download", ex);
 143  0
                     throw new UpdateException("The download was interrupted", ex);
 144  0
                 } catch (ExecutionException ex) {
 145  0
                     downloadExecutors.shutdownNow();
 146  0
                     processExecutor.shutdownNow();
 147  
 
 148  0
                     LOGGER.log(Level.FINE, "Thread was interrupted during download execution", ex);
 149  0
                     throw new UpdateException("The execution of the download was interrupted", ex);
 150  0
                 }
 151  0
                 if (task == null) {
 152  0
                     downloadExecutors.shutdownNow();
 153  0
                     processExecutor.shutdownNow();
 154  0
                     LOGGER.log(Level.FINE, "Thread was interrupted during download");
 155  0
                     throw new UpdateException("The download was interrupted; unable to complete the update");
 156  
                 } else {
 157  0
                     processFutures.add(task);
 158  
                 }
 159  0
             }
 160  
 
 161  0
             for (Future<ProcessTask> future : processFutures) {
 162  
                 try {
 163  0
                     final ProcessTask task = future.get();
 164  0
                     if (task.getException() != null) {
 165  0
                         throw task.getException();
 166  
                     }
 167  0
                 } catch (InterruptedException ex) {
 168  0
                     processExecutor.shutdownNow();
 169  0
                     LOGGER.log(Level.FINE, "Thread was interrupted during processing", ex);
 170  0
                     throw new UpdateException(ex);
 171  0
                 } catch (ExecutionException ex) {
 172  0
                     processExecutor.shutdownNow();
 173  0
                     LOGGER.log(Level.FINE, "Execution Exception during process", ex);
 174  0
                     throw new UpdateException(ex);
 175  
                 } finally {
 176  0
                     processExecutor.shutdown();
 177  0
                 }
 178  0
             }
 179  
 
 180  0
             if (maxUpdates >= 1) { //ensure the modified file date gets written (we may not have actually updated it)
 181  0
                 properties.save(updateable.get(MODIFIED));
 182  0
                 LOGGER.log(Level.INFO, "Begin database maintenance.");
 183  0
                 cveDB.cleanupDatabase();
 184  0
                 LOGGER.log(Level.INFO, "End database maintenance.");
 185  
             }
 186  
         } finally {
 187  0
             closeDataStores();
 188  0
         }
 189  0
     }
 190  
 
 191  
     /**
 192  
      * Determines if the index needs to be updated. This is done by fetching the NVD CVE meta data and checking the last
 193  
      * update date. If the data needs to be refreshed this method will return the NvdCveUrl for the files that need to
 194  
      * be updated.
 195  
      *
 196  
      * @return the collection of files that need to be updated
 197  
      * @throws MalformedURLException is thrown if the URL for the NVD CVE Meta data is incorrect
 198  
      * @throws DownloadFailedException is thrown if there is an error. downloading the NVD CVE download data file
 199  
      * @throws UpdateException Is thrown if there is an issue with the last updated properties file
 200  
      */
 201  
     protected final UpdateableNvdCve updatesNeeded() throws MalformedURLException, DownloadFailedException, UpdateException {
 202  0
         UpdateableNvdCve updates = null;
 203  
         try {
 204  0
             updates = retrieveCurrentTimestampsFromWeb();
 205  0
         } catch (InvalidDataException ex) {
 206  0
             final String msg = "Unable to retrieve valid timestamp from nvd cve downloads page";
 207  0
             LOGGER.log(Level.FINE, msg, ex);
 208  0
             throw new DownloadFailedException(msg, ex);
 209  0
         } catch (InvalidSettingException ex) {
 210  0
             LOGGER.log(Level.FINE, "Invalid setting found when retrieving timestamps", ex);
 211  0
             throw new DownloadFailedException("Invalid settings", ex);
 212  0
         }
 213  
 
 214  0
         if (updates == null) {
 215  0
             throw new DownloadFailedException("Unable to retrieve the timestamps of the currently published NVD CVE data");
 216  
         }
 217  0
         if (!properties.isEmpty()) {
 218  
             try {
 219  0
                 final long lastUpdated = Long.parseLong(properties.getProperty(DatabaseProperties.LAST_UPDATED, "0"));
 220  0
                 final Date now = new Date();
 221  0
                 final int days = Settings.getInt(Settings.KEYS.CVE_MODIFIED_VALID_FOR_DAYS, 7);
 222  0
                 if (lastUpdated == updates.getTimeStamp(MODIFIED)) {
 223  0
                     updates.clear(); //we don't need to update anything.
 224  0
                 } else if (DateUtil.withinDateRange(lastUpdated, now.getTime(), days)) {
 225  0
                     for (NvdCveInfo entry : updates) {
 226  0
                         if (MODIFIED.equals(entry.getId())) {
 227  0
                             entry.setNeedsUpdate(true);
 228  
                         } else {
 229  0
                             entry.setNeedsUpdate(false);
 230  
                         }
 231  0
                     }
 232  
                 } else { //we figure out which of the several XML files need to be downloaded.
 233  0
                     for (NvdCveInfo entry : updates) {
 234  0
                         if (MODIFIED.equals(entry.getId())) {
 235  0
                             entry.setNeedsUpdate(true);
 236  
                         } else {
 237  0
                             long currentTimestamp = 0;
 238  
                             try {
 239  0
                                 currentTimestamp = Long.parseLong(properties.getProperty(DatabaseProperties.LAST_UPDATED_BASE + entry.getId(), "0"));
 240  0
                             } catch (NumberFormatException ex) {
 241  0
                                 final String msg = String.format("Error parsing '%s' '%s' from nvdcve.lastupdated",
 242  
                                         DatabaseProperties.LAST_UPDATED_BASE, entry.getId());
 243  0
                                 LOGGER.log(Level.FINE, msg, ex);
 244  0
                             }
 245  0
                             if (currentTimestamp == entry.getTimestamp()) {
 246  0
                                 entry.setNeedsUpdate(false);
 247  
                             }
 248  
                         }
 249  0
                     }
 250  
                 }
 251  0
             } catch (NumberFormatException ex) {
 252  0
                 final String msg = "An invalid schema version or timestamp exists in the data.properties file.";
 253  0
                 LOGGER.log(Level.WARNING, msg);
 254  0
                 LOGGER.log(Level.FINE, "", ex);
 255  0
             }
 256  
         }
 257  0
         return updates;
 258  
     }
 259  
 
 260  
     /**
 261  
      * Retrieves the timestamps from the NVD CVE meta data file.
 262  
      *
 263  
      * @return the timestamp from the currently published nvdcve downloads page
 264  
      * @throws MalformedURLException thrown if the URL for the NVD CCE Meta data is incorrect.
 265  
      * @throws DownloadFailedException thrown if there is an error downloading the nvd cve meta data file
 266  
      * @throws InvalidDataException thrown if there is an exception parsing the timestamps
 267  
      * @throws InvalidSettingException thrown if the settings are invalid
 268  
      */
 269  
     private UpdateableNvdCve retrieveCurrentTimestampsFromWeb()
 270  
             throws MalformedURLException, DownloadFailedException, InvalidDataException, InvalidSettingException {
 271  
 
 272  0
         final UpdateableNvdCve updates = new UpdateableNvdCve();
 273  0
         updates.add(MODIFIED, Settings.getString(Settings.KEYS.CVE_MODIFIED_20_URL),
 274  
                 Settings.getString(Settings.KEYS.CVE_MODIFIED_12_URL),
 275  
                 false);
 276  
 
 277  0
         final int start = Settings.getInt(Settings.KEYS.CVE_START_YEAR);
 278  0
         final int end = Calendar.getInstance().get(Calendar.YEAR);
 279  0
         final String baseUrl20 = Settings.getString(Settings.KEYS.CVE_SCHEMA_2_0);
 280  0
         final String baseUrl12 = Settings.getString(Settings.KEYS.CVE_SCHEMA_1_2);
 281  0
         for (int i = start; i <= end; i++) {
 282  0
             updates.add(Integer.toString(i), String.format(baseUrl20, i),
 283  
                     String.format(baseUrl12, i),
 284  
                     true);
 285  
         }
 286  
 
 287  0
         return updates;
 288  
     }
 289  
 
 290  
     /**
 291  
      * Closes the CVE and CPE data stores.
 292  
      */
 293  
     protected void closeDataStores() {
 294  0
         if (cveDB != null) {
 295  
             try {
 296  0
                 cveDB.close();
 297  0
             } catch (Throwable ignore) {
 298  0
                 LOGGER.log(Level.FINEST, "Error closing the cveDB", ignore);
 299  0
             }
 300  
         }
 301  0
     }
 302  
 
 303  
     /**
 304  
      * Opens the CVE and CPE data stores.
 305  
      *
 306  
      * @throws UpdateException thrown if a data store cannot be opened
 307  
      */
 308  
     protected final void openDataStores() throws UpdateException {
 309  0
         if (cveDB != null) {
 310  0
             return;
 311  
         }
 312  
         try {
 313  0
             cveDB = new CveDB();
 314  0
             cveDB.open();
 315  0
         } catch (DatabaseException ex) {
 316  0
             closeDataStores();
 317  0
             LOGGER.log(Level.FINE, "Database Exception opening databases", ex);
 318  0
             throw new UpdateException("Error updating the CPE/CVE data, please see the log file for more details.");
 319  0
         }
 320  0
     }
 321  
 }