View Javadoc

    1   /*
    2    * This file is part of dependency-check-core.
    3    *
    4    * Dependency-check-core is free software: you can redistribute it and/or modify it
    5    * under the terms of the GNU General Public License as published by the Free
    6    * Software Foundation, either version 3 of the License, or (at your option) any
    7    * later version.
    8    *
    9    * Dependency-check-core is distributed in the hope that it will be useful, but
    10   * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
    11   * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
    12   * details.
    13   *
    14   * You should have received a copy of the GNU General Public License along with
    15   * dependency-check-core. If not, see http://www.gnu.org/licenses/.
    16   *
    17   * Copyright (c) 2012 Jeremy Long. All Rights Reserved.
    18   */
    19  package org.owasp.dependencycheck.data.update;
    20
    21  import org.owasp.dependencycheck.data.nvdcve.InvalidDataException;
    22  import java.io.File;
    23  import java.io.FileNotFoundException;
    24  import java.io.IOException;
    25  import javax.xml.parsers.ParserConfigurationException;
    26  import org.xml.sax.SAXException;
    27  import java.net.MalformedURLException;
    28  import java.net.URL;
    29  import java.sql.SQLException;
    30  import java.util.Calendar;
    31  import java.util.Date;
    32  import java.util.logging.Level;
    33  import java.util.logging.Logger;
    34  import org.owasp.dependencycheck.data.UpdateException;
    35  import org.owasp.dependencycheck.data.nvdcve.CveDB;
    36  import org.owasp.dependencycheck.utils.DownloadFailedException;
    37  import org.owasp.dependencycheck.utils.Downloader;
    38  import org.owasp.dependencycheck.utils.Settings;
    39  import org.owasp.dependencycheck.data.nvdcve.DatabaseException;
    40  import org.owasp.dependencycheck.utils.InvalidSettingException;
    41  import static org.owasp.dependencycheck.data.update.DataStoreMetaInfo.MODIFIED;
    42
    43  /**
    44   * Class responsible for updating the CPE and NVDCVE data stores.
    45   *
    46   * @author Jeremy Long (jeremy.long@owasp.org)
    47   */
    48  public class StandardUpdateTask extends AbstractUpdateTask {
    49
    50      /**
    51       * Constructs a new Standard Update Task.
    52       *
    53       * @param properties information about the data store
    54       * @throws MalformedURLException thrown if a configured URL is malformed
    55       * @throws DownloadFailedException thrown if a timestamp cannot be checked
    56       * on a configured URL
    57       * @throws UpdateException thrown if there is an exception generating the
    58       * update task
    59       */
    60      public StandardUpdateTask(DataStoreMetaInfo properties) throws MalformedURLException, DownloadFailedException, UpdateException {
    61          super(properties);
    62      }
    63
    64      /**
    65       * <p>Downloads the latest NVD CVE XML file from the web and imports it into
    66       * the current CVE Database.</p>
    67       *
    68       * @throws UpdateException is thrown if there is an error updating the
    69       * database
    70       */
    71      @Override
    72      public void update() throws UpdateException {
    73          try {
    74              int maxUpdates = 0;
    75              for (NvdCveInfo cve : getUpdateable()) {
    76                  if (cve.getNeedsUpdate()) {
    77                      maxUpdates += 1;
    78                  }
    79              }
    80              if (maxUpdates > 3) {
    81                  Logger.getLogger(StandardUpdateTask.class.getName()).log(Level.INFO,
    82                          "NVD CVE requires several updates; this could take a couple of minutes.");
    83              }
    84              if (maxUpdates > 0) {
    85                  openDataStores();
    86              }
    87
    88              int count = 0;
    89              for (NvdCveInfo cve : getUpdateable()) {
    90                  if (cve.getNeedsUpdate()) {
    91                      count += 1;
    92                      Logger.getLogger(StandardUpdateTask.class.getName()).log(Level.INFO,
    93                              "Updating NVD CVE ({0} of {1})", new Object[]{count, maxUpdates});
    94                      URL url = new URL(cve.getUrl());
    95                      File outputPath = null;
    96                      File outputPath12 = null;
    97                      try {
    98                          Logger.getLogger(StandardUpdateTask.class.getName()).log(Level.INFO,
    99                                  "Downloading {0}", cve.getUrl());
    100                         outputPath = File.createTempFile("cve" + cve.getId() + "_", ".xml");
    101                         Downloader.fetchFile(url, outputPath);
    102
    103                         url = new URL(cve.getOldSchemaVersionUrl());
    104                         outputPath12 = File.createTempFile("cve_1_2_" + cve.getId() + "_", ".xml");
    105                         Downloader.fetchFile(url, outputPath12);
    106
    107                         Logger.getLogger(StandardUpdateTask.class.getName()).log(Level.INFO,
    108                                 "Processing {0}", cve.getUrl());
    109
    110                         importXML(outputPath, outputPath12);
    111
    112                         getCveDB().commit();
    113                         getCpeIndex().commit();
    114                         getProperties().save(cve);
    115
    116                         Logger.getLogger(StandardUpdateTask.class.getName()).log(Level.INFO,
    117                                 "Completed update {0} of {1}", new Object[]{count, maxUpdates});
    118                     } catch (FileNotFoundException ex) {
    119                         throw new UpdateException(ex);
    120                     } catch (ParserConfigurationException ex) {
    121                         throw new UpdateException(ex);
    122                     } catch (SAXException ex) {
    123                         throw new UpdateException(ex);
    124                     } catch (IOException ex) {
    125                         throw new UpdateException(ex);
    126                     } catch (SQLException ex) {
    127                         throw new UpdateException(ex);
    128                     } catch (DatabaseException ex) {
    129                         throw new UpdateException(ex);
    130                     } catch (ClassNotFoundException ex) {
    131                         throw new UpdateException(ex);
    132                     } finally {
    133                         boolean deleted = false;
    134                         try {
    135                             if (outputPath != null && outputPath.exists()) {
    136                                 deleted = outputPath.delete();
    137                             }
    138                         } finally {
    139                             if (outputPath != null && (outputPath.exists() || !deleted)) {
    140                                 outputPath.deleteOnExit();
    141                             }
    142                         }
    143                         try {
    144                             deleted = false;
    145                             if (outputPath12 != null && outputPath12.exists()) {
    146                                 deleted = outputPath12.delete();
    147                             }
    148                         } finally {
    149                             if (outputPath12 != null && (outputPath12.exists() || !deleted)) {
    150                                 outputPath12.deleteOnExit();
    151                             }
    152                         }
    153                     }
    154                 }
    155             }
    156             if (maxUpdates >= 1) { //ensure the modified file date gets written
    157                 getProperties().save(getUpdateable().get(MODIFIED));
    158                 getCveDB().cleanupDatabase();
    159             }
    160         } catch (MalformedURLException ex) {
    161             throw new UpdateException(ex);
    162         } finally {
    163             closeDataStores();
    164         }
    165     }
    166
    167     /**
    168      * Determines if the index needs to be updated. This is done by fetching the
    169      * NVD CVE meta data and checking the last update date. If the data needs to
    170      * be refreshed this method will return the NvdCveUrl for the files that
    171      * need to be updated.
    172      *
    173      * @return the collection of files that need to be updated
    174      * @throws MalformedURLException is thrown if the URL for the NVD CVE Meta
    175      * data is incorrect
    176      * @throws DownloadFailedException is thrown if there is an error.
    177      * downloading the NVD CVE download data file
    178      * @throws UpdateException Is thrown if there is an issue with the last
    179      * updated properties file
    180      */
    181     @Override
    182     protected Updateable updatesNeeded() throws MalformedURLException, DownloadFailedException, UpdateException {
    183         Updateable updates = null;
    184         try {
    185             updates = retrieveCurrentTimestampsFromWeb();
    186         } catch (InvalidDataException ex) {
    187             final String msg = "Unable to retrieve valid timestamp from nvd cve downloads page";
    188             Logger.getLogger(StandardUpdateTask.class.getName()).log(Level.FINE, msg, ex);
    189             throw new DownloadFailedException(msg, ex);
    190         } catch (InvalidSettingException ex) {
    191             Logger.getLogger(StandardUpdateTask.class.getName()).log(Level.FINE, "Invalid setting found when retrieving timestamps", ex);
    192             throw new DownloadFailedException("Invalid settings", ex);
    193         }
    194
    195         if (updates == null) {
    196             throw new DownloadFailedException("Unable to retrieve the timestamps of the currently published NVD CVE data");
    197         }
    198         final DataStoreMetaInfo properties = getProperties();
    199         if (!properties.isEmpty()) {
    200             try {
    201                 float version;
    202
    203                 if (properties.getProperty("version") == null) {
    204                     setDeleteAndRecreate(true);
    205                 } else {
    206                     try {
    207                         version = Float.parseFloat(properties.getProperty("version"));
    208                         final float currentVersion = Float.parseFloat(CveDB.DB_SCHEMA_VERSION);
    209                         if (currentVersion > version) {
    210                             setDeleteAndRecreate(true);
    211                         }
    212                     } catch (NumberFormatException ex) {
    213                         setDeleteAndRecreate(true);
    214                     }
    215                 }
    216
    217                 if (shouldDeleteAndRecreate()) {
    218                     return updates;
    219                 }
    220
    221                 final long lastUpdated = Long.parseLong(properties.getProperty(DataStoreMetaInfo.LAST_UPDATED, "0"));
    222                 final Date now = new Date();
    223                 final int days = Settings.getInt(Settings.KEYS.CVE_MODIFIED_VALID_FOR_DAYS, 7);
    224                 if (lastUpdated == updates.getTimeStamp(MODIFIED)) {
    225                     updates.clear(); //we don't need to update anything.
    226                 } else if (withinRange(lastUpdated, now.getTime(), days)) {
    227                     for (NvdCveInfo entry : updates) {
    228                         if (MODIFIED.equals(entry.getId())) {
    229                             entry.setNeedsUpdate(true);
    230                         } else {
    231                             entry.setNeedsUpdate(false);
    232                         }
    233                     }
    234                 } else { //we figure out which of the several XML files need to be downloaded.
    235                     for (NvdCveInfo entry : updates) {
    236                         if (MODIFIED.equals(entry.getId())) {
    237                             entry.setNeedsUpdate(true);
    238                         } else {
    239                             long currentTimestamp = 0;
    240                             try {
    241                                 currentTimestamp = Long.parseLong(properties.getProperty(DataStoreMetaInfo.LAST_UPDATED_BASE + entry.getId(), "0"));
    242                             } catch (NumberFormatException ex) {
    243                                 final String msg = String.format("Error parsing '%s' '%s' from nvdcve.lastupdated",
    244                                         DataStoreMetaInfo.LAST_UPDATED_BASE, entry.getId());
    245                                 Logger.getLogger(StandardUpdateTask.class.getName()).log(Level.FINE, msg, ex);
    246                             }
    247                             if (currentTimestamp == entry.getTimestamp()) {
    248                                 entry.setNeedsUpdate(false);
    249                             }
    250                         }
    251                     }
    252                 }
    253             } catch (NumberFormatException ex) {
    254                 final String msg = "An invalid schema version or timestamp exists in the data.properties file.";
    255                 Logger.getLogger(StandardUpdateTask.class.getName()).log(Level.WARNING, msg);
    256                 Logger.getLogger(StandardUpdateTask.class.getName()).log(Level.FINE, null, ex);
    257             }
    258         }
    259         return updates;
    260     }
    261
    262     /**
    263      * Retrieves the timestamps from the NVD CVE meta data file.
    264      *
    265      * @return the timestamp from the currently published nvdcve downloads page
    266      * @throws MalformedURLException thrown if the URL for the NVD CCE Meta data
    267      * is incorrect.
    268      * @throws DownloadFailedException thrown if there is an error downloading
    269      * the nvd cve meta data file
    270      * @throws InvalidDataException thrown if there is an exception parsing the
    271      * timestamps
    272      * @throws InvalidSettingException thrown if the settings are invalid
    273      */
    274     private Updateable retrieveCurrentTimestampsFromWeb()
    275             throws MalformedURLException, DownloadFailedException, InvalidDataException, InvalidSettingException {
    276
    277         final Updateable updates = new Updateable();
    278         updates.add(MODIFIED, Settings.getString(Settings.KEYS.CVE_MODIFIED_20_URL),
    279                 Settings.getString(Settings.KEYS.CVE_MODIFIED_12_URL),
    280                 false);
    281
    282         final int start = Settings.getInt(Settings.KEYS.CVE_START_YEAR);
    283         final int end = Calendar.getInstance().get(Calendar.YEAR);
    284         final String baseUrl20 = Settings.getString(Settings.KEYS.CVE_SCHEMA_2_0);
    285         final String baseUrl12 = Settings.getString(Settings.KEYS.CVE_SCHEMA_1_2);
    286         for (int i = start; i <= end; i++) {
    287             updates.add(Integer.toString(i), String.format(baseUrl20, i),
    288                     String.format(baseUrl12, i),
    289                     true);
    290         }
    291
    292         return updates;
    293     }
    294 }