View Javadoc

    1   /*
    2    * This file is part of dependency-check-core.
    3    *
    4    * Dependency-check-core is free software: you can redistribute it and/or modify it
    5    * under the terms of the GNU General Public License as published by the Free
    6    * Software Foundation, either version 3 of the License, or (at your option) any
    7    * later version.
    8    *
    9    * Dependency-check-core is distributed in the hope that it will be useful, but
    10   * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
    11   * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
    12   * details.
    13   *
    14   * You should have received a copy of the GNU General Public License along with
    15   * dependency-check-core. If not, see http://www.gnu.org/licenses/.
    16   *
    17   * Copyright (c) 2012 Jeremy Long. All Rights Reserved.
    18   */
    19  package org.owasp.dependencycheck.data.update;
    20
    21  import org.owasp.dependencycheck.data.nvdcve.InvalidDataException;
    22  import java.io.File;
    23  import java.io.IOException;
    24  import java.net.MalformedURLException;
    25  import java.net.URISyntaxException;
    26  import java.net.URL;
    27  import java.util.Calendar;
    28  import java.util.Date;
    29  import java.util.logging.Level;
    30  import java.util.logging.Logger;
    31  import org.owasp.dependencycheck.data.UpdateException;
    32  import org.owasp.dependencycheck.data.nvdcve.CveDB;
    33  import org.owasp.dependencycheck.utils.DownloadFailedException;
    34  import org.owasp.dependencycheck.utils.Downloader;
    35  import org.owasp.dependencycheck.utils.FileUtils;
    36  import org.owasp.dependencycheck.utils.Settings;
    37  import org.owasp.dependencycheck.utils.InvalidSettingException;
    38  import static org.owasp.dependencycheck.data.update.DataStoreMetaInfo.BATCH;
    39  import static org.owasp.dependencycheck.data.update.DataStoreMetaInfo.MODIFIED;
    40
    41  /**
    42   * Class responsible for updating the CPE and NVDCVE data stores.
    43   *
    44   * @author Jeremy Long (jeremy.long@owasp.org)
    45   */
    46  public class BatchUpdateTask extends AbstractUpdateTask {
    47
    48      /**
    49       * Constructs a new BatchUpdateTask.
    50       *
    51       * @param properties information about the data store
    52       * @throws MalformedURLException thrown if a configured URL is malformed
    53       * @throws DownloadFailedException thrown if a timestamp cannot be checked
    54       * on a configured URL
    55       * @throws UpdateException thrown if there is an exception generating the
    56       * update task
    57       */
    58      public BatchUpdateTask(DataStoreMetaInfo properties) throws MalformedURLException, DownloadFailedException, UpdateException {
    59          super(properties);
    60      }
    61      /**
    62       * A flag indicating whether or not the batch update should be performed.
    63       */
    64      private boolean doBatchUpdate;
    65
    66      /**
    67       * Get the value of doBatchUpdate
    68       *
    69       * @return the value of doBatchUpdate
    70       */
    71      protected boolean isDoBatchUpdate() {
    72          return doBatchUpdate;
    73      }
    74
    75      /**
    76       * Set the value of doBatchUpdate
    77       *
    78       * @param doBatchUpdate new value of doBatchUpdate
    79       */
    80      protected void setDoBatchUpdate(boolean doBatchUpdate) {
    81          this.doBatchUpdate = doBatchUpdate;
    82      }
    83
    84      /**
    85       * <p>Downloads the latest NVD CVE XML file from the web and imports it into
    86       * the current CVE Database.</p>
    87       *
    88       * @throws UpdateException is thrown if there is an error updating the
    89       * database
    90       */
    91      @Override
    92      public void update() throws UpdateException {
    93          if (getProperties().isBatchUpdateMode() && doBatchUpdate) {
    94              final String batchSrc = Settings.getString(Settings.KEYS.BATCH_UPDATE_URL);
    95              File tmp = null;
    96              try {
    97                  deleteExistingData();
    98                  final File dataDirectory = CveDB.getDataDirectory().getParentFile();
    99                  final URL batchUrl = new URL(batchSrc);
    100                 if ("file".equals(batchUrl.getProtocol())) {
    101                     try {
    102                         tmp = new File(batchUrl.toURI());
    103                     } catch (URISyntaxException ex) {
    104                         final String msg = String.format("Invalid batch update URI: %s", batchSrc);
    105                         throw new UpdateException(msg, ex);
    106                     }
    107                 } else if ("http".equals(batchUrl.getProtocol())
    108                         || "https".equals(batchUrl.getProtocol())) {
    109                     tmp = File.createTempFile("batch_", ".zip");
    110                     Downloader.fetchFile(batchUrl, tmp);
    111                 }
    112                 //TODO add FTP?
    113                 FileUtils.extractFiles(tmp, dataDirectory);
    114
    115             } catch (IOException ex) {
    116                 final String msg = String.format("IO Exception Occured performing batch update using: %s", batchSrc);
    117                 throw new UpdateException(msg, ex);
    118             } finally {
    119                 if (tmp != null && !tmp.delete()) {
    120                     tmp.deleteOnExit();
    121                 }
    122             }
    123         }
    124     }
    125
    126     /**
    127      * Determines if the index needs to be updated. This is done by fetching the
    128      * NVD CVE meta data and checking the last update date. If the data needs to
    129      * be refreshed this method will return the NvdCveUrl for the files that
    130      * need to be updated.
    131      *
    132      * @return the collection of files that need to be updated
    133      * @throws MalformedURLException is thrown if the URL for the NVD CVE Meta
    134      * data is incorrect
    135      * @throws DownloadFailedException is thrown if there is an error.
    136      * downloading the NVD CVE download data file
    137      * @throws UpdateException Is thrown if there is an issue with the last
    138      * updated properties file
    139      */
    140     @Override
    141     public Updateable updatesNeeded() throws MalformedURLException, DownloadFailedException, UpdateException {
    142         Updateable updates = null;
    143         try {
    144             updates = retrieveCurrentTimestampsFromWeb();
    145         } catch (InvalidDataException ex) {
    146             final String msg = "Unable to retrieve valid timestamp from nvd cve downloads page";
    147             Logger.getLogger(BatchUpdateTask.class.getName()).log(Level.FINE, msg, ex);
    148             throw new DownloadFailedException(msg, ex);
    149         } catch (InvalidSettingException ex) {
    150             Logger.getLogger(BatchUpdateTask.class.getName()).log(Level.FINE, "Invalid setting found when retrieving timestamps", ex);
    151             throw new DownloadFailedException("Invalid settings", ex);
    152         }
    153
    154         if (updates == null) {
    155             throw new DownloadFailedException("Unable to retrieve the timestamps of the currently published NVD CVE data");
    156         }
    157         final DataStoreMetaInfo properties = getProperties();
    158         if (!properties.isEmpty()) {
    159             try {
    160                 boolean deleteAndRecreate = false;
    161                 float version;
    162
    163                 if (properties.getProperty("version") == null) {
    164                     deleteAndRecreate = true;
    165                 } else {
    166                     try {
    167                         version = Float.parseFloat(properties.getProperty("version"));
    168                         final float currentVersion = Float.parseFloat(CveDB.DB_SCHEMA_VERSION);
    169                         if (currentVersion > version) {
    170                             deleteAndRecreate = true;
    171                         }
    172                     } catch (NumberFormatException ex) {
    173                         deleteAndRecreate = true;
    174                     }
    175                 }
    176
    177                 final NvdCveInfo batchInfo = updates.get(BATCH);
    178                 if (properties.isBatchUpdateMode() && batchInfo != null) {
    179                     final long lastUpdated = Long.parseLong(properties.getProperty(DataStoreMetaInfo.BATCH, "0"));
    180                     if (lastUpdated != batchInfo.getTimestamp()) {
    181                         deleteAndRecreate = true;
    182                     }
    183                 }
    184
    185                 if (deleteAndRecreate) {
    186                     setDoBatchUpdate(properties.isBatchUpdateMode());
    187                     try {
    188                         deleteExistingData();
    189                     } catch (IOException ex) {
    190                         final String msg = "Unable to delete existing data";
    191                         Logger.getLogger(BatchUpdateTask.class.getName()).log(Level.WARNING, msg);
    192                         Logger.getLogger(BatchUpdateTask.class.getName()).log(Level.FINE, null, ex);
    193                     }
    194                     return updates;
    195                 }
    196
    197                 final long lastUpdated = Long.parseLong(properties.getProperty(DataStoreMetaInfo.LAST_UPDATED, "0"));
    198                 final Date now = new Date();
    199                 final int days = Settings.getInt(Settings.KEYS.CVE_MODIFIED_VALID_FOR_DAYS, 7);
    200                 final int start = Settings.getInt(Settings.KEYS.CVE_START_YEAR, 2002);
    201                 final int end = Calendar.getInstance().get(Calendar.YEAR);
    202                 if (lastUpdated == updates.get(MODIFIED).getTimestamp()) {
    203                     updates.clear(); //we don't need to update anything.
    204                     setDoBatchUpdate(properties.isBatchUpdateMode());
    205                 } else if (withinRange(lastUpdated, now.getTime(), days)) {
    206                     updates.get(MODIFIED).setNeedsUpdate(true);
    207                     if (properties.isBatchUpdateMode()) {
    208                         setDoBatchUpdate(false);
    209                     } else {
    210                         for (int i = start; i <= end; i++) {
    211                             updates.get(String.valueOf(i)).setNeedsUpdate(false);
    212                         }
    213                     }
    214                 } else if (properties.isBatchUpdateMode()) {
    215                     updates.get(MODIFIED).setNeedsUpdate(true);
    216                     setDoBatchUpdate(true);
    217                 } else { //we figure out which of the several XML files need to be downloaded.
    218                     updates.get(MODIFIED).setNeedsUpdate(false);
    219                     for (int i = start; i <= end; i++) {
    220                         final NvdCveInfo cve = updates.get(String.valueOf(i));
    221                         long currentTimestamp = 0;
    222                         try {
    223                             currentTimestamp = Long.parseLong(properties.getProperty(DataStoreMetaInfo.LAST_UPDATED_BASE + String.valueOf(i), "0"));
    224                         } catch (NumberFormatException ex) {
    225                             final String msg = String.format("Error parsing '%s' '%s' from nvdcve.lastupdated",
    226                                     DataStoreMetaInfo.LAST_UPDATED_BASE, String.valueOf(i));
    227                             Logger.getLogger(BatchUpdateTask.class.getName()).log(Level.FINE, msg, ex);
    228                         }
    229                         if (currentTimestamp == cve.getTimestamp()) {
    230                             cve.setNeedsUpdate(false); //they default to true.
    231                         }
    232                     }
    233                 }
    234             } catch (NumberFormatException ex) {
    235                 final String msg = "An invalid schema version or timestamp exists in the data.properties file.";
    236                 Logger.getLogger(BatchUpdateTask.class.getName()).log(Level.WARNING, msg);
    237                 Logger.getLogger(BatchUpdateTask.class.getName()).log(Level.FINE, null, ex);
    238                 setDoBatchUpdate(properties.isBatchUpdateMode());
    239             }
    240         }
    241         return updates;
    242     }
    243
    244     /**
    245      * Retrieves the timestamps from the NVD CVE meta data file.
    246      *
    247      * @return the timestamp from the currently published nvdcve downloads page
    248      * @throws MalformedURLException thrown if the URL for the NVD CCE Meta data
    249      * is incorrect.
    250      * @throws DownloadFailedException thrown if there is an error downloading
    251      * the nvd cve meta data file
    252      * @throws InvalidDataException thrown if there is an exception parsing the
    253      * timestamps
    254      * @throws InvalidSettingException thrown if the settings are invalid
    255      */
    256     private Updateable retrieveCurrentTimestampsFromWeb()
    257             throws MalformedURLException, DownloadFailedException, InvalidDataException, InvalidSettingException {
    258         final Updateable updates = new Updateable();
    259         updates.add(BATCH, Settings.getString(Settings.KEYS.BATCH_UPDATE_URL),
    260                 null, false);
    261
    262         final String url = Settings.getString(Settings.KEYS.CVE_MODIFIED_20_URL, "");
    263         if (!url.isEmpty()) {
    264             updates.add(MODIFIED, url,
    265                     Settings.getString(Settings.KEYS.CVE_MODIFIED_12_URL),
    266                     false);
    267         }
    268         return updates;
    269     }
    270 }