##### Coverage Report - org.owasp.dependencycheck.data.update.BatchUpdateTask

Classes in this File | Line Coverage | Branch Coverage | Complexity
BatchUpdateTask |
28% | 30/106
17% | 8/46
| 7.166666666666667;7.167

1 |  |
     /*
2 |  |
      * This file is part of dependency-check-core.
3 |  |
      *
4 |  |
      * Dependency-check-core is free software: you can redistribute it and/or modify it
5 |  |
      * under the terms of the GNU General Public License as published by the Free
6 |  |
      * Software Foundation, either version 3 of the License, or (at your option) any
7 |  |
      * later version.
8 |  |
      *
9 |  |
      * Dependency-check-core is distributed in the hope that it will be useful, but
10 |  |
      * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 |  |
      * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
12 |  |
      * details.
13 |  |
      *
14 |  |
      * You should have received a copy of the GNU General Public License along with
15 |  |
      * dependency-check-core. If not, see http://www.gnu.org/licenses/.
16 |  |
      *
17 |  |
      * Copyright (c) 2012 Jeremy Long. All Rights Reserved.
18 |  |
      */
19 |  |
     package org.owasp.dependencycheck.data.update;
20 |  |
21 |  |
     import org.owasp.dependencycheck.data.nvdcve.InvalidDataException;
22 |  |
     import java.io.File;
23 |  |
     import java.io.IOException;
24 |  |
     import java.net.MalformedURLException;
25 |  |
     import java.net.URISyntaxException;
26 |  |
     import java.net.URL;
27 |  |
     import java.util.Calendar;
28 |  |
     import java.util.Date;
29 |  |
     import java.util.logging.Level;
30 |  |
     import java.util.logging.Logger;
31 |  |
     import org.owasp.dependencycheck.data.UpdateException;
32 |  |
     import org.owasp.dependencycheck.data.nvdcve.CveDB;
33 |  |
     import org.owasp.dependencycheck.utils.DownloadFailedException;
34 |  |
     import org.owasp.dependencycheck.utils.Downloader;
35 |  |
     import org.owasp.dependencycheck.utils.FileUtils;
36 |  |
     import org.owasp.dependencycheck.utils.Settings;
37 |  |
     import org.owasp.dependencycheck.utils.InvalidSettingException;
38 |  |
     import static org.owasp.dependencycheck.data.update.DataStoreMetaInfo.BATCH;
39 |  |
     import static org.owasp.dependencycheck.data.update.DataStoreMetaInfo.MODIFIED;
40 |  |
41 |  |
     /**
42 |  |
      * Class responsible for updating the CPE and NVDCVE data stores.
43 |  |
      *
44 |  |
      * @author Jeremy Long (jeremy.long@owasp.org)
45 |  |
      */
46 |  |
     public class BatchUpdateTask extends AbstractUpdateTask {
47 |  |
48 |  |
         /**
49 |  |
          * Constructs a new BatchUpdateTask.
50 |  |
          *
51 |  |
          * @param properties information about the data store
52 |  |
          * @throws MalformedURLException thrown if a configured URL is malformed
53 |  |
          * @throws DownloadFailedException thrown if a timestamp cannot be checked
54 |  |
          * on a configured URL
55 |  |
          * @throws UpdateException thrown if there is an exception generating the
56 |  |
          * update task
57 |  |
          */
58 |  |
         public BatchUpdateTask(DataStoreMetaInfo properties) throws MalformedURLException, DownloadFailedException, UpdateException {
59 | 2 |
             super(properties);
60 | 2 |
         }
61 |  |
         /**
62 |  |
          * A flag indicating whether or not the batch update should be performed.
63 |  |
          */
64 |  |
         private boolean doBatchUpdate;
65 |  |
66 |  |
         /**
67 |  |
          * Get the value of doBatchUpdate
68 |  |
          *
69 |  |
          * @return the value of doBatchUpdate
70 |  |
          */
71 |  |
         protected boolean isDoBatchUpdate() {
72 | 1 |
             return doBatchUpdate;
73 |  |
         }
74 |  |
75 |  |
         /**
76 |  |
          * Set the value of doBatchUpdate
77 |  |
          *
78 |  |
          * @param doBatchUpdate new value of doBatchUpdate
79 |  |
          */
80 |  |
         protected void setDoBatchUpdate(boolean doBatchUpdate) {
81 | 2 |
             this.doBatchUpdate = doBatchUpdate;
82 | 2 |
         }
83 |  |
84 |  |
         /**
85 |  |
          * <p>Downloads the latest NVD CVE XML file from the web and imports it into
86 |  |
          * the current CVE Database.</p>
87 |  |
          *
88 |  |
          * @throws UpdateException is thrown if there is an error updating the
89 |  |
          * database
90 |  |
          */
91 |  |
         @Override
92 |  |
         public void update() throws UpdateException {
93 | 1 |
             if (getProperties().isBatchUpdateMode() && doBatchUpdate) {
94 | 1 |
                 final String batchSrc = Settings.getString(Settings.KEYS.BATCH_UPDATE_URL);
95 | 1 |
                 File tmp = null;
96 |  |
                 try {
97 | 1 |
                     deleteExistingData();
98 | 1 |
                     final File dataDirectory = CveDB.getDataDirectory().getParentFile();
99 | 1 |
                     final URL batchUrl = new URL(batchSrc);
100 | 1 |
                     if ("file".equals(batchUrl.getProtocol())) {
101 |  |
                         try {
102 | 1 |
                             tmp = new File(batchUrl.toURI());
103 | 0 |
                         } catch (URISyntaxException ex) {
104 | 0 |
                             final String msg = String.format("Invalid batch update URI: %s", batchSrc);
105 | 0 |
                             throw new UpdateException(msg, ex);
106 | 1 |
                         }
107 | 0 |
                     } else if ("http".equals(batchUrl.getProtocol())
108 |  |
                             || "https".equals(batchUrl.getProtocol())) {
109 | 0 |
                         tmp = File.createTempFile("batch_", ".zip");
110 | 0 |
                         Downloader.fetchFile(batchUrl, tmp);
111 |  |
                     }
112 |  |
                     //TODO add FTP?
113 | 1 |
                     FileUtils.extractFiles(tmp, dataDirectory);
114 |  |
115 | 0 |
                 } catch (IOException ex) {
116 | 0 |
                     final String msg = String.format("IO Exception Occured performing batch update using: %s", batchSrc);
117 | 0 |
                     throw new UpdateException(msg, ex);
118 |  |
                 } finally {
119 | 1 |
                     if (tmp != null && !tmp.delete()) {
120 | 0 |
                         tmp.deleteOnExit();
121 |  |
                     }
122 |  |
                 }
123 |  |
             }
124 | 1 |
         }
125 |  |
126 |  |
         /**
127 |  |
          * Determines if the index needs to be updated. This is done by fetching the
128 |  |
          * NVD CVE meta data and checking the last update date. If the data needs to
129 |  |
          * be refreshed this method will return the NvdCveUrl for the files that
130 |  |
          * need to be updated.
131 |  |
          *
132 |  |
          * @return the collection of files that need to be updated
133 |  |
          * @throws MalformedURLException is thrown if the URL for the NVD CVE Meta
134 |  |
          * data is incorrect
135 |  |
          * @throws DownloadFailedException is thrown if there is an error.
136 |  |
          * downloading the NVD CVE download data file
137 |  |
          * @throws UpdateException Is thrown if there is an issue with the last
138 |  |
          * updated properties file
139 |  |
          */
140 |  |
         @Override
141 |  |
         public Updateable updatesNeeded() throws MalformedURLException, DownloadFailedException, UpdateException {
142 | 2 |
             Updateable updates = null;
143 |  |
             try {
144 | 2 |
                 updates = retrieveCurrentTimestampsFromWeb();
145 | 0 |
             } catch (InvalidDataException ex) {
146 | 0 |
                 final String msg = "Unable to retrieve valid timestamp from nvd cve downloads page";
147 | 0 |
                 Logger.getLogger(BatchUpdateTask.class.getName()).log(Level.FINE, msg, ex);
148 | 0 |
                 throw new DownloadFailedException(msg, ex);
149 | 0 |
             } catch (InvalidSettingException ex) {
150 | 0 |
                 Logger.getLogger(BatchUpdateTask.class.getName()).log(Level.FINE, "Invalid setting found when retrieving timestamps", ex);
151 | 0 |
                 throw new DownloadFailedException("Invalid settings", ex);
152 | 2 |
             }
153 |  |
154 | 2 |
             if (updates == null) {
155 | 0 |
                 throw new DownloadFailedException("Unable to retrieve the timestamps of the currently published NVD CVE data");
156 |  |
             }
157 | 2 |
             final DataStoreMetaInfo properties = getProperties();
158 | 2 |
             if (!properties.isEmpty()) {
159 |  |
                 try {
160 | 0 |
                     boolean deleteAndRecreate = false;
161 |  |
                     float version;
162 |  |
163 | 0 |
                     if (properties.getProperty("version") == null) {
164 | 0 |
                         deleteAndRecreate = true;
165 |  |
                     } else {
166 |  |
                         try {
167 | 0 |
                             version = Float.parseFloat(properties.getProperty("version"));
168 | 0 |
                             final float currentVersion = Float.parseFloat(CveDB.DB_SCHEMA_VERSION);
169 | 0 |
                             if (currentVersion > version) {
170 | 0 |
                                 deleteAndRecreate = true;
171 |  |
                             }
172 | 0 |
                         } catch (NumberFormatException ex) {
173 | 0 |
                             deleteAndRecreate = true;
174 | 0 |
                         }
175 |  |
                     }
176 |  |
177 | 0 |
                     final NvdCveInfo batchInfo = updates.get(BATCH);
178 | 0 |
                     if (properties.isBatchUpdateMode() && batchInfo != null) {
179 | 0 |
                         final long lastUpdated = Long.parseLong(properties.getProperty(DataStoreMetaInfo.BATCH, "0"));
180 | 0 |
                         if (lastUpdated != batchInfo.getTimestamp()) {
181 | 0 |
                             deleteAndRecreate = true;
182 |  |
                         }
183 |  |
                     }
184 |  |
185 | 0 |
                     if (deleteAndRecreate) {
186 | 0 |
                         setDoBatchUpdate(properties.isBatchUpdateMode());
187 |  |
                         try {
188 | 0 |
                             deleteExistingData();
189 | 0 |
                         } catch (IOException ex) {
190 | 0 |
                             final String msg = "Unable to delete existing data";
191 | 0 |
                             Logger.getLogger(BatchUpdateTask.class.getName()).log(Level.WARNING, msg);
192 | 0 |
                             Logger.getLogger(BatchUpdateTask.class.getName()).log(Level.FINE, null, ex);
193 | 0 |
                         }
194 | 0 |
                         return updates;
195 |  |
                     }
196 |  |
197 | 0 |
                     final long lastUpdated = Long.parseLong(properties.getProperty(DataStoreMetaInfo.LAST_UPDATED, "0"));
198 | 0 |
                     final Date now = new Date();
199 | 0 |
                     final int days = Settings.getInt(Settings.KEYS.CVE_MODIFIED_VALID_FOR_DAYS, 7);
200 | 0 |
                     final int start = Settings.getInt(Settings.KEYS.CVE_START_YEAR, 2002);
201 | 0 |
                     final int end = Calendar.getInstance().get(Calendar.YEAR);
202 | 0 |
                     if (lastUpdated == updates.get(MODIFIED).getTimestamp()) {
203 | 0 |
                         updates.clear(); //we don't need to update anything.
204 | 0 |
                         setDoBatchUpdate(properties.isBatchUpdateMode());
205 | 0 |
                     } else if (withinRange(lastUpdated, now.getTime(), days)) {
206 | 0 |
                         updates.get(MODIFIED).setNeedsUpdate(true);
207 | 0 |
                         if (properties.isBatchUpdateMode()) {
208 | 0 |
                             setDoBatchUpdate(false);
209 |  |
                         } else {
210 | 0 |
                             for (int i = start; i <= end; i++) {
211 | 0 |
                                 updates.get(String.valueOf(i)).setNeedsUpdate(false);
212 |  |
                             }
213 |  |
                         }
214 | 0 |
                     } else if (properties.isBatchUpdateMode()) {
215 | 0 |
                         updates.get(MODIFIED).setNeedsUpdate(true);
216 | 0 |
                         setDoBatchUpdate(true);
217 |  |
                     } else { //we figure out which of the several XML files need to be downloaded.
218 | 0 |
                         updates.get(MODIFIED).setNeedsUpdate(false);
219 | 0 |
                         for (int i = start; i <= end; i++) {
220 | 0 |
                             final NvdCveInfo cve = updates.get(String.valueOf(i));
221 | 0 |
                             long currentTimestamp = 0;
222 |  |
                             try {
223 | 0 |
                                 currentTimestamp = Long.parseLong(properties.getProperty(DataStoreMetaInfo.LAST_UPDATED_BASE + String.valueOf(i), "0"));
224 | 0 |
                             } catch (NumberFormatException ex) {
225 | 0 |
                                 final String msg = String.format("Error parsing '%s' '%s' from nvdcve.lastupdated",
226 |  |
                                         DataStoreMetaInfo.LAST_UPDATED_BASE, String.valueOf(i));
227 | 0 |
                                 Logger.getLogger(BatchUpdateTask.class.getName()).log(Level.FINE, msg, ex);
228 | 0 |
                             }
229 | 0 |
                             if (currentTimestamp == cve.getTimestamp()) {
230 | 0 |
                                 cve.setNeedsUpdate(false); //they default to true.
231 |  |
                             }
232 |  |
                         }
233 |  |
                     }
234 | 0 |
                 } catch (NumberFormatException ex) {
235 | 0 |
                     final String msg = "An invalid schema version or timestamp exists in the data.properties file.";
236 | 0 |
                     Logger.getLogger(BatchUpdateTask.class.getName()).log(Level.WARNING, msg);
237 | 0 |
                     Logger.getLogger(BatchUpdateTask.class.getName()).log(Level.FINE, null, ex);
238 | 0 |
                     setDoBatchUpdate(properties.isBatchUpdateMode());
239 | 0 |
                 }
240 |  |
             }
241 | 2 |
             return updates;
242 |  |
         }
243 |  |
244 |  |
         /**
245 |  |
          * Retrieves the timestamps from the NVD CVE meta data file.
246 |  |
          *
247 |  |
          * @return the timestamp from the currently published nvdcve downloads page
248 |  |
          * @throws MalformedURLException thrown if the URL for the NVD CCE Meta data
249 |  |
          * is incorrect.
250 |  |
          * @throws DownloadFailedException thrown if there is an error downloading
251 |  |
          * the nvd cve meta data file
252 |  |
          * @throws InvalidDataException thrown if there is an exception parsing the
253 |  |
          * timestamps
254 |  |
          * @throws InvalidSettingException thrown if the settings are invalid
255 |  |
          */
256 |  |
         private Updateable retrieveCurrentTimestampsFromWeb()
257 |  |
                 throws MalformedURLException, DownloadFailedException, InvalidDataException, InvalidSettingException {
258 | 2 |
             final Updateable updates = new Updateable();
259 | 2 |
             updates.add(BATCH, Settings.getString(Settings.KEYS.BATCH_UPDATE_URL),
260 |  |
                     null, false);
261 |  |
262 | 2 |
             final String url = Settings.getString(Settings.KEYS.CVE_MODIFIED_20_URL, "");
263 | 2 |
             if (!url.isEmpty()) {
264 | 2 |
                 updates.add(MODIFIED, url,
265 |  |
                         Settings.getString(Settings.KEYS.CVE_MODIFIED_12_URL),
266 |  |
                         false);
267 |  |
             }
268 | 2 |
             return updates;
269 |  |
         }
270 |  |
     }