##### Coverage Report - org.owasp.dependencycheck.data.update.StandardUpdateTask

Classes in this File | Line Coverage | Branch Coverage | Complexity
StandardUpdateTask |
0% | 0/126
0% | 0/60
| 14.75;14.75

1 |  |
     /*
2 |  |
      * This file is part of dependency-check-core.
3 |  |
      *
4 |  |
      * Dependency-check-core is free software: you can redistribute it and/or modify it
5 |  |
      * under the terms of the GNU General Public License as published by the Free
6 |  |
      * Software Foundation, either version 3 of the License, or (at your option) any
7 |  |
      * later version.
8 |  |
      *
9 |  |
      * Dependency-check-core is distributed in the hope that it will be useful, but
10 |  |
      * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 |  |
      * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
12 |  |
      * details.
13 |  |
      *
14 |  |
      * You should have received a copy of the GNU General Public License along with
15 |  |
      * dependency-check-core. If not, see http://www.gnu.org/licenses/.
16 |  |
      *
17 |  |
      * Copyright (c) 2012 Jeremy Long. All Rights Reserved.
18 |  |
      */
19 |  |
     package org.owasp.dependencycheck.data.update;
20 |  |
21 |  |
     import org.owasp.dependencycheck.data.nvdcve.InvalidDataException;
22 |  |
     import java.io.File;
23 |  |
     import java.io.FileNotFoundException;
24 |  |
     import java.io.IOException;
25 |  |
     import javax.xml.parsers.ParserConfigurationException;
26 |  |
     import org.xml.sax.SAXException;
27 |  |
     import java.net.MalformedURLException;
28 |  |
     import java.net.URL;
29 |  |
     import java.sql.SQLException;
30 |  |
     import java.util.Calendar;
31 |  |
     import java.util.Date;
32 |  |
     import java.util.logging.Level;
33 |  |
     import java.util.logging.Logger;
34 |  |
     import org.owasp.dependencycheck.data.UpdateException;
35 |  |
     import org.owasp.dependencycheck.data.nvdcve.CveDB;
36 |  |
     import org.owasp.dependencycheck.utils.DownloadFailedException;
37 |  |
     import org.owasp.dependencycheck.utils.Downloader;
38 |  |
     import org.owasp.dependencycheck.utils.Settings;
39 |  |
     import org.owasp.dependencycheck.data.nvdcve.DatabaseException;
40 |  |
     import org.owasp.dependencycheck.utils.InvalidSettingException;
41 |  |
     import static org.owasp.dependencycheck.data.update.DataStoreMetaInfo.MODIFIED;
42 |  |
43 |  |
     /**
44 |  |
      * Class responsible for updating the CPE and NVDCVE data stores.
45 |  |
      *
46 |  |
      * @author Jeremy Long (jeremy.long@owasp.org)
47 |  |
      */
48 |  |
     public class StandardUpdateTask extends AbstractUpdateTask {
49 |  |
50 |  |
         /**
51 |  |
          * Constructs a new Standard Update Task.
52 |  |
          *
53 |  |
          * @param properties information about the data store
54 |  |
          * @throws MalformedURLException thrown if a configured URL is malformed
55 |  |
          * @throws DownloadFailedException thrown if a timestamp cannot be checked
56 |  |
          * on a configured URL
57 |  |
          * @throws UpdateException thrown if there is an exception generating the
58 |  |
          * update task
59 |  |
          */
60 |  |
         public StandardUpdateTask(DataStoreMetaInfo properties) throws MalformedURLException, DownloadFailedException, UpdateException {
61 | 0 |
             super(properties);
62 | 0 |
         }
63 |  |
64 |  |
         /**
65 |  |
          * <p>Downloads the latest NVD CVE XML file from the web and imports it into
66 |  |
          * the current CVE Database.</p>
67 |  |
          *
68 |  |
          * @throws UpdateException is thrown if there is an error updating the
69 |  |
          * database
70 |  |
          */
71 |  |
         @Override
72 |  |
         public void update() throws UpdateException {
73 |  |
             try {
74 | 0 |
                 int maxUpdates = 0;
75 | 0 |
                 for (NvdCveInfo cve : getUpdateable()) {
76 | 0 |
                     if (cve.getNeedsUpdate()) {
77 | 0 |
                         maxUpdates += 1;
78 |  |
                     }
79 |  |
                 }
80 | 0 |
                 if (maxUpdates > 3) {
81 | 0 |
                     Logger.getLogger(StandardUpdateTask.class.getName()).log(Level.INFO,
82 |  |
                             "NVD CVE requires several updates; this could take a couple of minutes.");
83 |  |
                 }
84 | 0 |
                 if (maxUpdates > 0) {
85 | 0 |
                     openDataStores();
86 |  |
                 }
87 |  |
88 | 0 |
                 int count = 0;
89 | 0 |
                 for (NvdCveInfo cve : getUpdateable()) {
90 | 0 |
                     if (cve.getNeedsUpdate()) {
91 | 0 |
                         count += 1;
92 | 0 |
                         Logger.getLogger(StandardUpdateTask.class.getName()).log(Level.INFO,
93 |  |
                                 "Updating NVD CVE ({0} of {1})", new Object[]{count, maxUpdates});
94 | 0 |
                         URL url = new URL(cve.getUrl());
95 | 0 |
                         File outputPath = null;
96 | 0 |
                         File outputPath12 = null;
97 |  |
                         try {
98 | 0 |
                             Logger.getLogger(StandardUpdateTask.class.getName()).log(Level.INFO,
99 |  |
                                     "Downloading {0}", cve.getUrl());
100 | 0 |
                             outputPath = File.createTempFile("cve" + cve.getId() + "_", ".xml");
101 | 0 |
                             Downloader.fetchFile(url, outputPath);
102 |  |
103 | 0 |
                             url = new URL(cve.getOldSchemaVersionUrl());
104 | 0 |
                             outputPath12 = File.createTempFile("cve_1_2_" + cve.getId() + "_", ".xml");
105 | 0 |
                             Downloader.fetchFile(url, outputPath12);
106 |  |
107 | 0 |
                             Logger.getLogger(StandardUpdateTask.class.getName()).log(Level.INFO,
108 |  |
                                     "Processing {0}", cve.getUrl());
109 |  |
110 | 0 |
                             importXML(outputPath, outputPath12);
111 |  |
112 | 0 |
                             getCveDB().commit();
113 | 0 |
                             getCpeIndex().commit();
114 | 0 |
                             getProperties().save(cve);
115 |  |
116 | 0 |
                             Logger.getLogger(StandardUpdateTask.class.getName()).log(Level.INFO,
117 |  |
                                     "Completed update {0} of {1}", new Object[]{count, maxUpdates});
118 | 0 |
                         } catch (FileNotFoundException ex) {
119 | 0 |
                             throw new UpdateException(ex);
120 | 0 |
                         } catch (ParserConfigurationException ex) {
121 | 0 |
                             throw new UpdateException(ex);
122 | 0 |
                         } catch (SAXException ex) {
123 | 0 |
                             throw new UpdateException(ex);
124 | 0 |
                         } catch (IOException ex) {
125 | 0 |
                             throw new UpdateException(ex);
126 | 0 |
                         } catch (SQLException ex) {
127 | 0 |
                             throw new UpdateException(ex);
128 | 0 |
                         } catch (DatabaseException ex) {
129 | 0 |
                             throw new UpdateException(ex);
130 | 0 |
                         } catch (ClassNotFoundException ex) {
131 | 0 |
                             throw new UpdateException(ex);
132 |  |
                         } finally {
133 | 0 |
                             boolean deleted = false;
134 |  |
                             try {
135 | 0 |
                                 if (outputPath != null && outputPath.exists()) {
136 | 0 |
                                     deleted = outputPath.delete();
137 |  |
                                 }
138 |  |
                             } finally {
139 | 0 |
                                 if (outputPath != null && (outputPath.exists() || !deleted)) {
140 | 0 |
                                     outputPath.deleteOnExit();
141 |  |
                                 }
142 |  |
                             }
143 |  |
                             try {
144 | 0 |
                                 deleted = false;
145 | 0 |
                                 if (outputPath12 != null && outputPath12.exists()) {
146 | 0 |
                                     deleted = outputPath12.delete();
147 |  |
                                 }
148 |  |
                             } finally {
149 | 0 |
                                 if (outputPath12 != null && (outputPath12.exists() || !deleted)) {
150 | 0 |
                                     outputPath12.deleteOnExit();
151 |  |
                                 }
152 |  |
                             }
153 | 0 |
                         }
154 | 0 |
                     }
155 |  |
                 }
156 | 0 |
                 if (maxUpdates >= 1) { //ensure the modified file date gets written
157 | 0 |
                     getProperties().save(getUpdateable().get(MODIFIED));
158 | 0 |
                     getCveDB().cleanupDatabase();
159 |  |
                 }
160 | 0 |
             } catch (MalformedURLException ex) {
161 | 0 |
                 throw new UpdateException(ex);
162 |  |
             } finally {
163 | 0 |
                 closeDataStores();
164 | 0 |
             }
165 | 0 |
         }
166 |  |
167 |  |
         /**
168 |  |
          * Determines if the index needs to be updated. This is done by fetching the
169 |  |
          * NVD CVE meta data and checking the last update date. If the data needs to
170 |  |
          * be refreshed this method will return the NvdCveUrl for the files that
171 |  |
          * need to be updated.
172 |  |
          *
173 |  |
          * @return the collection of files that need to be updated
174 |  |
          * @throws MalformedURLException is thrown if the URL for the NVD CVE Meta
175 |  |
          * data is incorrect
176 |  |
          * @throws DownloadFailedException is thrown if there is an error.
177 |  |
          * downloading the NVD CVE download data file
178 |  |
          * @throws UpdateException Is thrown if there is an issue with the last
179 |  |
          * updated properties file
180 |  |
          */
181 |  |
         @Override
182 |  |
         protected Updateable updatesNeeded() throws MalformedURLException, DownloadFailedException, UpdateException {
183 | 0 |
             Updateable updates = null;
184 |  |
             try {
185 | 0 |
                 updates = retrieveCurrentTimestampsFromWeb();
186 | 0 |
             } catch (InvalidDataException ex) {
187 | 0 |
                 final String msg = "Unable to retrieve valid timestamp from nvd cve downloads page";
188 | 0 |
                 Logger.getLogger(StandardUpdateTask.class.getName()).log(Level.FINE, msg, ex);
189 | 0 |
                 throw new DownloadFailedException(msg, ex);
190 | 0 |
             } catch (InvalidSettingException ex) {
191 | 0 |
                 Logger.getLogger(StandardUpdateTask.class.getName()).log(Level.FINE, "Invalid setting found when retrieving timestamps", ex);
192 | 0 |
                 throw new DownloadFailedException("Invalid settings", ex);
193 | 0 |
             }
194 |  |
195 | 0 |
             if (updates == null) {
196 | 0 |
                 throw new DownloadFailedException("Unable to retrieve the timestamps of the currently published NVD CVE data");
197 |  |
             }
198 | 0 |
             final DataStoreMetaInfo properties = getProperties();
199 | 0 |
             if (!properties.isEmpty()) {
200 |  |
                 try {
201 |  |
                     float version;
202 |  |
203 | 0 |
                     if (properties.getProperty("version") == null) {
204 | 0 |
                         setDeleteAndRecreate(true);
205 |  |
                     } else {
206 |  |
                         try {
207 | 0 |
                             version = Float.parseFloat(properties.getProperty("version"));
208 | 0 |
                             final float currentVersion = Float.parseFloat(CveDB.DB_SCHEMA_VERSION);
209 | 0 |
                             if (currentVersion > version) {
210 | 0 |
                                 setDeleteAndRecreate(true);
211 |  |
                             }
212 | 0 |
                         } catch (NumberFormatException ex) {
213 | 0 |
                             setDeleteAndRecreate(true);
214 | 0 |
                         }
215 |  |
                     }
216 |  |
217 | 0 |
                     if (shouldDeleteAndRecreate()) {
218 | 0 |
                         return updates;
219 |  |
                     }
220 |  |
221 | 0 |
                     final long lastUpdated = Long.parseLong(properties.getProperty(DataStoreMetaInfo.LAST_UPDATED, "0"));
222 | 0 |
                     final Date now = new Date();
223 | 0 |
                     final int days = Settings.getInt(Settings.KEYS.CVE_MODIFIED_VALID_FOR_DAYS, 7);
224 | 0 |
                     if (lastUpdated == updates.getTimeStamp(MODIFIED)) {
225 | 0 |
                         updates.clear(); //we don't need to update anything.
226 | 0 |
                     } else if (withinRange(lastUpdated, now.getTime(), days)) {
227 | 0 |
                         for (NvdCveInfo entry : updates) {
228 | 0 |
                             if (MODIFIED.equals(entry.getId())) {
229 | 0 |
                                 entry.setNeedsUpdate(true);
230 |  |
                             } else {
231 | 0 |
                                 entry.setNeedsUpdate(false);
232 |  |
                             }
233 |  |
                         }
234 |  |
                     } else { //we figure out which of the several XML files need to be downloaded.
235 | 0 |
                         for (NvdCveInfo entry : updates) {
236 | 0 |
                             if (MODIFIED.equals(entry.getId())) {
237 | 0 |
                                 entry.setNeedsUpdate(true);
238 |  |
                             } else {
239 | 0 |
                                 long currentTimestamp = 0;
240 |  |
                                 try {
241 | 0 |
                                     currentTimestamp = Long.parseLong(properties.getProperty(DataStoreMetaInfo.LAST_UPDATED_BASE + entry.getId(), "0"));
242 | 0 |
                                 } catch (NumberFormatException ex) {
243 | 0 |
                                     final String msg = String.format("Error parsing '%s' '%s' from nvdcve.lastupdated",
244 |  |
                                             DataStoreMetaInfo.LAST_UPDATED_BASE, entry.getId());
245 | 0 |
                                     Logger.getLogger(StandardUpdateTask.class.getName()).log(Level.FINE, msg, ex);
246 | 0 |
                                 }
247 | 0 |
                                 if (currentTimestamp == entry.getTimestamp()) {
248 | 0 |
                                     entry.setNeedsUpdate(false);
249 |  |
                                 }
250 | 0 |
                             }
251 |  |
                         }
252 |  |
                     }
253 | 0 |
                 } catch (NumberFormatException ex) {
254 | 0 |
                     final String msg = "An invalid schema version or timestamp exists in the data.properties file.";
255 | 0 |
                     Logger.getLogger(StandardUpdateTask.class.getName()).log(Level.WARNING, msg);
256 | 0 |
                     Logger.getLogger(StandardUpdateTask.class.getName()).log(Level.FINE, null, ex);
257 | 0 |
                 }
258 |  |
             }
259 | 0 |
             return updates;
260 |  |
         }
261 |  |
262 |  |
         /**
263 |  |
          * Retrieves the timestamps from the NVD CVE meta data file.
264 |  |
          *
265 |  |
          * @return the timestamp from the currently published nvdcve downloads page
266 |  |
          * @throws MalformedURLException thrown if the URL for the NVD CCE Meta data
267 |  |
          * is incorrect.
268 |  |
          * @throws DownloadFailedException thrown if there is an error downloading
269 |  |
          * the nvd cve meta data file
270 |  |
          * @throws InvalidDataException thrown if there is an exception parsing the
271 |  |
          * timestamps
272 |  |
          * @throws InvalidSettingException thrown if the settings are invalid
273 |  |
          */
274 |  |
         private Updateable retrieveCurrentTimestampsFromWeb()
275 |  |
                 throws MalformedURLException, DownloadFailedException, InvalidDataException, InvalidSettingException {
276 |  |
277 | 0 |
             final Updateable updates = new Updateable();
278 | 0 |
             updates.add(MODIFIED, Settings.getString(Settings.KEYS.CVE_MODIFIED_20_URL),
279 |  |
                     Settings.getString(Settings.KEYS.CVE_MODIFIED_12_URL),
280 |  |
                     false);
281 |  |
282 | 0 |
             final int start = Settings.getInt(Settings.KEYS.CVE_START_YEAR);
283 | 0 |
             final int end = Calendar.getInstance().get(Calendar.YEAR);
284 | 0 |
             final String baseUrl20 = Settings.getString(Settings.KEYS.CVE_SCHEMA_2_0);
285 | 0 |
             final String baseUrl12 = Settings.getString(Settings.KEYS.CVE_SCHEMA_1_2);
286 | 0 |
             for (int i = start; i <= end; i++) {
287 | 0 |
                 updates.add(Integer.toString(i), String.format(baseUrl20, i),
288 |  |
                         String.format(baseUrl12, i),
289 |  |
                         true);
290 |  |
             }
291 |  |
292 | 0 |
             return updates;
293 |  |
         }
294 |  |
     }