1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.owasp.dependencycheck.data.update;
19
20 import java.net.MalformedURLException;
21 import java.util.Calendar;
22 import java.util.HashSet;
23 import java.util.Set;
24 import java.util.concurrent.ExecutionException;
25 import java.util.concurrent.ExecutorService;
26 import java.util.concurrent.Executors;
27 import java.util.concurrent.Future;
28 import org.owasp.dependencycheck.data.nvdcve.DatabaseProperties;
29 import static org.owasp.dependencycheck.data.nvdcve.DatabaseProperties.MODIFIED;
30 import org.owasp.dependencycheck.data.update.exception.InvalidDataException;
31 import org.owasp.dependencycheck.data.update.exception.UpdateException;
32 import org.owasp.dependencycheck.data.update.nvd.DownloadTask;
33 import org.owasp.dependencycheck.data.update.nvd.NvdCveInfo;
34 import org.owasp.dependencycheck.data.update.nvd.ProcessTask;
35 import org.owasp.dependencycheck.data.update.nvd.UpdateableNvdCve;
36 import org.owasp.dependencycheck.utils.DateUtil;
37 import org.owasp.dependencycheck.utils.DownloadFailedException;
38 import org.owasp.dependencycheck.utils.InvalidSettingException;
39 import org.owasp.dependencycheck.utils.Settings;
40 import org.slf4j.Logger;
41 import org.slf4j.LoggerFactory;
42
43
44
45
46
47
48 public class NvdCveUpdater extends BaseUpdater implements CachedWebDataSource {
49
50
51
52
53 private static final Logger LOGGER = LoggerFactory.getLogger(NvdCveUpdater.class);
54
55
56
57 public static final int MAX_THREAD_POOL_SIZE = Settings.getInt(Settings.KEYS.MAX_DOWNLOAD_THREAD_POOL_SIZE, 3);
58
59
60
61
62
63
64
65 @Override
66 public void update() throws UpdateException {
67 try {
68 openDataStores();
69 final UpdateableNvdCve updateable = getUpdatesNeeded();
70 if (updateable.isUpdateNeeded()) {
71 performUpdate(updateable);
72 }
73 } catch (MalformedURLException ex) {
74 LOGGER.warn(
75 "NVD CVE properties files contain an invalid URL, unable to update the data to use the most current data.");
76 LOGGER.debug("", ex);
77 } catch (DownloadFailedException ex) {
78 LOGGER.warn(
79 "Unable to download the NVD CVE data; the results may not include the most recent CPE/CVEs from the NVD.");
80 if (Settings.getString(Settings.KEYS.PROXY_SERVER) == null) {
81 LOGGER.info(
82 "If you are behind a proxy you may need to configure dependency-check to use the proxy.");
83 }
84 LOGGER.debug("", ex);
85 } finally {
86 closeDataStores();
87 }
88 }
89
90
91
92
93
94
95
96
97 public void performUpdate(UpdateableNvdCve updateable) throws UpdateException {
98 int maxUpdates = 0;
99 try {
100 for (NvdCveInfo cve : updateable) {
101 if (cve.getNeedsUpdate()) {
102 maxUpdates += 1;
103 }
104 }
105 if (maxUpdates <= 0) {
106 return;
107 }
108 if (maxUpdates > 3) {
109 LOGGER.info(
110 "NVD CVE requires several updates; this could take a couple of minutes.");
111 }
112 if (maxUpdates > 0) {
113 openDataStores();
114 }
115
116 final int poolSize = (MAX_THREAD_POOL_SIZE < maxUpdates) ? MAX_THREAD_POOL_SIZE : maxUpdates;
117
118 final ExecutorService downloadExecutors = Executors.newFixedThreadPool(poolSize);
119 final ExecutorService processExecutor = Executors.newSingleThreadExecutor();
120 final Set<Future<Future<ProcessTask>>> downloadFutures = new HashSet<Future<Future<ProcessTask>>>(maxUpdates);
121 for (NvdCveInfo cve : updateable) {
122 if (cve.getNeedsUpdate()) {
123 final DownloadTask call = new DownloadTask(cve, processExecutor, getCveDB(), Settings.getInstance());
124 downloadFutures.add(downloadExecutors.submit(call));
125 }
126 }
127 downloadExecutors.shutdown();
128
129
130 final Set<Future<ProcessTask>> processFutures = new HashSet<Future<ProcessTask>>(maxUpdates);
131 for (Future<Future<ProcessTask>> future : downloadFutures) {
132 Future<ProcessTask> task = null;
133 try {
134 task = future.get();
135 } catch (InterruptedException ex) {
136 downloadExecutors.shutdownNow();
137 processExecutor.shutdownNow();
138
139 LOGGER.debug("Thread was interrupted during download", ex);
140 throw new UpdateException("The download was interrupted", ex);
141 } catch (ExecutionException ex) {
142 downloadExecutors.shutdownNow();
143 processExecutor.shutdownNow();
144
145 LOGGER.debug("Thread was interrupted during download execution", ex);
146 throw new UpdateException("The execution of the download was interrupted", ex);
147 }
148 if (task == null) {
149 downloadExecutors.shutdownNow();
150 processExecutor.shutdownNow();
151 LOGGER.debug("Thread was interrupted during download");
152 throw new UpdateException("The download was interrupted; unable to complete the update");
153 } else {
154 processFutures.add(task);
155 }
156 }
157
158 for (Future<ProcessTask> future : processFutures) {
159 try {
160 final ProcessTask task = future.get();
161 if (task.getException() != null) {
162 throw task.getException();
163 }
164 } catch (InterruptedException ex) {
165 processExecutor.shutdownNow();
166 LOGGER.debug("Thread was interrupted during processing", ex);
167 throw new UpdateException(ex);
168 } catch (ExecutionException ex) {
169 processExecutor.shutdownNow();
170 LOGGER.debug("Execution Exception during process", ex);
171 throw new UpdateException(ex);
172 } finally {
173 processExecutor.shutdown();
174 }
175 }
176
177 if (maxUpdates >= 1) {
178 getProperties().save(updateable.get(MODIFIED));
179 LOGGER.info("Begin database maintenance.");
180 getCveDB().cleanupDatabase();
181 LOGGER.info("End database maintenance.");
182 }
183 } finally {
184 closeDataStores();
185 }
186 }
187
188
189
190
191
192
193
194
195
196
197 protected final UpdateableNvdCve getUpdatesNeeded() throws MalformedURLException, DownloadFailedException, UpdateException {
198 UpdateableNvdCve updates = null;
199 try {
200 updates = retrieveCurrentTimestampsFromWeb();
201 } catch (InvalidDataException ex) {
202 final String msg = "Unable to retrieve valid timestamp from nvd cve downloads page";
203 LOGGER.debug(msg, ex);
204 throw new DownloadFailedException(msg, ex);
205 } catch (InvalidSettingException ex) {
206 LOGGER.debug("Invalid setting found when retrieving timestamps", ex);
207 throw new DownloadFailedException("Invalid settings", ex);
208 }
209
210 if (updates == null) {
211 throw new DownloadFailedException("Unable to retrieve the timestamps of the currently published NVD CVE data");
212 }
213 if (!getProperties().isEmpty()) {
214 try {
215 final long lastUpdated = Long.parseLong(getProperties().getProperty(DatabaseProperties.LAST_UPDATED, "0"));
216 final long now = System.currentTimeMillis();
217 final int days = Settings.getInt(Settings.KEYS.CVE_MODIFIED_VALID_FOR_DAYS, 7);
218 if (lastUpdated == updates.getTimeStamp(MODIFIED)) {
219 updates.clear();
220 } else if (DateUtil.withinDateRange(lastUpdated, now, days)) {
221 for (NvdCveInfo entry : updates) {
222 if (MODIFIED.equals(entry.getId())) {
223 entry.setNeedsUpdate(true);
224 } else {
225 entry.setNeedsUpdate(false);
226 }
227 }
228 } else {
229 for (NvdCveInfo entry : updates) {
230 if (MODIFIED.equals(entry.getId())) {
231 entry.setNeedsUpdate(true);
232 } else {
233 long currentTimestamp = 0;
234 try {
235 currentTimestamp = Long.parseLong(getProperties().getProperty(DatabaseProperties.LAST_UPDATED_BASE
236 + entry.getId(), "0"));
237 } catch (NumberFormatException ex) {
238 LOGGER.debug("Error parsing '{}' '{}' from nvdcve.lastupdated",
239 DatabaseProperties.LAST_UPDATED_BASE, entry.getId(), ex);
240 }
241 if (currentTimestamp == entry.getTimestamp()) {
242 entry.setNeedsUpdate(false);
243 }
244 }
245 }
246 }
247 } catch (NumberFormatException ex) {
248 LOGGER.warn("An invalid schema version or timestamp exists in the data.properties file.");
249 LOGGER.debug("", ex);
250 }
251 }
252 return updates;
253 }
254
255
256
257
258
259
260
261
262
263
264 private UpdateableNvdCve retrieveCurrentTimestampsFromWeb()
265 throws MalformedURLException, DownloadFailedException, InvalidDataException, InvalidSettingException {
266
267 final UpdateableNvdCve updates = new UpdateableNvdCve();
268 updates.add(MODIFIED, Settings.getString(Settings.KEYS.CVE_MODIFIED_20_URL),
269 Settings.getString(Settings.KEYS.CVE_MODIFIED_12_URL),
270 false);
271
272 final int start = Settings.getInt(Settings.KEYS.CVE_START_YEAR);
273 final int end = Calendar.getInstance().get(Calendar.YEAR);
274 final String baseUrl20 = Settings.getString(Settings.KEYS.CVE_SCHEMA_2_0);
275 final String baseUrl12 = Settings.getString(Settings.KEYS.CVE_SCHEMA_1_2);
276 for (int i = start; i <= end; i++) {
277 updates.add(Integer.toString(i), String.format(baseUrl20, i),
278 String.format(baseUrl12, i),
279 true);
280 }
281 return updates;
282 }
283
284 }