github-mirrors/DependencyCheck
1
0
Fork 0
mirror of https://github.com/ysoftdevs/DependencyCheck.git synced 2026-01-15 16:23:37 +01:00
Code Issues Packages Projects Releases Wiki Activity

Refactoring: Move retrieval of last modified timestamps from UpdateableNvdCve to NvdCveUpdater

Browse Source 
- UpdateableNvdCve is from its nature more like a simple value object
- Facilitates performance optimization for retrieval of last modification timestamps
This commit is contained in:
Stefan Neuhaus
2017-02-16 07:48:06 +01:00
parent a9fc6bf02c
commit 3a70e25983
3 changed files with 66 additions and 37 deletions
Download Patch File Download Diff File Expand all files Collapse all files

55
dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/NvdCveUpdater.java
View File

@@ -19,8 +19,11 @@ package org.owasp.dependencycheck.data.update;
import java.net.MalformedURLException;
import java.util.Calendar;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
import java.net.URL;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
@@ -36,6 +39,7 @@ import org.owasp.dependencycheck.data.update.nvd.NvdCveInfo;
import org.owasp.dependencycheck.data.update.nvd.ProcessTask;
import org.owasp.dependencycheck.data.update.nvd.UpdateableNvdCve;
import org.owasp.dependencycheck.utils.DateUtil;
import org.owasp.dependencycheck.utils.Downloader;
import org.owasp.dependencycheck.utils.DownloadFailedException;
import org.owasp.dependencycheck.utils.InvalidSettingException;
import org.owasp.dependencycheck.utils.Settings;
@@ -357,20 +361,57 @@ public class NvdCveUpdater extends BaseUpdater implements CachedWebDataSource {
private UpdateableNvdCve retrieveCurrentTimestampsFromWeb()
throws MalformedURLException, DownloadFailedException, InvalidDataException, InvalidSettingException {
final UpdateableNvdCve updates = new UpdateableNvdCve();
updates.add(MODIFIED, Settings.getString(Settings.KEYS.CVE_MODIFIED_20_URL),
Settings.getString(Settings.KEYS.CVE_MODIFIED_12_URL),
false);
final int start = Settings.getInt(Settings.KEYS.CVE_START_YEAR);
final int end = Calendar.getInstance().get(Calendar.YEAR);
final Map<String, Long> lastModifiedDates = retrieveLastModifiedDates(start, end);
final UpdateableNvdCve updates = new UpdateableNvdCve();
final String baseUrl20 = Settings.getString(Settings.KEYS.CVE_SCHEMA_2_0);
final String baseUrl12 = Settings.getString(Settings.KEYS.CVE_SCHEMA_1_2);
for (int i = start; i <= end; i++) {
updates.add(Integer.toString(i), String.format(baseUrl20, i),
String.format(baseUrl12, i),
true);
final String url = String.format(baseUrl20, i);
updates.add(Integer.toString(i), url, String.format(baseUrl12, i),
lastModifiedDates.get(url), true);
}
final String url = Settings.getString(Settings.KEYS.CVE_MODIFIED_20_URL);
updates.add(MODIFIED, url, Settings.getString(Settings.KEYS.CVE_MODIFIED_12_URL),
lastModifiedDates.get(url), false);
return updates;
}
/**
* Retrieves the timestamps from the NVD CVE meta data file.
*
* @param startYear the first year whose item to check for the timestamp
* @param endYear the last year whose item to check for the timestamp
* @return the timestamps from the currently published nvdcve downloads page
* @throws MalformedURLException thrown if the URL for the NVD CCE Meta data
* is incorrect.
* @throws DownloadFailedException thrown if there is an error downloading
* the nvd cve meta data file
*/
private Map<String, Long> retrieveLastModifiedDates(int startYear, int endYear)
throws MalformedURLException, DownloadFailedException {
final Set<String> urls = new HashSet<String>();
final String baseUrl20 = Settings.getString(Settings.KEYS.CVE_SCHEMA_2_0);
for (int i = startYear; i <= endYear; i++) {
final String url = String.format(baseUrl20, i);
urls.add(url);
}
urls.add(Settings.getString(Settings.KEYS.CVE_MODIFIED_20_URL));
final Map<String, Long> lastModifiedDates = new HashMap<String, Long>();
for(String url: urls) {
LOGGER.debug("Checking for updates from: {}", url);
lastModifiedDates.put(url, Downloader.getLastModified(new URL(url)));
}
return lastModifiedDates;
}
}

12
dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/nvd/UpdateableNvdCve.java
View File

@@ -17,14 +17,10 @@
*/
package org.owasp.dependencycheck.data.update.nvd;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.Iterator;
import java.util.Map;
import java.util.Map.Entry;
import java.util.TreeMap;
import org.owasp.dependencycheck.utils.DownloadFailedException;
import org.owasp.dependencycheck.utils.Downloader;
/**
* Contains a collection of updateable NvdCveInfo objects. This is used to determine which files need to be downloaded and
@@ -68,18 +64,16 @@ public class UpdateableNvdCve implements Iterable<NvdCveInfo>, Iterator<NvdCveIn
* @param id the key for the item to be added
* @param url the URL to download the item
* @param oldUrl the URL for the old version of the item (the NVD CVE old schema still contains useful data we need).
* @param timestamp the last modified date of the downloaded item
* @param needsUpdate whether or not the data needs to be updated
* @throws MalformedURLException thrown if the URL provided is invalid
* @throws DownloadFailedException thrown if the download fails.
*/
public void add(String id, String url, String oldUrl, boolean needsUpdate) throws MalformedURLException, DownloadFailedException {
public void add(String id, String url, String oldUrl, long timestamp, boolean needsUpdate) {
final NvdCveInfo item = new NvdCveInfo();
item.setNeedsUpdate(needsUpdate); //the others default to true, to make life easier later this should default to false.
item.setId(id);
item.setUrl(url);
item.setOldSchemaVersionUrl(oldUrl);
LOGGER.debug("Checking for updates from: {}", url);
item.setTimestamp(Downloader.getLastModified(new URL(url)));
item.setTimestamp(timestamp);
collection.put(id, item);
}

36
dependency-check-core/src/test/java/org/owasp/dependencycheck/data/update/nvd/UpdateableNvdCveTest.java
View File

@@ -17,17 +17,12 @@
*/
package org.owasp.dependencycheck.data.update.nvd;
import org.owasp.dependencycheck.data.update.nvd.UpdateableNvdCve;
import org.owasp.dependencycheck.data.update.nvd.NvdCveInfo;
import java.io.File;
import java.io.IOException;
import java.net.MalformedURLException;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import org.junit.Test;
import org.owasp.dependencycheck.BaseTest;
import org.owasp.dependencycheck.utils.DownloadFailedException;
/**
*
@@ -39,18 +34,18 @@ public class UpdateableNvdCveTest extends BaseTest {
* Test of isUpdateNeeded method, of class UpdateableNvdCve.
*/
@Test
public void testIsUpdateNeeded() throws MalformedURLException, DownloadFailedException, IOException {
public void testIsUpdateNeeded() {
String id = "key";
//use a local file as this test will load the result and check the timestamp
String url = new File("target/test-classes/nvdcve-2.0-2012.xml").toURI().toString();
long timestamp = 42;
UpdateableNvdCve instance = new UpdateableNvdCve();
instance.add(id, url, url, false);
instance.add(id, url, url, timestamp, false);
boolean expResult = false;
boolean result = instance.isUpdateNeeded();
assertEquals(expResult, result);
instance.add("nextId", url, url, true);
instance.add("nextId", url, url, 23, true);
expResult = true;
result = instance.isUpdateNeeded();
@@ -63,34 +58,34 @@ public class UpdateableNvdCveTest extends BaseTest {
@Test
public void testAdd() throws Exception {
String id = "key";
//use a local file as this test will load the result and check the timestamp
String url = new File("target/test-classes/nvdcve-2.0-2012.xml").toURI().toString();
long timestamp = 42;
UpdateableNvdCve instance = new UpdateableNvdCve();
instance.add(id, url, url, false);
instance.add(id, url, url, timestamp, false);
boolean expResult = false;
boolean result = instance.isUpdateNeeded();
assertEquals(expResult, result);
instance.add("nextId", url, url, false);
instance.add("nextId", url, url, 23, false);
NvdCveInfo results = instance.get(id);
assertEquals(id, results.getId());
assertEquals(url, results.getUrl());
assertEquals(url, results.getOldSchemaVersionUrl());
assertEquals(timestamp, results.getTimestamp());
}
/**
* Test of clear method, of class UpdateableNvdCve.
*/
@Test
public void testClear() throws MalformedURLException, DownloadFailedException, IOException {
public void testClear() {
String id = "key";
//use a local file as this test will load the result and check the timestamp
String url = new File("target/test-classes/nvdcve-2.0-2012.xml").toURI().toString();
long timestamp = 42;
UpdateableNvdCve instance = new UpdateableNvdCve();
instance.add(id, url, url, false);
instance.add(id, url, url, timestamp, false);
assertFalse(instance.getCollection().isEmpty());
instance.clear();
assertTrue(instance.getCollection().isEmpty());
@@ -100,13 +95,12 @@ public class UpdateableNvdCveTest extends BaseTest {
* Test of iterator method, of class UpdatableNvdCve.
*/
@Test
public void testIterator() throws IOException {
//use a local file as this test will load the result and check the timestamp
public void testIterator() {
String url = new File("target/test-classes/nvdcve-2.0-2012.xml").toURI().toString();
UpdateableNvdCve instance = new UpdateableNvdCve();
instance.add("one", url, url, false);
instance.add("two", url, url, false);
instance.add("three", url, url, false);
instance.add("one", url, url, 42, false);
instance.add("two", url, url, 23, false);
instance.add("three", url, url, 17, false);
int itemsProcessed = 0;
for (NvdCveInfo item : instance) {
if ("one".equals(item.getId())) {