github-mirrors/DependencyCheck
1
0
Fork 0
mirror of https://github.com/ysoftdevs/DependencyCheck.git synced 2026-01-13 07:20:40 +01:00
Code Issues Packages Projects Releases Wiki Activity

updated batch update functionality

Browse Source 
Former-commit-id: 891c0148c081ac191258f5310d2077ed61039353
This commit is contained in:
Jeremy Long
2013-08-06 19:34:11 -04:00
parent 99bc57e75d
commit 32ad8e8ca1
8 changed files with 156 additions and 36 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
dependency-check-core/src/main/java/org/owasp/dependencycheck/data/nvdcve/CveDB.java
View File

@@ -176,7 +176,7 @@ public class CveDB {
final File f = new File(fileName, "cve." + DB_SCHEMA_VERSION);
final File check = new File(f.getAbsolutePath() + ".h2.db");
final boolean createTables = !check.exists();
final String connStr = "jdbc:h2:file:" + f.getAbsolutePath();
final String connStr = String.format("jdbc:h2:file:%s;AUTO_SERVER=TRUE", f.getAbsolutePath());
Class.forName("org.h2.Driver");
conn = DriverManager.getConnection(connStr, "sa", "");
if (createTables) {

49
dependency-check-core/src/main/java/org/owasp/dependencycheck/data/nvdcve/xml/DatabaseUpdater.java
View File

@@ -29,6 +29,7 @@ import java.io.OutputStreamWriter;
import javax.xml.parsers.ParserConfigurationException;
import org.owasp.dependencycheck.data.CachedWebDataSource;
import java.net.MalformedURLException;
import java.net.URISyntaxException;
import java.net.URL;
import java.sql.SQLException;
import java.util.Calendar;
@@ -114,13 +115,14 @@ public class DatabaseUpdater implements CachedWebDataSource {
Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.INFO,
"NVD CVE requires several updates; this could take a couple of minutes.");
}
if (maxUpdates > 0) {
if (maxUpdates > 0 && !isDoBatchUpdate()) {
openDataStores();
}
if (isBatchUpdateMode() && isDoBatchUpdate()) {
try {
performBatchUpdate();
openDataStores();
} catch (IOException ex) {
throw new UpdateException("Unable to perform batch update", ex);
}
@@ -498,7 +500,12 @@ public class DatabaseUpdater implements CachedWebDataSource {
}
}
}
} else {
//properties file does not exist - check about batch update
setDoBatchUpdate(isBatchUpdateMode());
}
} else { //this condition will likely never exist - but just in case we need to handle batch updates
setDoBatchUpdate(isBatchUpdateMode());
}
return currentlyPublished;
}
@@ -627,7 +634,7 @@ public class DatabaseUpdater implements CachedWebDataSource {
*
* @throws IOException thrown if the directory cannot be deleted
*/
private void deleteExistingData() throws IOException {
protected void deleteExistingData() throws IOException {
Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.INFO, "The database version is old. Rebuilding the database.");
final File cveDir = CveDB.getDataDirectory();
@@ -637,15 +644,37 @@ public class DatabaseUpdater implements CachedWebDataSource {
FileUtils.delete(cpeDir);
}
private void performBatchUpdate() throws IOException {
private void performBatchUpdate() throws UpdateException {
if (batchUpdateMode && doBatchUpdate) {
deleteExistingData();
String batchSrc = Settings.getString(Settings.KEYS.BATCH_UPDATE_URL);
File dataDirectory = CveDB.getDataDirectory().getParentFile();
URL batchUrl = new URL(batchSrc);
File tmp = File.createTempFile("batch_", ".zip");
Downloader.fetchFile(batchUrl, tmp);
FileUtils.extractFiles(tmp, dataDirectory);
final String batchSrc = Settings.getString(Settings.KEYS.BATCH_UPDATE_URL);
File tmp = null;
try {
deleteExistingData();
final File dataDirectory = CveDB.getDataDirectory().getParentFile();
final URL batchUrl = new URL(batchSrc);
if ("file".equals(batchUrl.getProtocol())) {
try {
tmp = new File(batchUrl.toURI());
} catch (URISyntaxException ex) {
final String msg = String.format("Invalid batch update URI: %s", batchSrc);
throw new UpdateException(msg, ex);
}
} else if ("http".equals(batchUrl.getProtocol())
|| "https".equals(batchUrl.getProtocol())) {
tmp = File.createTempFile("batch_", ".zip");
Downloader.fetchFile(batchUrl, tmp);
}
//TODO add FTP?
FileUtils.extractFiles(tmp, dataDirectory);
} catch (IOException ex) {
final String msg = String.format("IO Exception Occured performing batch update using: %s", batchSrc);
throw new UpdateException(msg, ex);
} finally {
if (tmp != null && !tmp.delete()) {
tmp.deleteOnExit();
}
}
}
}

27
dependency-check-core/src/main/java/org/owasp/dependencycheck/utils/Downloader.java
View File

@@ -131,23 +131,23 @@ public final class Downloader {
long timestamp = 0;
//TODO add the FPR protocol?
if ("file".equalsIgnoreCase(url.getProtocol())) {
File f;
File lastModifiedFile;
try {
if (System.getProperty("os.name").toLowerCase().startsWith("windows")) {
String filePath = url.toString();
if (filePath.matches("file://[a-zA-Z]:.*")) {
f = new File(filePath.substring(7));
} else {
f = new File(url.toURI());
}
} else {
f = new File(url.toURI());
}
// if (System.getProperty("os.name").toLowerCase().startsWith("windows")) {
// String filePath = url.toString();
// if (filePath.matches("file://[a-zA-Z]:.*")) {
// f = new File(filePath.substring(7));
// } else {
// f = new File(url.toURI());
// }
// } else {
lastModifiedFile = new File(url.toURI());
// }
} catch (URISyntaxException ex) {
final String msg = String.format("Unable to locate '%s'; is the cve.url-2.0.modified property set correctly?", url.toString());
throw new DownloadFailedException(msg);
}
timestamp = f.lastModified();
timestamp = lastModifiedFile.lastModified();
} else {
HttpURLConnection conn = null;
try {
@@ -192,11 +192,8 @@ public final class Downloader {
} else {
conn = (HttpURLConnection) url.openConnection();
}
//added a default timeout of 20000
//if (Settings.getString(Settings.KEYS.CONNECTION_TIMEOUT) != null) {
final int timeout = Settings.getInt(Settings.KEYS.CONNECTION_TIMEOUT, 60000);
conn.setConnectTimeout(timeout);
//}
} catch (IOException ex) {
if (conn != null) {
try {

2
dependency-check-core/src/main/java/org/owasp/dependencycheck/utils/FileUtils.java
View File

@@ -200,7 +200,7 @@ public final class FileUtils {
while ((entry = zis.getNextEntry()) != null) {
if (entry.isDirectory()) {
final File d = new File(extractTo, entry.getName());
if (!d.mkdirs()) {
if (!d.exists() && !d.mkdirs()) {
final String msg = String.format("Unable to create '%s'.", d.getAbsolutePath());
throw new ExtractionException(msg);
}

2
dependency-check-core/src/main/resources/dependencycheck.properties
View File

@@ -24,7 +24,7 @@ cve.url.modified.validfordays=7
# into the specified "data" directory. Additionally, after pulling the data the
# system will attempt to update the modified. Thus, if one were maintaining an
# internal copy of the data one would not need to update it nightly.
batch.update.url=file://C:/Users/jeremy/Desktop/demo/cli/data/data.zip
#batch.update.url=file:///C:/path/to/data.zip
# the path to the modified nvd cve xml file.
cve.url-1.2.modified=http://nvd.nist.gov/download/nvdcve-modified.xml

17
dependency-check-core/src/test/java/org/owasp/dependencycheck/data/nvdcve/xml/DatabaseUpdaterIntegrationTest.java
View File

@@ -18,12 +18,15 @@
*/
package org.owasp.dependencycheck.data.nvdcve.xml;
import java.io.File;
import java.net.URL;
import org.owasp.dependencycheck.data.nvdcve.xml.DatabaseUpdater;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
import org.owasp.dependencycheck.utils.Settings;
/**
*
@@ -60,4 +63,18 @@ public class DatabaseUpdaterIntegrationTest {
DatabaseUpdater instance = new DatabaseUpdater();
instance.update();
}
/**
* Test of update method (when in batch mode), of class DatabaseUpdater.
*
* @throws Exception
*/
@Test
public void testBatchUpdate() throws Exception {
File file = new File("target/test-classes/nvdcve-2.0-2012.xml");
String path = "file:///" + file.getCanonicalPath();
Settings.setString(Settings.KEYS.BATCH_UPDATE_URL, path);
DatabaseUpdater instance = new DatabaseUpdater();
instance.update();
}
}

85
dependency-check-core/src/test/java/org/owasp/dependencycheck/data/nvdcve/xml/DatabaseUpdaterTest.java Normal file
View File

@@ -0,0 +1,85 @@
/*
* This file is part of dependency-check-core.
*
* Dependency-check-core is free software: you can redistribute it and/or modify it
* under the terms of the GNU General Public License as published by the Free
* Software Foundation, either version 3 of the License, or (at your option) any
* later version.
*
* Dependency-check-core is distributed in the hope that it will be useful, but
* WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
* details.
*
* You should have received a copy of the GNU General Public License along with
* dependency-check-core. If not, see http://www.gnu.org/licenses/.
*
* Copyright (c) 2012 Jeremy Long. All Rights Reserved.
*/
package org.owasp.dependencycheck.data.nvdcve.xml;
import org.owasp.dependencycheck.data.nvdcve.xml.DatabaseUpdater;
import java.io.File;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
import org.owasp.dependencycheck.utils.Settings;
/**
*
* @author Jeremy Long (jeremy.long@owasp.org)
*/
public class DatabaseUpdaterTest {
public DatabaseUpdaterTest() {
}
@BeforeClass
public static void setUpClass() throws Exception {
}
@AfterClass
public static void tearDownClass() throws Exception {
}
private String old12;
private String old20;
@Before
public void setUp() throws Exception {
old12 = Settings.getString(Settings.KEYS.CVE_MODIFIED_12_URL);
old20 = Settings.getString(Settings.KEYS.CVE_MODIFIED_20_URL);
File file = new File("target/test-classes/nvdcve-2012.xml");
String path = "file:///" + file.getCanonicalPath();
Settings.setString(Settings.KEYS.CVE_MODIFIED_12_URL, path);
file = new File("target/test-classes/nvdcve-2.0-2012.xml");
path = "file:///" + file.getCanonicalPath();
Settings.setString(Settings.KEYS.CVE_MODIFIED_20_URL, path);
file = new File("target/test-classes/data.zip");
path = "file:///" + file.getCanonicalPath();
Settings.setString(Settings.KEYS.BATCH_UPDATE_URL, path);
}
@After
public void tearDown() {
Settings.setString(Settings.KEYS.CVE_MODIFIED_12_URL, old12);
Settings.setString(Settings.KEYS.CVE_MODIFIED_20_URL, old20);
Settings.setString(Settings.KEYS.BATCH_UPDATE_URL, "");
}
/**
* Test of update method (when in batch mode), of class DatabaseUpdater.
*
* @throws Exception
*/
@Test
public void testBatchUpdate() throws Exception {
DatabaseUpdater instance = new DatabaseUpdater();
instance.deleteExistingData();
instance.update();
}
}

8
dependency-check-core/src/test/java/org/owasp/dependencycheck/utils/DownloaderIntegrationTest.java
View File

@@ -82,12 +82,4 @@ public class DownloaderIntegrationTest {
long timestamp = Downloader.getLastModified(url);
assertTrue("timestamp equal to zero?", timestamp > 0);
}
@Test
public void testGetLastModified_file() throws Exception {
File f = new File("target/test-classes/nvdcve-2.0-2012.xml");
URL url = new URL("file://" + f.getCanonicalPath());
long timestamp = Downloader.getLastModified(url);
assertTrue("timestamp equal to zero?", timestamp > 0);
}
}