github-mirrors/DependencyCheck
1
0
Fork 0
mirror of https://github.com/ysoftdevs/DependencyCheck.git synced 2026-01-15 00:03:43 +01:00
Code Issues Packages Projects Releases Wiki Activity

reorganized code, made the database file name version independent so that upgrades can be made, and implemented a CPE updates per issue #149

Browse Source 
Former-commit-id: fdb57afa28ecffdb7ca90971851844718ecb8bb9
This commit is contained in:
Jeremy Long
2015-07-08 06:20:19 -04:00
parent da058fcaf5
commit 9daa9b6cca
38 changed files with 1397 additions and 523 deletions
Download Patch File Download Diff File Expand all files Collapse all files

83
dependency-check-core/src/main/java/org/owasp/dependencycheck/data/nvdcve/ConnectionFactory.java
View File

@@ -35,9 +35,8 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* Loads the configured database driver and returns the database connection. If the embedded H2 database is used
* obtaining a connection will ensure the database file exists and that the appropriate table structure has been
* created.
* Loads the configured database driver and returns the database connection. If the embedded H2 database is used obtaining a
* connection will ensure the database file exists and that the appropriate table structure has been created.
*
* @author Jeremy Long
*/
@@ -55,6 +54,10 @@ public final class ConnectionFactory {
* Resource location for SQL file used to create the database schema.
*/
public static final String DB_STRUCTURE_RESOURCE = "data/initialize.sql";
/**
* Resource location for SQL file used to create the database schema.
*/
public static final String DB_STRUCTURE_UPDATE_RESOURCE = "data/upgrade_%s.sql";
/**
* The database driver used to connect to the database.
*/
@@ -79,8 +82,8 @@ public final class ConnectionFactory {
}
/**
* Initializes the connection factory. Ensuring that the appropriate drivers are loaded and that a connection can be
* made successfully.
* Initializes the connection factory. Ensuring that the appropriate drivers are loaded and that a connection can be made
* successfully.
*
* @throws DatabaseException thrown if we are unable to connect to the database
*/
@@ -114,8 +117,7 @@ public final class ConnectionFactory {
try {
connectionString = Settings.getConnectionString(
Settings.KEYS.DB_CONNECTION_STRING,
Settings.KEYS.DB_FILE_NAME,
Settings.KEYS.DB_VERSION);
Settings.KEYS.DB_FILE_NAME);
} catch (IOException ex) {
LOGGER.debug(
"Unable to retrieve the database connection string", ex);
@@ -162,13 +164,12 @@ public final class ConnectionFactory {
LOGGER.debug("", dex);
throw new DatabaseException("Unable to create the database structure");
}
} else {
try {
ensureSchemaVersion(conn);
} catch (DatabaseException dex) {
LOGGER.debug("", dex);
throw new DatabaseException("Database schema does not match this version of dependency-check");
}
}
try {
ensureSchemaVersion(conn);
} catch (DatabaseException dex) {
LOGGER.debug("", dex);
throw new DatabaseException("Database schema does not match this version of dependency-check", dex);
}
} finally {
if (conn != null) {
@@ -183,8 +184,8 @@ public final class ConnectionFactory {
/**
* Cleans up resources and unloads any registered database drivers. This needs to be called to ensure the driver is
* unregistered prior to the finalize method being called as during shutdown the class loader used to load the
* driver may be unloaded prior to the driver being de-registered.
* unregistered prior to the finalize method being called as during shutdown the class loader used to load the driver may be
* unloaded prior to the driver being de-registered.
*/
public static synchronized void cleanup() {
if (driver != null) {
@@ -229,8 +230,7 @@ public final class ConnectionFactory {
*/
private static boolean h2DataFileExists() throws IOException {
final File dir = Settings.getDataDirectory();
final String name = Settings.getString(Settings.KEYS.DB_FILE_NAME);
final String fileName = String.format(name, DB_SCHEMA_VERSION);
final String fileName = Settings.getString(Settings.KEYS.DB_FILE_NAME);
final File file = new File(dir, fileName);
return file.exists();
}
@@ -278,6 +278,46 @@ public final class ConnectionFactory {
}
}
private static void updateSchema(Connection conn, String schema) throws DatabaseException {
LOGGER.debug("Updating database structure");
InputStream is;
InputStreamReader reader;
BufferedReader in = null;
String updateFile = null;
try {
updateFile = String.format(DB_STRUCTURE_UPDATE_RESOURCE, schema);
is = ConnectionFactory.class.getClassLoader().getResourceAsStream(updateFile);
reader = new InputStreamReader(is, "UTF-8");
in = new BufferedReader(reader);
final StringBuilder sb = new StringBuilder(2110);
String tmp;
while ((tmp = in.readLine()) != null) {
sb.append(tmp);
}
Statement statement = null;
try {
statement = conn.createStatement();
statement.execute(sb.toString());
} catch (SQLException ex) {
LOGGER.debug("", ex);
throw new DatabaseException("Unable to update database schema", ex);
} finally {
DBUtils.closeStatement(statement);
}
} catch (IOException ex) {
final String msg = String.format("Upgrade SQL file does not exist: %s", updateFile);
throw new DatabaseException(msg, ex);
} finally {
if (in != null) {
try {
in.close();
} catch (IOException ex) {
LOGGER.trace("", ex);
}
}
}
}
/**
* Uses the provided connection to check the specified schema version within the database.
*
@@ -288,12 +328,13 @@ public final class ConnectionFactory {
ResultSet rs = null;
CallableStatement cs = null;
try {
//TODO convert this to use DatabaseProperties
cs = conn.prepareCall("SELECT value FROM properties WHERE id = 'version'");
rs = cs.executeQuery();
if (rs.next()) {
final boolean isWrongSchema = !DB_SCHEMA_VERSION.equals(rs.getString(1));
if (isWrongSchema) {
throw new DatabaseException("Incorrect database schema; unable to continue");
if (!DB_SCHEMA_VERSION.equals(rs.getString(1))) {
LOGGER.debug("Updating from version: " + rs.getString(1));
updateSchema(conn, rs.getString(1));
}
} else {
throw new DatabaseException("Database schema is missing");

43
dependency-check-core/src/main/java/org/owasp/dependencycheck/data/nvdcve/CveDB.java
View File

@@ -19,6 +19,7 @@ package org.owasp.dependencycheck.data.nvdcve;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.sql.CallableStatement;
import java.sql.Connection;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
@@ -33,6 +34,7 @@ import java.util.Map.Entry;
import java.util.Properties;
import java.util.ResourceBundle;
import java.util.Set;
import java.util.logging.Level;
import org.owasp.dependencycheck.data.cwe.CweDB;
import org.owasp.dependencycheck.dependency.Reference;
import org.owasp.dependencycheck.dependency.Vulnerability;
@@ -766,8 +768,8 @@ public class CveDB {
DependencyVersion cpeVersion;
if (cpe.getVersion() != null && !cpe.getVersion().isEmpty()) {
String versionText;
if (cpe.getRevision() != null && !cpe.getRevision().isEmpty()) {
versionText = String.format("%s.%s", cpe.getVersion(), cpe.getRevision());
if (cpe.getUpdate() != null && !cpe.getUpdate().isEmpty()) {
versionText = String.format("%s.%s", cpe.getVersion(), cpe.getUpdate());
} else {
versionText = cpe.getVersion();
}
@@ -777,4 +779,41 @@ public class CveDB {
}
return cpeVersion;
}
/**
* Deletes unused dictionary entries from the database.
*/
public void deleteUnusedCpe() {
CallableStatement cs = null;
try {
cs = getConnection().prepareCall(statementBundle.getString("DELETE_UNUSED_DICT_CPE"));
cs.executeUpdate();
} catch (SQLException ex) {
LOGGER.error("Unable to delete CPE dictionary entries", ex);
} finally {
DBUtils.closeStatement(cs);
}
}
/**
* Merges CPE entries into the database.
*
* @param cpe the CPE identifier
* @param vendor the CPE vendor
* @param product the CPE product
*/
public void addCpe(String cpe, String vendor, String product) {
PreparedStatement ps = null;
try {
ps = getConnection().prepareCall(statementBundle.getString("ADD_DICT_CPE"));
ps.setString(1, cpe);
ps.setString(2, vendor);
ps.setString(3, product);
ps.executeUpdate();
} catch (SQLException ex) {
LOGGER.error("Unable to add CPE dictionary entry", ex);
} finally {
DBUtils.closeStatement(ps);
}
}
}

33
dependency-check-core/src/main/java/org/owasp/dependencycheck/data/nvdcve/DatabaseProperties.java
View File

@@ -24,7 +24,7 @@ import java.util.Map;
import java.util.Map.Entry;
import java.util.Properties;
import java.util.TreeMap;
import org.owasp.dependencycheck.data.update.NvdCveInfo;
import org.owasp.dependencycheck.data.update.nvd.NvdCveInfo;
import org.owasp.dependencycheck.data.update.exception.UpdateException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -41,20 +41,28 @@ public class DatabaseProperties {
*/
private static final Logger LOGGER = LoggerFactory.getLogger(DatabaseProperties.class);
/**
* Modified key word, used as a key to store information about the modified file (i.e. the containing the last 8
* days of updates)..
* Modified key word, used as a key to store information about the modified file (i.e. the containing the last 8 days of
* updates)..
*/
public static final String MODIFIED = "Modified";
/**
* The properties file key for the last updated field - used to store the last updated time of the Modified NVD CVE
* xml file.
* The properties file key for the last updated field - used to store the last updated time of the Modified NVD CVE xml file.
*/
public static final String LAST_UPDATED = "NVD CVE Modified";
/**
* Stores the last updated time for each of the NVD CVE files. These timestamps should be updated if we process the
* modified file within 7 days of the last update.
* Stores the last updated time for each of the NVD CVE files. These timestamps should be updated if we process the modified
* file within 7 days of the last update.
*/
public static final String LAST_UPDATED_BASE = "NVD CVE ";
/**
* The key for the last time the CPE data was updated.
*/
public static final String LAST_CPE_UPDATE = "LAST_CPE_UPDATE";
/**
* The key for the database schema version.
*/
public static final String VERSION = "version";
/**
* A collection of properties about the data.
*/
@@ -116,8 +124,7 @@ public class DatabaseProperties {
}
/**
* Returns the property value for the given key. If the key is not contained in the underlying properties null is
* returned.
* Returns the property value for the given key. If the key is not contained in the underlying properties null is returned.
*
* @param key the property key
* @return the value of the property
@@ -127,8 +134,8 @@ public class DatabaseProperties {
}
/**
* Returns the property value for the given key. If the key is not contained in the underlying properties the
* default value is returned.
* Returns the property value for the given key. If the key is not contained in the underlying properties the default value is
* returned.
*
* @param key the property key
* @param defaultValue the default value
@@ -148,8 +155,8 @@ public class DatabaseProperties {
}
/**
* Returns a map of the meta data from the database properties. This primarily contains timestamps of when the NVD
* CVE information was last updated.
* Returns a map of the meta data from the database properties. This primarily contains timestamps of when the NVD CVE
* information was last updated.
*
* @return a map of the database meta data
*/

86
dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/BaseUpdater.java Normal file
View File

@@ -0,0 +1,86 @@
/*
* This file is part of dependency-check-core.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
* Copyright (c) 2015 Jeremy Long. All Rights Reserved.
*/
package org.owasp.dependencycheck.data.update;
import org.owasp.dependencycheck.data.nvdcve.CveDB;
import org.owasp.dependencycheck.data.nvdcve.DatabaseException;
import org.owasp.dependencycheck.data.nvdcve.DatabaseProperties;
import org.owasp.dependencycheck.data.update.exception.UpdateException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
*
* @author Jeremy Long
*/
public abstract class BaseUpdater {
/**
* Static logger.
*/
private static final Logger LOGGER = LoggerFactory.getLogger(BaseUpdater.class);
/**
* Information about the timestamps and URLs for data that needs to be updated.
*/
private DatabaseProperties properties;
/**
* Reference to the Cve Database.
*/
private CveDB cveDB = null;
protected CveDB getCveDB() {
return cveDB;
}
protected DatabaseProperties getProperties() {
return properties;
}
/**
* Closes the CVE and CPE data stores.
*/
protected void closeDataStores() {
if (cveDB != null) {
try {
cveDB.close();
} catch (Throwable ignore) {
LOGGER.trace("Error closing the database", ignore);
}
}
}
/**
* Opens the data store.
*
* @throws UpdateException thrown if a data store cannot be opened
*/
protected final void openDataStores() throws UpdateException {
if (cveDB != null) {
return;
}
try {
cveDB = new CveDB();
cveDB.open();
} catch (DatabaseException ex) {
closeDataStores();
LOGGER.debug("Database Exception opening databases", ex);
throw new UpdateException("Error updating the database, please see the log file for more details.");
}
properties = cveDB.getDatabaseProperties();
}
}

196
dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/CpeUpdater.java Normal file
View File

@@ -0,0 +1,196 @@
/*
* This file is part of dependency-check-core.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
* Copyright (c) 2015 Jeremy Long. All Rights Reserved.
*/
package org.owasp.dependencycheck.data.update;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.Date;
import java.util.List;
import java.util.logging.Level;
import java.util.zip.GZIPInputStream;
import javax.xml.parsers.ParserConfigurationException;
import javax.xml.parsers.SAXParser;
import javax.xml.parsers.SAXParserFactory;
import org.apache.commons.io.FileUtils;
import static org.owasp.dependencycheck.data.nvdcve.DatabaseProperties.LAST_CPE_UPDATE;
import org.owasp.dependencycheck.data.update.cpe.CPEHandler;
import org.owasp.dependencycheck.data.update.cpe.Cpe;
import org.owasp.dependencycheck.data.update.exception.UpdateException;
import org.owasp.dependencycheck.utils.DateUtil;
import org.owasp.dependencycheck.utils.DownloadFailedException;
import org.owasp.dependencycheck.utils.Downloader;
import org.owasp.dependencycheck.utils.Settings;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.xml.sax.SAXException;
/**
*
* @author Jeremy Long
*/
public class CpeUpdater extends BaseUpdater implements CachedWebDataSource {
/**
* Static logger.
*/
private static final Logger LOGGER = LoggerFactory.getLogger(CpeUpdater.class);
@Override
public void update() throws UpdateException {
try {
openDataStores();
if (updateNeeded()) {
LOGGER.info("Updating the Common Platform Enumeration (CPE)");
final File xml = downloadCpe();
List<Cpe> cpes = processXML(xml);
getCveDB().deleteUnusedCpe();
for (Cpe cpe : cpes) {
getCveDB().addCpe(cpe.getValue(), cpe.getVendor(), cpe.getProduct());
}
final Date now = new Date();
getProperties().save(LAST_CPE_UPDATE, Long.toString(now.getTime()));
LOGGER.info("CPE update complete");
}
} finally {
closeDataStores();
}
}
/**
* Downloads the CPE XML file.
*
* @return the file reference to the CPE.xml file
* @throws UpdateException thrown if there is an issue downloading the XML file
*/
private File downloadCpe() throws UpdateException {
File xml;
final URL url;
try {
url = new URL(Settings.getString(Settings.KEYS.CPE_URL));
xml = File.createTempFile("cpe", ".xml", Settings.getTempDirectory());
Downloader.fetchFile(url, xml);
if (url.toExternalForm().endsWith(".xml.gz")) {
extractGzip(xml);
}
} catch (MalformedURLException ex) {
throw new UpdateException("Invalid CPE URL", ex);
} catch (DownloadFailedException ex) {
throw new UpdateException("Unable to download CPE XML file", ex);
} catch (IOException ex) {
throw new UpdateException("Unable to create temporary file to download CPE", ex);
}
return xml;
}
/**
* Parses the CPE XML file to return a list of CPE entries.
*
* @param xml the CPE data file
* @return the list of CPE entries
* @throws UpdateException thrown if there is an issue with parsing the XML file
*/
private List<Cpe> processXML(final File xml) throws UpdateException {
try {
final SAXParserFactory factory = SAXParserFactory.newInstance();
final SAXParser saxParser = factory.newSAXParser();
CPEHandler handler = new CPEHandler();
saxParser.parse(xml, handler);
return handler.getData();
} catch (ParserConfigurationException ex) {
throw new UpdateException("Unable to parse CPE XML file due to SAX Parser Issue", ex);
} catch (SAXException ex) {
throw new UpdateException("Unable to parse CPE XML file due to SAX Parser Exception", ex);
} catch (IOException ex) {
throw new UpdateException("Unable to parse CPE XML file due to IO Failure", ex);
}
}
/**
* Checks to find the last time the CPE data was refreshed and if it needs to be updated.
*
* @return true if the CPE data should be refreshed
*/
private boolean updateNeeded() {
final Date now = new Date();
final int days = Settings.getInt(Settings.KEYS.CVE_MODIFIED_VALID_FOR_DAYS, 30);
long timestamp = 0;
String ts = getProperties().getProperty(LAST_CPE_UPDATE);
if (ts != null && ts.matches("^[0-9]+$")) {
timestamp = Long.parseLong(ts);
}
return !DateUtil.withinDateRange(timestamp, now.getTime(), days);
}
/**
* Extracts the file contained in a gzip archive. The extracted file is placed in the exact same path as the file specified.
*
* @param file the archive file
* @throws FileNotFoundException thrown if the file does not exist
* @throws IOException thrown if there is an error extracting the file.
*/
private void extractGzip(File file) throws FileNotFoundException, IOException {
//TODO - move this to a util class as it is duplicative of (copy of) code in the DownloadTask
final String originalPath = file.getPath();
final File gzip = new File(originalPath + ".gz");
if (gzip.isFile() && !gzip.delete()) {
gzip.deleteOnExit();
}
if (!file.renameTo(gzip)) {
throw new IOException("Unable to rename '" + file.getPath() + "'");
}
final File newfile = new File(originalPath);
final byte[] buffer = new byte[4096];
GZIPInputStream cin = null;
FileOutputStream out = null;
try {
cin = new GZIPInputStream(new FileInputStream(gzip));
out = new FileOutputStream(newfile);
int len;
while ((len = cin.read(buffer)) > 0) {
out.write(buffer, 0, len);
}
} finally {
if (cin != null) {
try {
cin.close();
} catch (IOException ex) {
LOGGER.trace("ignore", ex);
}
}
if (out != null) {
try {
out.close();
} catch (IOException ex) {
LOGGER.trace("ignore", ex);
}
}
if (gzip.isFile()) {
FileUtils.deleteQuietly(gzip);
}
}
}
}

228
dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/NvdCveUpdater.java
View File

@@ -18,23 +18,44 @@
package org.owasp.dependencycheck.data.update;
import java.net.MalformedURLException;
import java.util.Calendar;
import java.util.Date;
import java.util.HashSet;
import java.util.Set;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import org.owasp.dependencycheck.data.nvdcve.DatabaseProperties;
import static org.owasp.dependencycheck.data.nvdcve.DatabaseProperties.MODIFIED;
import org.owasp.dependencycheck.data.update.exception.InvalidDataException;
import org.owasp.dependencycheck.data.update.exception.UpdateException;
import org.owasp.dependencycheck.data.update.nvd.DownloadTask;
import org.owasp.dependencycheck.data.update.nvd.NvdCveInfo;
import org.owasp.dependencycheck.data.update.nvd.ProcessTask;
import org.owasp.dependencycheck.data.update.nvd.UpdateableNvdCve;
import org.owasp.dependencycheck.utils.DateUtil;
import org.owasp.dependencycheck.utils.DownloadFailedException;
import org.owasp.dependencycheck.utils.InvalidSettingException;
import org.owasp.dependencycheck.utils.Settings;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* Class responsible for updating the NVD CVE and CPE data stores.
* Class responsible for updating the NVD CVE data.
*
* @author Jeremy Long
*/
public class NvdCveUpdater implements CachedWebDataSource {
public class NvdCveUpdater extends BaseUpdater implements CachedWebDataSource {
/**
* The logger
*/
private static final Logger LOGGER = LoggerFactory.getLogger(NvdCveUpdater.class);
/**
* The max thread pool size to use when downloading files.
*/
public static final int MAX_THREAD_POOL_SIZE = Settings.getInt(Settings.KEYS.MAX_DOWNLOAD_THREAD_POOL_SIZE, 3);
/**
* <p>
@@ -45,9 +66,10 @@ public class NvdCveUpdater implements CachedWebDataSource {
@Override
public void update() throws UpdateException {
try {
final StandardUpdate task = new StandardUpdate();
if (task.isUpdateNeeded()) {
task.update();
openDataStores();
UpdateableNvdCve updateable = getUpdatesNeeded();
if (updateable.isUpdateNeeded()) {
performUpdate(updateable);
}
} catch (MalformedURLException ex) {
LOGGER.warn(
@@ -61,6 +83,202 @@ public class NvdCveUpdater implements CachedWebDataSource {
"If you are behind a proxy you may need to configure dependency-check to use the proxy.");
}
LOGGER.debug("", ex);
} finally {
closeDataStores();
}
}
/**
* Downloads the latest NVD CVE XML file from the web and imports it into the current CVE Database.
*
* @param updateable a collection of NVD CVE data file references that need to be downloaded and processed to update the
* database
* @throws UpdateException is thrown if there is an error updating the database
*/
public void performUpdate(UpdateableNvdCve updateable) throws UpdateException {
int maxUpdates = 0;
try {
for (NvdCveInfo cve : updateable) {
if (cve.getNeedsUpdate()) {
maxUpdates += 1;
}
}
if (maxUpdates <= 0) {
return;
}
if (maxUpdates > 3) {
LOGGER.info(
"NVD CVE requires several updates; this could take a couple of minutes.");
}
if (maxUpdates > 0) {
openDataStores();
}
final int poolSize = (MAX_THREAD_POOL_SIZE < maxUpdates) ? MAX_THREAD_POOL_SIZE : maxUpdates;
final ExecutorService downloadExecutors = Executors.newFixedThreadPool(poolSize);
final ExecutorService processExecutor = Executors.newSingleThreadExecutor();
final Set<Future<Future<ProcessTask>>> downloadFutures = new HashSet<Future<Future<ProcessTask>>>(maxUpdates);
for (NvdCveInfo cve : updateable) {
if (cve.getNeedsUpdate()) {
final DownloadTask call = new DownloadTask(cve, processExecutor, getCveDB(), Settings.getInstance());
downloadFutures.add(downloadExecutors.submit(call));
}
}
downloadExecutors.shutdown();
//next, move the future future processTasks to just future processTasks
final Set<Future<ProcessTask>> processFutures = new HashSet<Future<ProcessTask>>(maxUpdates);
for (Future<Future<ProcessTask>> future : downloadFutures) {
Future<ProcessTask> task = null;
try {
task = future.get();
} catch (InterruptedException ex) {
downloadExecutors.shutdownNow();
processExecutor.shutdownNow();
LOGGER.debug("Thread was interrupted during download", ex);
throw new UpdateException("The download was interrupted", ex);
} catch (ExecutionException ex) {
downloadExecutors.shutdownNow();
processExecutor.shutdownNow();
LOGGER.debug("Thread was interrupted during download execution", ex);
throw new UpdateException("The execution of the download was interrupted", ex);
}
if (task == null) {
downloadExecutors.shutdownNow();
processExecutor.shutdownNow();
LOGGER.debug("Thread was interrupted during download");
throw new UpdateException("The download was interrupted; unable to complete the update");
} else {
processFutures.add(task);
}
}
for (Future<ProcessTask> future : processFutures) {
try {
final ProcessTask task = future.get();
if (task.getException() != null) {
throw task.getException();
}
} catch (InterruptedException ex) {
processExecutor.shutdownNow();
LOGGER.debug("Thread was interrupted during processing", ex);
throw new UpdateException(ex);
} catch (ExecutionException ex) {
processExecutor.shutdownNow();
LOGGER.debug("Execution Exception during process", ex);
throw new UpdateException(ex);
} finally {
processExecutor.shutdown();
}
}
if (maxUpdates >= 1) { //ensure the modified file date gets written (we may not have actually updated it)
getProperties().save(updateable.get(MODIFIED));
LOGGER.info("Begin database maintenance.");
getCveDB().cleanupDatabase();
LOGGER.info("End database maintenance.");
}
} finally {
closeDataStores();
}
}
/**
* Determines if the index needs to be updated. This is done by fetching the NVD CVE meta data and checking the last update
* date. If the data needs to be refreshed this method will return the NvdCveUrl for the files that need to be updated.
*
* @return the collection of files that need to be updated
* @throws MalformedURLException is thrown if the URL for the NVD CVE Meta data is incorrect
* @throws DownloadFailedException is thrown if there is an error. downloading the NVD CVE download data file
* @throws UpdateException Is thrown if there is an issue with the last updated properties file
*/
protected final UpdateableNvdCve getUpdatesNeeded() throws MalformedURLException, DownloadFailedException, UpdateException {
UpdateableNvdCve updates = null;
try {
updates = retrieveCurrentTimestampsFromWeb();
} catch (InvalidDataException ex) {
final String msg = "Unable to retrieve valid timestamp from nvd cve downloads page";
LOGGER.debug(msg, ex);
throw new DownloadFailedException(msg, ex);
} catch (InvalidSettingException ex) {
LOGGER.debug("Invalid setting found when retrieving timestamps", ex);
throw new DownloadFailedException("Invalid settings", ex);
}
if (updates == null) {
throw new DownloadFailedException("Unable to retrieve the timestamps of the currently published NVD CVE data");
}
if (!getProperties().isEmpty()) {
try {
final long lastUpdated = Long.parseLong(getProperties().getProperty(DatabaseProperties.LAST_UPDATED, "0"));
final Date now = new Date();
final int days = Settings.getInt(Settings.KEYS.CVE_MODIFIED_VALID_FOR_DAYS, 7);
if (lastUpdated == updates.getTimeStamp(MODIFIED)) {
updates.clear(); //we don't need to update anything.
} else if (DateUtil.withinDateRange(lastUpdated, now.getTime(), days)) {
for (NvdCveInfo entry : updates) {
if (MODIFIED.equals(entry.getId())) {
entry.setNeedsUpdate(true);
} else {
entry.setNeedsUpdate(false);
}
}
} else { //we figure out which of the several XML files need to be downloaded.
for (NvdCveInfo entry : updates) {
if (MODIFIED.equals(entry.getId())) {
entry.setNeedsUpdate(true);
} else {
long currentTimestamp = 0;
try {
currentTimestamp = Long.parseLong(getProperties().getProperty(DatabaseProperties.LAST_UPDATED_BASE + entry.getId(), "0"));
} catch (NumberFormatException ex) {
LOGGER.debug("Error parsing '{}' '{}' from nvdcve.lastupdated",
DatabaseProperties.LAST_UPDATED_BASE, entry.getId(), ex);
}
if (currentTimestamp == entry.getTimestamp()) {
entry.setNeedsUpdate(false);
}
}
}
}
} catch (NumberFormatException ex) {
LOGGER.warn("An invalid schema version or timestamp exists in the data.properties file.");
LOGGER.debug("", ex);
}
}
return updates;
}
/**
* Retrieves the timestamps from the NVD CVE meta data file.
*
* @return the timestamp from the currently published nvdcve downloads page
* @throws MalformedURLException thrown if the URL for the NVD CCE Meta data is incorrect.
* @throws DownloadFailedException thrown if there is an error downloading the nvd cve meta data file
* @throws InvalidDataException thrown if there is an exception parsing the timestamps
* @throws InvalidSettingException thrown if the settings are invalid
*/
private UpdateableNvdCve retrieveCurrentTimestampsFromWeb()
throws MalformedURLException, DownloadFailedException, InvalidDataException, InvalidSettingException {
final UpdateableNvdCve updates = new UpdateableNvdCve();
updates.add(MODIFIED, Settings.getString(Settings.KEYS.CVE_MODIFIED_20_URL),
Settings.getString(Settings.KEYS.CVE_MODIFIED_12_URL),
false);
final int start = Settings.getInt(Settings.KEYS.CVE_START_YEAR);
final int end = Calendar.getInstance().get(Calendar.YEAR);
final String baseUrl20 = Settings.getString(Settings.KEYS.CVE_SCHEMA_2_0);
final String baseUrl12 = Settings.getString(Settings.KEYS.CVE_SCHEMA_1_2);
for (int i = start; i <= end; i++) {
updates.add(Integer.toString(i), String.format(baseUrl20, i),
String.format(baseUrl12, i),
true);
}
return updates;
}
}

318
dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/StandardUpdate.java
View File

@@ -1,318 +0,0 @@
/*
* This file is part of dependency-check-core.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
* Copyright (c) 2012 Jeremy Long. All Rights Reserved.
*/
package org.owasp.dependencycheck.data.update;
import java.net.MalformedURLException;
import java.util.Calendar;
import java.util.Date;
import java.util.HashSet;
import java.util.Set;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import org.owasp.dependencycheck.data.nvdcve.CveDB;
import org.owasp.dependencycheck.data.nvdcve.DatabaseException;
import org.owasp.dependencycheck.data.nvdcve.DatabaseProperties;
import static org.owasp.dependencycheck.data.nvdcve.DatabaseProperties.MODIFIED;
import org.owasp.dependencycheck.data.update.exception.InvalidDataException;
import org.owasp.dependencycheck.data.update.exception.UpdateException;
import org.owasp.dependencycheck.data.update.task.DownloadTask;
import org.owasp.dependencycheck.data.update.task.ProcessTask;
import org.owasp.dependencycheck.utils.DateUtil;
import org.owasp.dependencycheck.utils.DownloadFailedException;
import org.owasp.dependencycheck.utils.InvalidSettingException;
import org.owasp.dependencycheck.utils.Settings;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* Class responsible for updating the NVDCVE data store.
*
* @author Jeremy Long
*/
public class StandardUpdate {
/**
* Static logger.
*/
private static final Logger LOGGER = LoggerFactory.getLogger(StandardUpdate.class);
/**
* The max thread pool size to use when downloading files.
*/
public static final int MAX_THREAD_POOL_SIZE = Settings.getInt(Settings.KEYS.MAX_DOWNLOAD_THREAD_POOL_SIZE, 3);
/**
* Information about the timestamps and URLs for data that needs to be updated.
*/
private DatabaseProperties properties;
/**
* A collection of updateable NVD CVE items.
*/
private UpdateableNvdCve updateable;
/**
* Reference to the Cve Database.
*/
private CveDB cveDB = null;
/**
* Gets whether or not an update is needed.
*
* @return true or false depending on whether an update is needed
*/
public boolean isUpdateNeeded() {
return updateable.isUpdateNeeded();
}
/**
* Constructs a new Standard Update Task.
*
* @throws MalformedURLException thrown if a configured URL is malformed
* @throws DownloadFailedException thrown if a timestamp cannot be checked on a configured URL
* @throws UpdateException thrown if there is an exception generating the update task
*/
public StandardUpdate() throws MalformedURLException, DownloadFailedException, UpdateException {
openDataStores();
properties = cveDB.getDatabaseProperties();
updateable = updatesNeeded();
}
/**
* <p>
* Downloads the latest NVD CVE XML file from the web and imports it into the current CVE Database.</p>
*
* @throws UpdateException is thrown if there is an error updating the database
*/
public void update() throws UpdateException {
int maxUpdates = 0;
try {
for (NvdCveInfo cve : updateable) {
if (cve.getNeedsUpdate()) {
maxUpdates += 1;
}
}
if (maxUpdates <= 0) {
return;
}
if (maxUpdates > 3) {
LOGGER.info(
"NVD CVE requires several updates; this could take a couple of minutes.");
}
if (maxUpdates > 0) {
openDataStores();
}
final int poolSize = (MAX_THREAD_POOL_SIZE < maxUpdates) ? MAX_THREAD_POOL_SIZE : maxUpdates;
final ExecutorService downloadExecutors = Executors.newFixedThreadPool(poolSize);
final ExecutorService processExecutor = Executors.newSingleThreadExecutor();
final Set<Future<Future<ProcessTask>>> downloadFutures = new HashSet<Future<Future<ProcessTask>>>(maxUpdates);
for (NvdCveInfo cve : updateable) {
if (cve.getNeedsUpdate()) {
final DownloadTask call = new DownloadTask(cve, processExecutor, cveDB, Settings.getInstance());
downloadFutures.add(downloadExecutors.submit(call));
}
}
downloadExecutors.shutdown();
//next, move the future future processTasks to just future processTasks
final Set<Future<ProcessTask>> processFutures = new HashSet<Future<ProcessTask>>(maxUpdates);
for (Future<Future<ProcessTask>> future : downloadFutures) {
Future<ProcessTask> task = null;
try {
task = future.get();
} catch (InterruptedException ex) {
downloadExecutors.shutdownNow();
processExecutor.shutdownNow();
LOGGER.debug("Thread was interrupted during download", ex);
throw new UpdateException("The download was interrupted", ex);
} catch (ExecutionException ex) {
downloadExecutors.shutdownNow();
processExecutor.shutdownNow();
LOGGER.debug("Thread was interrupted during download execution", ex);
throw new UpdateException("The execution of the download was interrupted", ex);
}
if (task == null) {
downloadExecutors.shutdownNow();
processExecutor.shutdownNow();
LOGGER.debug("Thread was interrupted during download");
throw new UpdateException("The download was interrupted; unable to complete the update");
} else {
processFutures.add(task);
}
}
for (Future<ProcessTask> future : processFutures) {
try {
final ProcessTask task = future.get();
if (task.getException() != null) {
throw task.getException();
}
} catch (InterruptedException ex) {
processExecutor.shutdownNow();
LOGGER.debug("Thread was interrupted during processing", ex);
throw new UpdateException(ex);
} catch (ExecutionException ex) {
processExecutor.shutdownNow();
LOGGER.debug("Execution Exception during process", ex);
throw new UpdateException(ex);
} finally {
processExecutor.shutdown();
}
}
if (maxUpdates >= 1) { //ensure the modified file date gets written (we may not have actually updated it)
properties.save(updateable.get(MODIFIED));
LOGGER.info("Begin database maintenance.");
cveDB.cleanupDatabase();
LOGGER.info("End database maintenance.");
}
} finally {
closeDataStores();
}
}
/**
* Determines if the index needs to be updated. This is done by fetching the NVD CVE meta data and checking the last update
* date. If the data needs to be refreshed this method will return the NvdCveUrl for the files that need to be updated.
*
* @return the collection of files that need to be updated
* @throws MalformedURLException is thrown if the URL for the NVD CVE Meta data is incorrect
* @throws DownloadFailedException is thrown if there is an error. downloading the NVD CVE download data file
* @throws UpdateException Is thrown if there is an issue with the last updated properties file
*/
protected final UpdateableNvdCve updatesNeeded() throws MalformedURLException, DownloadFailedException, UpdateException {
UpdateableNvdCve updates = null;
try {
updates = retrieveCurrentTimestampsFromWeb();
} catch (InvalidDataException ex) {
final String msg = "Unable to retrieve valid timestamp from nvd cve downloads page";
LOGGER.debug(msg, ex);
throw new DownloadFailedException(msg, ex);
} catch (InvalidSettingException ex) {
LOGGER.debug("Invalid setting found when retrieving timestamps", ex);
throw new DownloadFailedException("Invalid settings", ex);
}
if (updates == null) {
throw new DownloadFailedException("Unable to retrieve the timestamps of the currently published NVD CVE data");
}
if (!properties.isEmpty()) {
try {
final long lastUpdated = Long.parseLong(properties.getProperty(DatabaseProperties.LAST_UPDATED, "0"));
final Date now = new Date();
final int days = Settings.getInt(Settings.KEYS.CVE_MODIFIED_VALID_FOR_DAYS, 7);
if (lastUpdated == updates.getTimeStamp(MODIFIED)) {
updates.clear(); //we don't need to update anything.
} else if (DateUtil.withinDateRange(lastUpdated, now.getTime(), days)) {
for (NvdCveInfo entry : updates) {
if (MODIFIED.equals(entry.getId())) {
entry.setNeedsUpdate(true);
} else {
entry.setNeedsUpdate(false);
}
}
} else { //we figure out which of the several XML files need to be downloaded.
for (NvdCveInfo entry : updates) {
if (MODIFIED.equals(entry.getId())) {
entry.setNeedsUpdate(true);
} else {
long currentTimestamp = 0;
try {
currentTimestamp = Long.parseLong(properties.getProperty(DatabaseProperties.LAST_UPDATED_BASE + entry.getId(), "0"));
} catch (NumberFormatException ex) {
LOGGER.debug("Error parsing '{}' '{}' from nvdcve.lastupdated",
DatabaseProperties.LAST_UPDATED_BASE, entry.getId(), ex);
}
if (currentTimestamp == entry.getTimestamp()) {
entry.setNeedsUpdate(false);
}
}
}
}
} catch (NumberFormatException ex) {
LOGGER.warn("An invalid schema version or timestamp exists in the data.properties file.");
LOGGER.debug("", ex);
}
}
return updates;
}
/**
* Retrieves the timestamps from the NVD CVE meta data file.
*
* @return the timestamp from the currently published nvdcve downloads page
* @throws MalformedURLException thrown if the URL for the NVD CCE Meta data is incorrect.
* @throws DownloadFailedException thrown if there is an error downloading the nvd cve meta data file
* @throws InvalidDataException thrown if there is an exception parsing the timestamps
* @throws InvalidSettingException thrown if the settings are invalid
*/
private UpdateableNvdCve retrieveCurrentTimestampsFromWeb()
throws MalformedURLException, DownloadFailedException, InvalidDataException, InvalidSettingException {
final UpdateableNvdCve updates = new UpdateableNvdCve();
updates.add(MODIFIED, Settings.getString(Settings.KEYS.CVE_MODIFIED_20_URL),
Settings.getString(Settings.KEYS.CVE_MODIFIED_12_URL),
false);
final int start = Settings.getInt(Settings.KEYS.CVE_START_YEAR);
final int end = Calendar.getInstance().get(Calendar.YEAR);
final String baseUrl20 = Settings.getString(Settings.KEYS.CVE_SCHEMA_2_0);
final String baseUrl12 = Settings.getString(Settings.KEYS.CVE_SCHEMA_1_2);
for (int i = start; i <= end; i++) {
updates.add(Integer.toString(i), String.format(baseUrl20, i),
String.format(baseUrl12, i),
true);
}
return updates;
}
/**
* Closes the CVE and CPE data stores.
*/
protected void closeDataStores() {
if (cveDB != null) {
try {
cveDB.close();
} catch (Throwable ignore) {
LOGGER.trace("Error closing the cveDB", ignore);
}
}
}
/**
* Opens the CVE and CPE data stores.
*
* @throws UpdateException thrown if a data store cannot be opened
*/
protected final void openDataStores() throws UpdateException {
if (cveDB != null) {
return;
}
try {
cveDB = new CveDB();
cveDB.open();
} catch (DatabaseException ex) {
closeDataStores();
LOGGER.debug("Database Exception opening databases", ex);
throw new UpdateException("Error updating the CPE/CVE data, please see the log file for more details.");
}
}
}

1
dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/UpdateService.java
View File

@@ -19,6 +19,7 @@ package org.owasp.dependencycheck.data.update;
import java.util.Iterator;
import java.util.ServiceLoader;
import org.owasp.dependencycheck.data.update.CachedWebDataSource;
/**
* The CachedWebDataSource Service Loader. This class loads all services that implement

362
dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/cpe/CPEHandler.java Normal file
View File

@@ -0,0 +1,362 @@
/*
* This file is part of dependency-check-core.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
* Copyright (c) 2015 Jeremy Long. All Rights Reserved.
*/
package org.owasp.dependencycheck.data.update.cpe;
import java.io.UnsupportedEncodingException;
import java.net.URLDecoder;
import java.util.ArrayList;
import java.util.List;
import org.owasp.dependencycheck.data.update.NvdCveUpdater;
import org.owasp.dependencycheck.data.update.exception.InvalidDataException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.xml.sax.Attributes;
import org.xml.sax.SAXException;
import org.xml.sax.helpers.DefaultHandler;
/**
* A SAX Handler that will parse the CPE XML and load it into the databse.
*
* @author Jeremy Long
*/
public class CPEHandler extends DefaultHandler {
/**
* The current CPE schema.
*/
private static final String CURRENT_SCHEMA_VERSION = "2.3";
/**
* The text content of the node being processed. This can be used during the end element event.
*/
StringBuilder nodeText = null;
/**
* A reference to the current element.
*/
Element current = new Element();
/**
* The logger.
*/
private static final Logger LOGGER = LoggerFactory.getLogger(NvdCveUpdater.class);
/**
* The list of CPE values.
*/
List<Cpe> data = new ArrayList<Cpe>();
/**
* Returns the list of CPE values.
*
* @return the list of CPE values
*/
public List<Cpe> getData() {
return data;
}
/**
* Handles the start element event
*
* @param uri the elements uri
* @param localName the local name
* @param qName the qualified name
* @param attributes the attributes
* @throws SAXException thrown if there is an exception processing the element
*/
@Override
public void startElement(String uri, String localName, String qName, Attributes attributes) throws SAXException {
nodeText = null;
current.setNode(qName);
if (current.isCpeItemNode()) {
String temp = attributes.getValue("deprecated");
String value = attributes.getValue("name");
boolean delete = (temp != null && temp.equalsIgnoreCase("true"));
if (!delete && value.startsWith("cpe:/a:") && value.length() > 7) {
try {
Cpe cpe = new Cpe(value);
data.add(cpe);
} catch (UnsupportedEncodingException ex) {
LOGGER.debug("Unable to parse the CPE", ex);
} catch (InvalidDataException ex) {
LOGGER.debug("CPE is not the correct format", ex);
}
}
} else if (current.isSchemaVersionNode()) {
nodeText = new StringBuilder(3);
}
// } else if (current.isTitleNode()) {
// //do nothing
// } else if (current.isMetaNode()) {
// //do nothing
// } else if (current.isTimestampNode()) {
// //do nothing
// } else if (current.isCpeListNode()) {
// //do nothing
// } else if (current.isNotesNode()) {
// //do nothing
// } else if (current.isNoteNode()) {
// //do nothing
// } else if (current.isCheckNode()) {
// //do nothing
// } else if (current.isGeneratorNode()) {
// //do nothing
// } else if (current.isProductNameNode()) {
// //do nothing
// } else if (current.isProductVersionNode()) {
// //do nothing
}
/**
* Reads the characters in the current node.
*
* @param ch the char array
* @param start the start position of the data read
* @param length the length of the data read
* @throws SAXException thrown if there is an exception processing the characters
*/
@Override
public void characters(char[] ch, int start, int length) throws SAXException {
if (nodeText != null) {
nodeText.append(ch, start, length);
}
}
/**
* Handles the end element event. Stores the CPE data in the Cve Database if the cpe item node is ending.
*
* @param uri the element's uri
* @param localName the local name
* @param qName the qualified name
* @throws SAXException thrown if there is an exception processing the element
*/
@Override
public void endElement(String uri, String localName, String qName) throws SAXException {
current.setNode(qName);
if (current.isSchemaVersionNode() && !CURRENT_SCHEMA_VERSION.equals(nodeText.toString())) {
throw new SAXException("ERROR: Unexpecgted CPE Schema Version, expected: "
+ CURRENT_SCHEMA_VERSION + ", file is: " + nodeText);
}
// } else if (current.isCpeItemNode()) {
// //do nothing
// } else if (current.isTitleNode()) {
// //do nothing
// } else if (current.isCpeListNode()) {
// //do nothing
// } else if (current.isMetaNode()) {
// //do nothing
// } else if (current.isNotesNode()) {
// //do nothing
// } else if (current.isNoteNode()) {
// //do nothing
// } else if (current.isCheckNode()) {
// //do nothing
// } else if (current.isGeneratorNode()) {
// //do nothing
// } else if (current.isProductNameNode()) {
// //do nothing
// } else if (current.isProductVersionNode()) {
// //do nothing
// else if (current.isTimestampNode()) {
// //do nothing
// } else {
// throw new SAXException("ERROR STATE: Unexpected qName '" + qName + "'");
// }
}
// <editor-fold defaultstate="collapsed" desc="The Element Class that maintains state information about the current node">
/**
* A simple class to maintain information about the current element while parsing the CPE XML.
*/
protected class Element {
/**
* A node type in the CPE Schema 2.2
*/
public static final String CPE_LIST = "cpe-list";
/**
* A node type in the CPE Schema 2.2
*/
public static final String CPE_ITEM = "cpe-item";
/**
* A node type in the CPE Schema 2.2
*/
public static final String TITLE = "title";
/**
* A node type in the CPE Schema 2.2
*/
public static final String NOTES = "notes";
/**
* A node type in the CPE Schema 2.2
*/
public static final String NOTE = "note";
/**
* A node type in the CPE Schema 2.2
*/
public static final String CHECK = "check";
/**
* A node type in the CPE Schema 2.2
*/
public static final String META = "meta:item-metadata";
/**
* A node type in the CPE Schema 2.2
*/
public static final String GENERATOR = "generator";
/**
* A node type in the CPE Schema 2.2
*/
public static final String PRODUCT_NAME = "product_name";
/**
* A node type in the CPE Schema 2.2
*/
public static final String PRODUCT_VERSION = "product_version";
/**
* A node type in the CPE Schema 2.2
*/
public static final String SCHEMA_VERSION = "schema_version";
/**
* A node type in the CPE Schema 2.2
*/
public static final String TIMESTAMP = "timestamp";
private String node = null;
/**
* Gets the value of node
*
* @return the value of node
*/
public String getNode() {
return this.node;
}
/**
* Sets the value of node
*
* @param node new value of node
*/
public void setNode(String node) {
this.node = node;
}
/**
* Checks if the handler is at the CPE_LIST node
*
* @return true or false
*/
public boolean isCpeListNode() {
return CPE_LIST.equals(node);
}
/**
* Checks if the handler is at the CPE_ITEM node
*
* @return true or false
*/
public boolean isCpeItemNode() {
return CPE_ITEM.equals(node);
}
/**
* Checks if the handler is at the TITLE node
*
* @return true or false
*/
public boolean isTitleNode() {
return TITLE.equals(node);
}
/**
* Checks if the handler is at the NOTES node
*
* @return true or false
*/
public boolean isNotesNode() {
return NOTES.equals(node);
}
/**
* Checks if the handler is at the NOTE node
*
* @return true or false
*/
public boolean isNoteNode() {
return NOTE.equals(node);
}
/**
* Checks if the handler is at the CHECK node
*
* @return true or false
*/
public boolean isCheckNode() {
return CHECK.equals(node);
}
/**
* Checks if the handler is at the META node
*
* @return true or false
*/
public boolean isMetaNode() {
return META.equals(node);
}
/**
* Checks if the handler is at the GENERATOR node
*
* @return true or false
*/
public boolean isGeneratorNode() {
return GENERATOR.equals(node);
}
/**
* Checks if the handler is at the PRODUCT_NAME node
*
* @return true or false
*/
public boolean isProductNameNode() {
return PRODUCT_NAME.equals(node);
}
/**
* Checks if the handler is at the PRODUCT_VERSION node
*
* @return true or false
*/
public boolean isProductVersionNode() {
return PRODUCT_VERSION.equals(node);
}
/**
* Checks if the handler is at the SCHEMA_VERSION node
*
* @return true or false
*/
public boolean isSchemaVersionNode() {
return SCHEMA_VERSION.equals(node);
}
/**
* Checks if the handler is at the TIMESTAMP node
*
* @return true or false
*/
public boolean isTimestampNode() {
return TIMESTAMP.equals(node);
}
}
// </editor-fold>
}

114
dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/cpe/Cpe.java Normal file
View File

@@ -0,0 +1,114 @@
/*
* This file is part of dependency-check-core.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
* Copyright (c) 2015 Jeremy Long. All Rights Reserved.
*/
package org.owasp.dependencycheck.data.update.cpe;
import java.io.UnsupportedEncodingException;
import java.net.URLDecoder;
import org.owasp.dependencycheck.data.update.exception.InvalidDataException;
/**
*
* @author Jeremy Long
*/
public class Cpe {
public Cpe(String value) throws UnsupportedEncodingException, InvalidDataException {
this.value = value;
final String[] data = value.substring(7).split(":");
if (data.length >= 2) {
vendor = URLDecoder.decode(data[0].replace("+", "%2B"), "UTF-8");
product = URLDecoder.decode(data[1].replace("+", "%2B"), "UTF-8");
} else {
throw new InvalidDataException(String.format("CPE has an invalid format: %s", value));
}
}
/**
* The CPE identifier string (cpe:/a:vendor:product:version).
*/
private String value;
/**
* Get the value of value.
*
* @return the value of value
*/
public String getValue() {
return value;
}
/**
* Set the value of value.
*
* @param value new value of value
*/
public void setValue(String value) {
this.value = value;
}
/**
* The vendor portion of the identifier.
*/
private String vendor;
/**
* Get the value of vendor
*
* @return the value of vendor
*/
public String getVendor() {
return vendor;
}
/**
* Set the value of vendor
*
* @param vendor new value of vendor
*/
public void setVendor(String vendor) {
this.vendor = vendor;
}
/**
* The product portion of the identifier.
*/
private String product;
/**
* Get the value of product
*
* @return the value of product
*/
public String getProduct() {
return product;
}
/**
* Set the value of product
*
* @param product new value of product
*/
public void setProduct(String product) {
this.product = product;
}
@Override
public String toString() {
return value;
}
}

10
dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/task/DownloadTask.java → dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/nvd/DownloadTask.java
View File

@@ -15,7 +15,7 @@
*
* Copyright (c) 2013 Jeremy Long. All Rights Reserved.
*/
package org.owasp.dependencycheck.data.update.task;
package org.owasp.dependencycheck.data.update.nvd;
import java.io.File;
import java.io.FileInputStream;
@@ -29,7 +29,6 @@ import java.util.concurrent.Future;
import java.util.zip.GZIPInputStream;
import org.apache.commons.io.FileUtils;
import org.owasp.dependencycheck.data.nvdcve.CveDB;
import org.owasp.dependencycheck.data.update.NvdCveInfo;
import org.owasp.dependencycheck.data.update.exception.UpdateException;
import org.owasp.dependencycheck.utils.DownloadFailedException;
import org.owasp.dependencycheck.utils.Downloader;
@@ -55,8 +54,8 @@ public class DownloadTask implements Callable<Future<ProcessTask>> {
* @param nvdCveInfo the NVD CVE info
* @param processor the processor service to submit the downloaded files to
* @param cveDB the CVE DB to use to store the vulnerability data
* @param settings a reference to the global settings object; this is necessary so that when the thread is started
* the dependencies have a correct reference to the global settings.
* @param settings a reference to the global settings object; this is necessary so that when the thread is started the
* dependencies have a correct reference to the global settings.
* @throws UpdateException thrown if temporary files could not be created
*/
public DownloadTask(NvdCveInfo nvdCveInfo, ExecutorService processor, CveDB cveDB, Settings settings) throws UpdateException {
@@ -248,8 +247,7 @@ public class DownloadTask implements Callable<Future<ProcessTask>> {
}
/**
* Extracts the file contained in a gzip archive. The extracted file is placed in the exact same path as the file
* specified.
* Extracts the file contained in a gzip archive. The extracted file is placed in the exact same path as the file specified.
*
* @param file the archive file
* @throws FileNotFoundException thrown if the file does not exist

2
dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/xml/NvdCve12Handler.java → dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/nvd/NvdCve12Handler.java
View File

@@ -15,7 +15,7 @@
*
* Copyright (c) 2012 Jeremy Long. All Rights Reserved.
*/
package org.owasp.dependencycheck.data.update.xml;
package org.owasp.dependencycheck.data.update.nvd;
import java.util.ArrayList;
import java.util.HashMap;

2
dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/xml/NvdCve20Handler.java → dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/nvd/NvdCve20Handler.java
View File

@@ -15,7 +15,7 @@
*
* Copyright (c) 2012 Jeremy Long. All Rights Reserved.
*/
package org.owasp.dependencycheck.data.update.xml;
package org.owasp.dependencycheck.data.update.nvd;
import java.io.IOException;
import java.util.List;

2
dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/NvdCveInfo.java → dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/nvd/NvdCveInfo.java
View File

@@ -15,7 +15,7 @@
*
* Copyright (c) 2013 Jeremy Long. All Rights Reserved.
*/
package org.owasp.dependencycheck.data.update;
package org.owasp.dependencycheck.data.update.nvd;
/**
* A pojo that contains the Url and timestamp of the current NvdCve XML files.

6
dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/task/ProcessTask.java → dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/nvd/ProcessTask.java
View File

@@ -15,7 +15,7 @@
*
* Copyright (c) 2013 Jeremy Long. All Rights Reserved.
*/
package org.owasp.dependencycheck.data.update.task;
package org.owasp.dependencycheck.data.update.nvd;
import java.io.File;
import java.io.FileNotFoundException;
@@ -31,8 +31,8 @@ import org.owasp.dependencycheck.data.nvdcve.CveDB;
import org.owasp.dependencycheck.data.nvdcve.DatabaseException;
import org.owasp.dependencycheck.data.nvdcve.DatabaseProperties;
import org.owasp.dependencycheck.data.update.exception.UpdateException;
import org.owasp.dependencycheck.data.update.xml.NvdCve12Handler;
import org.owasp.dependencycheck.data.update.xml.NvdCve20Handler;
import org.owasp.dependencycheck.data.update.nvd.NvdCve12Handler;
import org.owasp.dependencycheck.data.update.nvd.NvdCve20Handler;
import org.owasp.dependencycheck.dependency.VulnerableSoftware;
import org.owasp.dependencycheck.utils.Settings;
import org.slf4j.Logger;

14
dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/UpdateableNvdCve.java → dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/nvd/UpdateableNvdCve.java
View File

@@ -15,7 +15,7 @@
*
* Copyright (c) 2012 Jeremy Long. All Rights Reserved.
*/
package org.owasp.dependencycheck.data.update;
package org.owasp.dependencycheck.data.update.nvd;
import java.net.MalformedURLException;
import java.net.URL;
@@ -27,8 +27,8 @@ import org.owasp.dependencycheck.utils.DownloadFailedException;
import org.owasp.dependencycheck.utils.Downloader;
/**
* Contains a collection of updateable NvdCveInfo objects. This is used to determine which files need to be downloaded
* and processed.
* Contains a collection of updateable NvdCveInfo objects. This is used to determine which files need to be downloaded and
* processed.
*
* @author Jeremy Long
*/
@@ -67,8 +67,7 @@ public class UpdateableNvdCve implements java.lang.Iterable<NvdCveInfo>, Iterato
*
* @param id the key for the item to be added
* @param url the URL to download the item
* @param oldUrl the URL for the old version of the item (the NVD CVE old schema still contains useful data we
* need).
* @param oldUrl the URL for the old version of the item (the NVD CVE old schema still contains useful data we need).
* @throws MalformedURLException thrown if the URL provided is invalid
* @throws DownloadFailedException thrown if the download fails.
*/
@@ -81,8 +80,7 @@ public class UpdateableNvdCve implements java.lang.Iterable<NvdCveInfo>, Iterato
*
* @param id the key for the item to be added
* @param url the URL to download the item
* @param oldUrl the URL for the old version of the item (the NVD CVE old schema still contains useful data we
* need).
* @param oldUrl the URL for the old version of the item (the NVD CVE old schema still contains useful data we need).
* @param needsUpdate whether or not the data needs to be updated
* @throws MalformedURLException thrown if the URL provided is invalid
* @throws DownloadFailedException thrown if the download fails.
@@ -175,7 +173,7 @@ public class UpdateableNvdCve implements java.lang.Iterable<NvdCveInfo>, Iterato
* @param key the key to lookup the return value
* @return the NvdCveInfo object stored using the specified key
*/
NvdCveInfo get(String key) {
public NvdCveInfo get(String key) {
return collection.get(key);
}

2
dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/xml/package-info.java → dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/nvd/package-info.java
View File

@@ -5,4 +5,4 @@
* extends Index). The Indexer creates a partial-unmarshalling SAX parser (implemented in the NvdCveXmlFilter) that extracts
* VulnerabilityTypes (aka Entry) from the NVD CVE data file and stores these into a Lucene Index.
*/
package org.owasp.dependencycheck.data.update.xml;
package org.owasp.dependencycheck.data.update.nvd;

4
dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/task/package-info.java
View File

@@ -1,4 +0,0 @@
/**
* A collection of callable/runnable tasks used to speed up the update process.
*/
package org.owasp.dependencycheck.data.update.task;

3
dependency-check-core/src/main/resources/META-INF/services/org.owasp.dependencycheck.data.update.CachedWebDataSource
View File

@@ -1,2 +1,3 @@
org.owasp.dependencycheck.data.update.NvdCveUpdater
org.owasp.dependencycheck.data.update.EngineVersionCheck
org.owasp.dependencycheck.data.update.EngineVersionCheck
org.owasp.dependencycheck.data.update.CpeUpdater

3
dependency-check-core/src/main/resources/data/dbStatements.properties
View File

@@ -34,3 +34,6 @@ SELECT_PROPERTY=SELECT id, value FROM properties WHERE id = ?
INSERT_PROPERTY=INSERT INTO properties (id, value) VALUES (?, ?)
UPDATE_PROPERTY=UPDATE properties SET value = ? WHERE id = ?
DELETE_PROPERTY=DELETE FROM properties WHERE id = ?
DELETE_UNUSED_DICT_CPE=DELETE FROM cpeEntry WHERE dictionaryEntry=true AND id NOT IN (SELECT cpeEntryId FROM software)
ADD_DICT_CPE=MERGE INTO cpeEntry (cpe, vendor, product, dictionaryEntry) KEY(cpe) VALUES(?,?,?,true)

6
dependency-check-core/src/main/resources/data/upgrade_2.9.sql Normal file
View File

@@ -0,0 +1,6 @@
ALTER TABLE cpeEntry ADD COLUMN IF NOT EXISTS dictionaryEntry BOOLEAN;
ALTER TABLE cpeEntry ALTER COLUMN dictionaryEntry SET DEFAULT FALSE;
UPDATE cpeEntry SET dictionaryEntry=false;
UPDATE Properties SET value='3.0' WHERE ID='version';

6
dependency-check-core/src/main/resources/dependencycheck.properties
View File

@@ -17,8 +17,8 @@ engine.version.url=http://jeremylong.github.io/DependencyCheck/current.txt
# below contains a %s then the data.directory will replace the %s.
data.directory=[JAR]/data
#if the filename has a %s it will be replaced with the current expected version
data.file_name=cve.%s.h2.db
data.version=2.9
data.file_name=dc.h2.db
data.version=3.0
data.connection_string=jdbc:h2:file:%s;FILE_LOCK=SERIALIZED;AUTOCOMMIT=ON;
#data.connection_string=jdbc:mysql://localhost:3306/dependencycheck
@@ -53,6 +53,8 @@ cve.url-1.2.base=https://nvd.nist.gov/download/nvdcve-%d.xml.gz
cve.url-2.0.base=https://nvd.nist.gov/feeds/xml/cve/nvdcve-2.0-%d.xml.gz
#cve.url-2.0.base=http://static.nvd.nist.gov/feeds/xml/cve/nvdcve-2.0-%d.xml
cpe.validfordays=30
cpe.url=http://static.nvd.nist.gov/feeds/xml/cpe/dictionary/official-cpe-dictionary_v2.3.xml.gz
# file type analyzer settings:
analyzer.archive.enabled=true

2
dependency-check-core/src/test/java/org/owasp/dependencycheck/data/nvdcve/BaseDBTestCase.java
View File

@@ -45,7 +45,7 @@ public abstract class BaseDBTestCase extends BaseTest {
public static void ensureDBExists() throws Exception {
java.io.File dataPath = Settings.getDataDirectory();
String fileName = String.format(Settings.getString(Settings.KEYS.DB_FILE_NAME), Settings.getString(Settings.KEYS.DB_VERSION));
String fileName = Settings.getString(Settings.KEYS.DB_FILE_NAME);
java.io.File dataFile = new File(dataPath, fileName);
if (!dataPath.exists() || !dataFile.exists()) {
dataPath.mkdirs();

2
dependency-check-core/src/test/java/org/owasp/dependencycheck/data/nvdcve/DatabasePropertiesIntegrationTest.java
View File

@@ -21,7 +21,7 @@ import java.util.Properties;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import org.junit.Test;
import org.owasp.dependencycheck.data.update.NvdCveInfo;
import org.owasp.dependencycheck.data.update.nvd.NvdCveInfo;
/**
*

98
dependency-check-core/src/test/java/org/owasp/dependencycheck/data/update/BaseUpdaterTest.java Normal file
View File

@@ -0,0 +1,98 @@
/*
* This file is part of dependency-check-core.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
* Copyright (c) 2015 Jeremy Long. All Rights Reserved.
*/
package org.owasp.dependencycheck.data.update;
import org.junit.Test;
import static org.junit.Assert.*;
import org.owasp.dependencycheck.BaseTest;
import org.owasp.dependencycheck.data.nvdcve.CveDB;
import org.owasp.dependencycheck.data.nvdcve.DatabaseProperties;
import org.owasp.dependencycheck.data.update.exception.UpdateException;
/**
*
* @author Jeremy Long
*/
public class BaseUpdaterTest extends BaseTest {
/**
* Test of getCveDB method, of class BaseUpdater.
*/
@Test
public void testGetCveDB() {
BaseUpdater instance = new BaseUpdaterImpl();
CveDB expResult = null;
CveDB result = instance.getCveDB();
assertEquals(expResult, result);
}
/**
* Test of getProperties method, of class BaseUpdater.
*/
@Test
public void testGetProperties() throws UpdateException {
BaseUpdater instance = null;
try {
instance = new BaseUpdaterImpl();
instance.openDataStores();
DatabaseProperties result = instance.getProperties();
assertTrue(result.getProperties().keySet().size() > 1);
} finally {
if (instance != null) {
instance.closeDataStores();
}
}
}
/**
* Test of closeDataStores method, of class BaseUpdater.
*/
@Test
public void testCloseDataStores() throws UpdateException {
BaseUpdater instance = null;
try {
instance = new BaseUpdaterImpl();
instance.openDataStores();
} finally {
if (instance != null) {
instance.closeDataStores();
}
}
}
/**
* Test of openDataStores method, of class BaseUpdater.
*/
@Test
public void testOpenDataStores() throws Exception {
BaseUpdater instance = null;
try {
instance = new BaseUpdaterImpl();
instance.openDataStores();
} finally {
if (instance != null) {
instance.closeDataStores();
}
}
}
public class BaseUpdaterImpl extends BaseUpdater {
}
}

41
dependency-check-core/src/test/java/org/owasp/dependencycheck/data/update/CpeUpdaterIntegrationTest.java Normal file
View File

@@ -0,0 +1,41 @@
/*
* Copyright 2015 OWASP.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.owasp.dependencycheck.data.update;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
import static org.junit.Assert.*;
import org.owasp.dependencycheck.BaseTest;
/**
*
* @author jeremy
*/
public class CpeUpdaterIntegrationTest extends BaseTest {
/**
* Test of update method, of class CpeUpdater.
*/
@Test
public void testUpdate() throws Exception {
CpeUpdater instance = new CpeUpdater();
instance.update();
}
}

61
dependency-check-core/src/test/java/org/owasp/dependencycheck/data/update/NvdCveUpdaterIntegrationTest.java
View File

@@ -17,12 +17,13 @@
*/
package org.owasp.dependencycheck.data.update;
import java.io.File;
import java.util.Calendar;
import org.junit.Before;
import java.net.MalformedURLException;
import static org.junit.Assert.assertNotNull;
import org.junit.Test;
import org.owasp.dependencycheck.BaseTest;
import org.owasp.dependencycheck.utils.Settings;
import org.owasp.dependencycheck.data.update.exception.UpdateException;
import org.owasp.dependencycheck.data.update.nvd.UpdateableNvdCve;
import org.owasp.dependencycheck.utils.DownloadFailedException;
/**
*
@@ -30,39 +31,33 @@ import org.owasp.dependencycheck.utils.Settings;
*/
public class NvdCveUpdaterIntegrationTest extends BaseTest {
@Before
public void setUp() throws Exception {
int year = Calendar.getInstance().get(Calendar.YEAR);
if (year <= 2014) {
//File f = new File(NvdCveUpdaterIntegrationTest.class.getClassLoader().getResource("nvdcve-2.0-2014.xml").getPath());
File f = BaseTest.getResourceAsFile(this, "nvdcve-2.0-2014.xml");
String baseURL = f.toURI().toURL().toString();
String modified12 = baseURL.replace("nvdcve-2.0-2014.xml", "nvdcve-modified.xml");
String modified20 = baseURL.replace("nvdcve-2.0-2014.xml", "nvdcve-2.0-modified.xml");
String full12 = baseURL.replace("nvdcve-2.0-2014.xml", "nvdcve-%d.xml");
String full20 = baseURL.replace("nvdcve-2.0-2014.xml", "nvdcve-2.0-%d.xml");
// cve.url-1.2.modified=http://nvd.nist.gov/download/nvdcve-modified.xml
// cve.url-2.0.modified=http://static.nvd.nist.gov/feeds/xml/cve/nvdcve-2.0-modified.xml
// cve.startyear=2014
// cve.url-2.0.base=http://static.nvd.nist.gov/feeds/xml/cve/nvdcve-2.0-%d.xml
// cve.url-1.2.base=http://nvd.nist.gov/download/nvdcve-%d.xml
Settings.setString(Settings.KEYS.CVE_MODIFIED_12_URL, modified12);
Settings.setString(Settings.KEYS.CVE_MODIFIED_20_URL, modified20);
Settings.setString(Settings.KEYS.CVE_SCHEMA_1_2, full12);
Settings.setString(Settings.KEYS.CVE_SCHEMA_2_0, full20);
Settings.setString(Settings.KEYS.CVE_START_YEAR, "2014");
} else {
System.err.println("Consider updating the local data files to make the NvdCveUpdaterIntegrationTest perform faster");
}
public NvdCveUpdater getUpdater() throws MalformedURLException, DownloadFailedException, UpdateException {
NvdCveUpdater instance = new NvdCveUpdater();
return instance;
}
// test removed as it is duplicative of the EngineIntegrationTest and the NvdCveUpdaterIntergraionTest
// /**
// * Test of update method, of class StandardUpdate.
// */
// @Test
// public void testUpdate() throws Exception {
// StandardUpdate instance = getStandardUpdateTask();
// instance.update();
// //TODO make this an actual test
// }
/**
* Test of update method, of class NvdCveUpdater.
* Test of updatesNeeded method, of class StandardUpdate.
*/
@Test
public void testUpdate() throws Exception {
NvdCveUpdater instance = new NvdCveUpdater();
instance.update();
public void testUpdatesNeeded() throws Exception {
NvdCveUpdater instance = getUpdater();
try {
instance.openDataStores();
UpdateableNvdCve result = instance.getUpdatesNeeded();
assertNotNull(result);
} finally {
instance.closeDataStores();
}
}
}

67
dependency-check-core/src/test/java/org/owasp/dependencycheck/data/update/StandardUpdateIntegrationTest.java
View File

@@ -1,67 +0,0 @@
/*
* This file is part of dependency-check-core.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
* Copyright (c) 2013 Jeremy Long. All Rights Reserved.
*/
package org.owasp.dependencycheck.data.update;
import java.net.MalformedURLException;
import static org.junit.Assert.assertNotNull;
import org.junit.Test;
import org.owasp.dependencycheck.BaseTest;
import org.owasp.dependencycheck.data.update.exception.UpdateException;
import org.owasp.dependencycheck.utils.DownloadFailedException;
/**
*
* @author Jeremy Long
*/
public class StandardUpdateIntegrationTest extends BaseTest {
public StandardUpdate getStandardUpdateTask() throws MalformedURLException, DownloadFailedException, UpdateException {
StandardUpdate instance = new StandardUpdate();
return instance;
}
/**
* Test of openDataStores method, of class StandardUpdate.
*/
@Test
public void testOpenDataStores() throws Exception {
StandardUpdate instance = getStandardUpdateTask();
instance.openDataStores();
instance.closeDataStores();
}
// test removed as it is duplicative of the EngineIntegrationTest and the NvdCveUpdaterIntergraionTest
// /**
// * Test of update method, of class StandardUpdate.
// */
// @Test
// public void testUpdate() throws Exception {
// StandardUpdate instance = getStandardUpdateTask();
// instance.update();
// //TODO make this an actual test
// }
/**
* Test of updatesNeeded method, of class StandardUpdate.
*/
@Test
public void testUpdatesNeeded() throws Exception {
StandardUpdate instance = getStandardUpdateTask();
UpdateableNvdCve result = instance.updatesNeeded();
assertNotNull(result);
}
}

6
dependency-check-core/src/test/java/org/owasp/dependencycheck/data/update/task/DownloadTaskTest.java → dependency-check-core/src/test/java/org/owasp/dependencycheck/data/update/nvd/DownloadTaskTest.java
View File

@@ -15,8 +15,10 @@
*
* Copyright (c) 2014 Jeremy Long. All Rights Reserved.
*/
package org.owasp.dependencycheck.data.update.task;
package org.owasp.dependencycheck.data.update.nvd;
import org.owasp.dependencycheck.data.update.nvd.ProcessTask;
import org.owasp.dependencycheck.data.update.nvd.DownloadTask;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Future;
import org.junit.After;
@@ -26,7 +28,7 @@ import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
import org.owasp.dependencycheck.data.nvdcve.CveDB;
import org.owasp.dependencycheck.data.update.NvdCveInfo;
import org.owasp.dependencycheck.data.update.nvd.NvdCveInfo;
import org.owasp.dependencycheck.utils.Settings;
/**

3
dependency-check-core/src/test/java/org/owasp/dependencycheck/data/update/NvdCveInfoTest.java → dependency-check-core/src/test/java/org/owasp/dependencycheck/data/update/nvd/NvdCveInfoTest.java
View File

@@ -15,8 +15,9 @@
*
* Copyright (c) 2013 Jeremy Long. All Rights Reserved.
*/
package org.owasp.dependencycheck.data.update;
package org.owasp.dependencycheck.data.update.nvd;
import org.owasp.dependencycheck.data.update.nvd.NvdCveInfo;
import static org.junit.Assert.assertEquals;
import org.junit.Test;
import org.owasp.dependencycheck.BaseTest;

69
dependency-check-core/src/test/java/org/owasp/dependencycheck/data/update/nvd/NvdCveUpdaterIntegrationTest.java Normal file
View File

@@ -0,0 +1,69 @@
/*
* This file is part of dependency-check-core.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
* Copyright (c) 2013 Jeremy Long. All Rights Reserved.
*/
package org.owasp.dependencycheck.data.update.nvd;
import java.io.File;
import java.util.Calendar;
import org.junit.Before;
import org.junit.Test;
import org.owasp.dependencycheck.BaseTest;
import org.owasp.dependencycheck.data.update.NvdCveUpdater;
import org.owasp.dependencycheck.utils.Settings;
/**
*
* @author Jeremy Long
*/
public class NvdCveUpdaterIntegrationTest extends BaseTest {
@Before
public void setUp() throws Exception {
int year = Calendar.getInstance().get(Calendar.YEAR);
if (year <= 2014) {
//File f = new File(NvdCveUpdaterIntegrationTest.class.getClassLoader().getResource("nvdcve-2.0-2014.xml").getPath());
File f = BaseTest.getResourceAsFile(this, "nvdcve-2.0-2014.xml");
String baseURL = f.toURI().toURL().toString();
String modified12 = baseURL.replace("nvdcve-2.0-2014.xml", "nvdcve-modified.xml");
String modified20 = baseURL.replace("nvdcve-2.0-2014.xml", "nvdcve-2.0-modified.xml");
String full12 = baseURL.replace("nvdcve-2.0-2014.xml", "nvdcve-%d.xml");
String full20 = baseURL.replace("nvdcve-2.0-2014.xml", "nvdcve-2.0-%d.xml");
// cve.url-1.2.modified=http://nvd.nist.gov/download/nvdcve-modified.xml
// cve.url-2.0.modified=http://static.nvd.nist.gov/feeds/xml/cve/nvdcve-2.0-modified.xml
// cve.startyear=2014
// cve.url-2.0.base=http://static.nvd.nist.gov/feeds/xml/cve/nvdcve-2.0-%d.xml
// cve.url-1.2.base=http://nvd.nist.gov/download/nvdcve-%d.xml
Settings.setString(Settings.KEYS.CVE_MODIFIED_12_URL, modified12);
Settings.setString(Settings.KEYS.CVE_MODIFIED_20_URL, modified20);
Settings.setString(Settings.KEYS.CVE_SCHEMA_1_2, full12);
Settings.setString(Settings.KEYS.CVE_SCHEMA_2_0, full20);
Settings.setString(Settings.KEYS.CVE_START_YEAR, "2014");
} else {
System.err.println("Consider updating the local data files to make the NvdCveUpdaterIntegrationTest perform faster");
}
}
/**
* Test of update method, of class NvdCveUpdater.
*/
@Test
public void testUpdate() throws Exception {
NvdCveUpdater instance = new NvdCveUpdater();
instance.update();
}
}

3
dependency-check-core/src/test/java/org/owasp/dependencycheck/data/update/xml/NvdCve_1_2_HandlerTest.java → dependency-check-core/src/test/java/org/owasp/dependencycheck/data/update/nvd/NvdCve_1_2_HandlerTest.java
View File

@@ -15,8 +15,9 @@
*
* Copyright (c) 2012 Jeremy Long. All Rights Reserved.
*/
package org.owasp.dependencycheck.data.update.xml;
package org.owasp.dependencycheck.data.update.nvd;
import org.owasp.dependencycheck.data.update.nvd.NvdCve12Handler;
import java.io.File;
import java.util.List;
import java.util.Map;

3
dependency-check-core/src/test/java/org/owasp/dependencycheck/data/update/xml/NvdCve_2_0_HandlerTest.java → dependency-check-core/src/test/java/org/owasp/dependencycheck/data/update/nvd/NvdCve_2_0_HandlerTest.java
View File

@@ -15,8 +15,9 @@
*
* Copyright (c) 2012 Jeremy Long. All Rights Reserved.
*/
package org.owasp.dependencycheck.data.update.xml;
package org.owasp.dependencycheck.data.update.nvd;
import org.owasp.dependencycheck.data.update.nvd.NvdCve20Handler;
import java.io.File;
import javax.xml.parsers.SAXParser;
import javax.xml.parsers.SAXParserFactory;

4
dependency-check-core/src/test/java/org/owasp/dependencycheck/data/update/UpdateableNvdCveTest.java → dependency-check-core/src/test/java/org/owasp/dependencycheck/data/update/nvd/UpdateableNvdCveTest.java
View File

@@ -15,8 +15,10 @@
*
* Copyright (c) 2013 Jeremy Long. All Rights Reserved.
*/
package org.owasp.dependencycheck.data.update;
package org.owasp.dependencycheck.data.update.nvd;
import org.owasp.dependencycheck.data.update.nvd.UpdateableNvdCve;
import org.owasp.dependencycheck.data.update.nvd.NvdCveInfo;
import java.io.File;
import java.io.IOException;
import java.net.MalformedURLException;

4
dependency-check-core/src/test/resources/dependencycheck.properties
View File

@@ -19,8 +19,8 @@ data.directory=[JAR]/data
# if the filename has a %s it will be replaced with the current expected version. For file
# based databases the below filename will be added to the data directory above and then
# if the connection string has a %s it will be replaced by the directory/filename path.
data.file_name=cve.%s.h2.db
data.version=2.9
data.file_name=dc.h2.db
data.version=3.0
data.connection_string=jdbc:h2:file:%s;FILE_LOCK=SERIALIZED;AUTOCOMMIT=ON;
#data.connection_string=jdbc:mysql://localhost:3306/dependencycheck

17
dependency-check-utils/src/main/java/org/owasp/dependencycheck/utils/Settings.java
View File

@@ -736,16 +736,15 @@ public final class Settings {
*
* @param connectionStringKey the property file key for the connection string
* @param dbFileNameKey the settings key for the db filename
* @param dbVersionKey the settings key for the dbVersion
* @return the connection string
* @throws IOException thrown the data directory cannot be created
* @throws InvalidSettingException thrown if there is an invalid setting
*/
public static String getConnectionString(String connectionStringKey, String dbFileNameKey, String dbVersionKey)
public static String getConnectionString(String connectionStringKey, String dbFileNameKey)
throws IOException, InvalidSettingException {
final String connStr = Settings.getString(connectionStringKey);
if (connStr == null) {
final String msg = String.format("Invalid properties file to get the connection string; '%s' must be defined.",
final String msg = String.format("Invalid properties file; data.connection_string is missing.",
connectionStringKey);
throw new InvalidSettingException(msg);
}
@@ -760,18 +759,6 @@ public final class Settings {
dbFileNameKey);
throw new InvalidSettingException(msg);
}
if (fileName.contains("%s")) {
String version = null;
if (dbVersionKey != null) {
version = Settings.getString(dbVersionKey);
}
if (version == null) {
final String msg = String.format("Invalid properties file to get a file based connection string; '%s' must be defined.",
dbFileNameKey);
throw new InvalidSettingException(msg);
}
fileName = String.format(fileName, version);
}
if (connStr.startsWith("jdbc:h2:file:") && fileName.endsWith(".h2.db")) {
fileName = fileName.substring(0, fileName.length() - 6);
}

12
dependency-check-utils/src/test/java/org/owasp/dependencycheck/utils/SettingsTest.java
View File

@@ -163,21 +163,15 @@ public class SettingsTest extends BaseTest {
*/
@Test
public void testGetConnectionString() throws Exception {
String value = Settings.getConnectionString(Settings.KEYS.DB_CONNECTION_STRING, Settings.KEYS.DB_FILE_NAME, Settings.KEYS.DB_VERSION);
String value = Settings.getConnectionString(Settings.KEYS.DB_CONNECTION_STRING, Settings.KEYS.DB_FILE_NAME);
Assert.assertNotNull(value);
String msg = null;
try {
value = Settings.getConnectionString(Settings.KEYS.DB_CONNECTION_STRING, Settings.KEYS.DB_FILE_NAME, null);
value = Settings.getConnectionString("invalidKey", null);
} catch (InvalidSettingException e) {
msg = e.getMessage();
}
Assert.assertNotNull(msg, msg);
try {
value = Settings.getConnectionString("invalidKey", null, null);
} catch (InvalidSettingException e) {
msg = e.getMessage();
}
Assert.assertNotNull(msg, msg);
Assert.assertNotNull(msg);
}
/**

4
dependency-check-utils/src/test/resources/dependencycheck.properties
View File

@@ -16,8 +16,8 @@ engine.version.url=http://jeremylong.github.io/DependencyCheck/current.txt
# will not be used. The data.directory will be resolved and if the connection string
# below contains a %s then the data.directory will replace the %s.
data.directory=[JAR]/data
data.file_name=cve.%s.h2.db
data.version=2.9
data.file_name=dc.h2.db
data.version=3.0
data.connection_string=jdbc:h2:file:%s;FILE_LOCK=SERIALIZED;AUTOCOMMIT=ON;
#data.connection_string=jdbc:h2:file:%s;AUTO_SERVER=TRUE;AUTOCOMMIT=ON;
#data.connection_string=jdbc:mysql://localhost:3306/dependencycheck