github-mirrors/DependencyCheck
1
0
Fork 0
mirror of https://github.com/ysoftdevs/DependencyCheck.git synced 2026-01-14 15:53:36 +01:00
Code Issues Packages Projects Releases Wiki Activity

Updated to remove batch update and to remove the abstract class used to enable batch mode

Browse Source 
Former-commit-id: 2f33644d1a50a425c6ae5e1ca671156e4cdadf8e
This commit is contained in:
Jeremy Long
2013-12-02 05:43:54 -05:00
parent df0d0d820a
commit 7eb82f2e84
9 changed files with 234 additions and 924 deletions
Download Patch File Download Diff File Expand all files Collapse all files

260
dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/AbstractUpdateTask.java
View File

@@ -1,260 +0,0 @@
/*
* This file is part of dependency-check-core.
*
* Dependency-check-core is free software: you can redistribute it and/or modify it
* under the terms of the GNU General Public License as published by the Free
* Software Foundation, either version 3 of the License, or (at your option) any
* later version.
*
* Dependency-check-core is distributed in the hope that it will be useful, but
* WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
* details.
*
* You should have received a copy of the GNU General Public License along with
* dependency-check-core. If not, see http://www.gnu.org/licenses/.
*
* Copyright (c) 2012 Jeremy Long. All Rights Reserved.
*/
package org.owasp.dependencycheck.data.update;
import java.io.File;
import java.io.IOException;
import java.net.MalformedURLException;
import java.sql.SQLException;
import java.util.List;
import java.util.Map;
import java.util.logging.Level;
import java.util.logging.Logger;
import javax.xml.parsers.ParserConfigurationException;
import javax.xml.parsers.SAXParser;
import javax.xml.parsers.SAXParserFactory;
import org.owasp.dependencycheck.data.UpdateException;
import org.owasp.dependencycheck.data.nvdcve.CveDB;
import org.owasp.dependencycheck.utils.FileUtils;
import org.owasp.dependencycheck.utils.Settings;
import org.owasp.dependencycheck.data.nvdcve.DatabaseException;
import org.owasp.dependencycheck.data.nvdcve.NvdCve12Handler;
import org.owasp.dependencycheck.data.nvdcve.NvdCve20Handler;
import org.owasp.dependencycheck.dependency.VulnerableSoftware;
import org.owasp.dependencycheck.utils.DownloadFailedException;
import org.xml.sax.SAXException;
/**
* Class responsible for updating the CPE and NVDCVE data stores.
*
* @author Jeremy Long (jeremy.long@owasp.org)
*/
public abstract class AbstractUpdateTask implements UpdateTask {
/**
* Initializes the AbstractUpdateTask.
*
* @param properties information about the data store
* @throws MalformedURLException thrown if the configuration contains a
* malformed url
* @throws DownloadFailedException thrown if the timestamp on a file cannot
* be checked
* @throws UpdateException thrown if the update fails
*/
public AbstractUpdateTask(DataStoreMetaInfo properties) throws MalformedURLException, DownloadFailedException, UpdateException {
this.properties = properties;
this.updateable = updatesNeeded();
}
/**
* A collection of updateable NVD CVE items.
*/
private Updateable updateable;
/**
* Utility to read and write meta-data about the data.
*/
private DataStoreMetaInfo properties = null;
/**
* Returns the data store properties.
*
* @return the data store properties
*/
protected DataStoreMetaInfo getProperties() {
return properties;
}
/**
* Reference to the Cve Database.
*/
private CveDB cveDB = null;
/**
* Returns the CveDB.
*
* @return the CveDB
*/
protected CveDB getCveDB() {
return cveDB;
}
/**
* Gets whether or not an update is needed.
*
* @return true or false depending on whether an update is needed
*/
public boolean isUpdateNeeded() {
return updateable.isUpdateNeeded();
}
/**
* Gets the updateable NVD CVE Entries.
*
* @return an Updateable object containing the NVD CVE entries
*/
protected Updateable getUpdateable() {
return updateable;
}
/**
* Determines if the index needs to be updated.
*
* @return a collection of updateable resources.
* @throws MalformedURLException is thrown if the URL for the NVD CVE Meta
* data is incorrect.
* @throws DownloadFailedException is thrown if there is an error.
* downloading the NVD CVE download data file.
* @throws UpdateException Is thrown if there is an issue with the last
* updated properties file.
*/
protected abstract Updateable updatesNeeded() throws MalformedURLException, DownloadFailedException, UpdateException;
/**
* <p>Updates the data store to the latest version.</p>
*
* @throws UpdateException is thrown if there is an error updating the
* database
*/
public abstract void update() throws UpdateException;
/**
* A flag indicating whether or not the current data store should be
* deleted.
*/
private boolean deleteAndRecreate = false;
/**
* Get the value of deleteAndRecreate.
*
* @return the value of deleteAndRecreate
*/
public boolean shouldDeleteAndRecreate() {
return deleteAndRecreate;
}
/**
* Set the value of deleteAndRecreate.
*
* @param deleteAndRecreate new value of deleteAndRecreate
*/
protected void setDeleteAndRecreate(boolean deleteAndRecreate) {
this.deleteAndRecreate = deleteAndRecreate;
}
/**
* Deletes the existing data directories.
*
* @throws IOException thrown if the directory cannot be deleted
*/
protected void deleteExistingData() throws IOException {
File data = Settings.getDataFile(Settings.KEYS.CVE_DATA_DIRECTORY);
if (data.exists()) {
FileUtils.delete(data);
}
data = DataStoreMetaInfo.getPropertiesFile();
if (data.exists()) {
FileUtils.delete(data);
}
}
/**
* Closes the CVE and CPE data stores.
*/
protected void closeDataStores() {
if (cveDB != null) {
try {
cveDB.close();
} catch (Exception ignore) {
Logger.getLogger(AbstractUpdateTask.class.getName()).log(Level.FINEST, "Error closing the cveDB", ignore);
}
}
}
/**
* Opens the CVE and CPE data stores.
*
* @throws UpdateException thrown if a data store cannot be opened
*/
protected void openDataStores() throws UpdateException {
//open the cve and cpe data stores
try {
cveDB = new CveDB();
cveDB.open();
} catch (IOException ex) {
closeDataStores();
Logger.getLogger(AbstractUpdateTask.class.getName()).log(Level.FINE, "IO Error opening databases", ex);
throw new UpdateException("Error updating the CPE/CVE data, please see the log file for more details.");
} catch (SQLException ex) {
closeDataStores();
Logger.getLogger(AbstractUpdateTask.class.getName()).log(Level.FINE, "SQL Exception opening databases", ex);
throw new UpdateException("Error updating the CPE/CVE data, please see the log file for more details.");
} catch (DatabaseException ex) {
closeDataStores();
Logger.getLogger(AbstractUpdateTask.class.getName()).log(Level.FINE, "Database Exception opening databases", ex);
throw new UpdateException("Error updating the CPE/CVE data, please see the log file for more details.");
} catch (ClassNotFoundException ex) {
closeDataStores();
Logger.getLogger(AbstractUpdateTask.class.getName()).log(Level.FINE, "Class not found exception opening databases", ex);
throw new UpdateException("Error updating the CPE/CVE data, please see the log file for more details.");
}
}
/**
* Determines if the epoch date is within the range specified of the
* compareTo epoch time. This takes the (compareTo-date)/1000/60/60/24 to
* get the number of days. If the calculated days is less then the range the
* date is considered valid.
*
* @param date the date to be checked.
* @param compareTo the date to compare to.
* @param range the range in days to be considered valid.
* @return whether or not the date is within the range.
*/
protected boolean withinRange(long date, long compareTo, int range) {
final double differenceInDays = (compareTo - date) / 1000.0 / 60.0 / 60.0 / 24.0;
return differenceInDays < range;
}
/**
* Imports the NVD CVE XML File into the Lucene Index.
*
* @param file the file containing the NVD CVE XML
* @param oldVersion contains the file containing the NVD CVE XML 1.2
* @throws ParserConfigurationException is thrown if there is a parser
* configuration exception
* @throws SAXException is thrown if there is a SAXException
* @throws IOException is thrown if there is a IO Exception
* @throws SQLException is thrown if there is a SQL exception
* @throws DatabaseException is thrown if there is a database exception
* @throws ClassNotFoundException thrown if the h2 database driver cannot be
* loaded
*/
protected void importXML(File file, File oldVersion) throws ParserConfigurationException,
SAXException, IOException, SQLException, DatabaseException, ClassNotFoundException {
final SAXParserFactory factory = SAXParserFactory.newInstance();
final SAXParser saxParser = factory.newSAXParser();
final NvdCve12Handler cve12Handler = new NvdCve12Handler();
saxParser.parse(oldVersion, cve12Handler);
final Map<String, List<VulnerableSoftware>> prevVersionVulnMap = cve12Handler.getVulnerabilities();
final NvdCve20Handler cve20Handler = new NvdCve20Handler();
cve20Handler.setCveDB(cveDB);
cve20Handler.setPrevVersionVulnMap(prevVersionVulnMap);
saxParser.parse(file, cve20Handler);
}
}

272
dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/BatchUpdateTask.java
View File

@@ -1,272 +0,0 @@
/*
* This file is part of dependency-check-core.
*
* Dependency-check-core is free software: you can redistribute it and/or modify it
* under the terms of the GNU General Public License as published by the Free
* Software Foundation, either version 3 of the License, or (at your option) any
* later version.
*
* Dependency-check-core is distributed in the hope that it will be useful, but
* WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
* details.
*
* You should have received a copy of the GNU General Public License along with
* dependency-check-core. If not, see http://www.gnu.org/licenses/.
*
* Copyright (c) 2012 Jeremy Long. All Rights Reserved.
*/
package org.owasp.dependencycheck.data.update;
import org.owasp.dependencycheck.data.nvdcve.InvalidDataException;
import java.io.File;
import java.io.IOException;
import java.net.MalformedURLException;
import java.net.URISyntaxException;
import java.net.URL;
import java.util.Calendar;
import java.util.Date;
import java.util.logging.Level;
import java.util.logging.Logger;
import org.owasp.dependencycheck.data.UpdateException;
import org.owasp.dependencycheck.data.nvdcve.CveDB;
import org.owasp.dependencycheck.utils.DownloadFailedException;
import org.owasp.dependencycheck.utils.Downloader;
import org.owasp.dependencycheck.utils.FileUtils;
import org.owasp.dependencycheck.utils.Settings;
import org.owasp.dependencycheck.utils.InvalidSettingException;
import static org.owasp.dependencycheck.data.update.DataStoreMetaInfo.BATCH;
import static org.owasp.dependencycheck.data.update.DataStoreMetaInfo.MODIFIED;
/**
* Class responsible for updating the CPE and NVDCVE data stores.
*
* @author Jeremy Long (jeremy.long@owasp.org)
* @deprecated batch update mode will no longer be supported.
*/
@Deprecated
public class BatchUpdateTask extends AbstractUpdateTask {
/**
* Constructs a new BatchUpdateTask.
*
* @param properties information about the data store
* @throws MalformedURLException thrown if a configured URL is malformed
* @throws DownloadFailedException thrown if a timestamp cannot be checked
* on a configured URL
* @throws UpdateException thrown if there is an exception generating the
* update task
*/
public BatchUpdateTask(DataStoreMetaInfo properties) throws MalformedURLException, DownloadFailedException, UpdateException {
super(properties);
}
/**
* A flag indicating whether or not the batch update should be performed.
*/
private boolean doBatchUpdate;
/**
* Get the value of doBatchUpdate
*
* @return the value of doBatchUpdate
*/
protected boolean isDoBatchUpdate() {
return doBatchUpdate;
}
/**
* Set the value of doBatchUpdate
*
* @param doBatchUpdate new value of doBatchUpdate
*/
protected void setDoBatchUpdate(boolean doBatchUpdate) {
this.doBatchUpdate = doBatchUpdate;
}
/**
* <p>Downloads the latest NVD CVE XML file from the web and imports it into
* the current CVE Database.</p>
*
* @throws UpdateException is thrown if there is an error updating the
* database
*/
@Override
public void update() throws UpdateException {
if (getProperties().isBatchUpdateMode() && doBatchUpdate) {
final String batchSrc = Settings.getString(Settings.KEYS.BATCH_UPDATE_URL);
File tmp = null;
try {
deleteExistingData();
final File dataDirectory = CveDB.getDataDirectory().getParentFile();
final URL batchUrl = new URL(batchSrc);
if ("file".equals(batchUrl.getProtocol())) {
try {
tmp = new File(batchUrl.toURI());
} catch (URISyntaxException ex) {
final String msg = String.format("Invalid batch update URI: %s", batchSrc);
throw new UpdateException(msg, ex);
}
} else if ("http".equals(batchUrl.getProtocol())
|| "https".equals(batchUrl.getProtocol())) {
tmp = File.createTempFile("batch_", ".zip");
Downloader.fetchFile(batchUrl, tmp);
}
//TODO add FTP?
FileUtils.extractFiles(tmp, dataDirectory);
} catch (IOException ex) {
final String msg = String.format("IO Exception Occured performing batch update using: %s", batchSrc);
throw new UpdateException(msg, ex);
} finally {
if (tmp != null && !tmp.delete()) {
tmp.deleteOnExit();
}
}
}
}
/**
* Determines if the index needs to be updated. This is done by fetching the
* NVD CVE meta data and checking the last update date. If the data needs to
* be refreshed this method will return the NvdCveUrl for the files that
* need to be updated.
*
* @return the collection of files that need to be updated
* @throws MalformedURLException is thrown if the URL for the NVD CVE Meta
* data is incorrect
* @throws DownloadFailedException is thrown if there is an error.
* downloading the NVD CVE download data file
* @throws UpdateException Is thrown if there is an issue with the last
* updated properties file
*/
@Override
public Updateable updatesNeeded() throws MalformedURLException, DownloadFailedException, UpdateException {
Updateable updates = null;
try {
updates = retrieveCurrentTimestampsFromWeb();
} catch (InvalidDataException ex) {
final String msg = "Unable to retrieve valid timestamp from nvd cve downloads page";
Logger.getLogger(BatchUpdateTask.class.getName()).log(Level.FINE, msg, ex);
throw new DownloadFailedException(msg, ex);
} catch (InvalidSettingException ex) {
Logger.getLogger(BatchUpdateTask.class.getName()).log(Level.FINE, "Invalid setting found when retrieving timestamps", ex);
throw new DownloadFailedException("Invalid settings", ex);
}
if (updates == null) {
throw new DownloadFailedException("Unable to retrieve the timestamps of the currently published NVD CVE data");
}
final DataStoreMetaInfo properties = getProperties();
if (!properties.isEmpty()) {
try {
boolean deleteAndRecreate = false;
float version;
if (properties.getProperty("version") == null) {
deleteAndRecreate = true;
} else {
try {
version = Float.parseFloat(properties.getProperty("version"));
final float currentVersion = Float.parseFloat(CveDB.DB_SCHEMA_VERSION);
if (currentVersion > version) {
deleteAndRecreate = true;
}
} catch (NumberFormatException ex) {
deleteAndRecreate = true;
}
}
final NvdCveInfo batchInfo = updates.get(BATCH);
if (properties.isBatchUpdateMode() && batchInfo != null) {
final long lastUpdated = Long.parseLong(properties.getProperty(DataStoreMetaInfo.BATCH, "0"));
if (lastUpdated != batchInfo.getTimestamp()) {
deleteAndRecreate = true;
}
}
if (deleteAndRecreate) {
setDoBatchUpdate(properties.isBatchUpdateMode());
try {
deleteExistingData();
} catch (IOException ex) {
final String msg = "Unable to delete existing data";
Logger.getLogger(BatchUpdateTask.class.getName()).log(Level.WARNING, msg);
Logger.getLogger(BatchUpdateTask.class.getName()).log(Level.FINE, null, ex);
}
return updates;
}
final long lastUpdated = Long.parseLong(properties.getProperty(DataStoreMetaInfo.LAST_UPDATED, "0"));
final Date now = new Date();
final int days = Settings.getInt(Settings.KEYS.CVE_MODIFIED_VALID_FOR_DAYS, 7);
final int start = Settings.getInt(Settings.KEYS.CVE_START_YEAR, 2002);
final int end = Calendar.getInstance().get(Calendar.YEAR);
if (lastUpdated == updates.get(MODIFIED).getTimestamp()) {
updates.clear(); //we don't need to update anything.
setDoBatchUpdate(properties.isBatchUpdateMode());
} else if (withinRange(lastUpdated, now.getTime(), days)) {
updates.get(MODIFIED).setNeedsUpdate(true);
if (properties.isBatchUpdateMode()) {
setDoBatchUpdate(false);
} else {
for (int i = start; i <= end; i++) {
updates.get(String.valueOf(i)).setNeedsUpdate(false);
}
}
} else if (properties.isBatchUpdateMode()) {
updates.get(MODIFIED).setNeedsUpdate(true);
setDoBatchUpdate(true);
} else { //we figure out which of the several XML files need to be downloaded.
updates.get(MODIFIED).setNeedsUpdate(false);
for (int i = start; i <= end; i++) {
final NvdCveInfo cve = updates.get(String.valueOf(i));
long currentTimestamp = 0;
try {
currentTimestamp = Long.parseLong(properties.getProperty(DataStoreMetaInfo.LAST_UPDATED_BASE + String.valueOf(i), "0"));
} catch (NumberFormatException ex) {
final String msg = String.format("Error parsing '%s' '%s' from nvdcve.lastupdated",
DataStoreMetaInfo.LAST_UPDATED_BASE, String.valueOf(i));
Logger.getLogger(BatchUpdateTask.class.getName()).log(Level.FINE, msg, ex);
}
if (currentTimestamp == cve.getTimestamp()) {
cve.setNeedsUpdate(false); //they default to true.
}
}
}
} catch (NumberFormatException ex) {
final String msg = "An invalid schema version or timestamp exists in the data.properties file.";
Logger.getLogger(BatchUpdateTask.class.getName()).log(Level.WARNING, msg);
Logger.getLogger(BatchUpdateTask.class.getName()).log(Level.FINE, null, ex);
setDoBatchUpdate(properties.isBatchUpdateMode());
}
}
return updates;
}
/**
* Retrieves the timestamps from the NVD CVE meta data file.
*
* @return the timestamp from the currently published nvdcve downloads page
* @throws MalformedURLException thrown if the URL for the NVD CCE Meta data
* is incorrect.
* @throws DownloadFailedException thrown if there is an error downloading
* the nvd cve meta data file
* @throws InvalidDataException thrown if there is an exception parsing the
* timestamps
* @throws InvalidSettingException thrown if the settings are invalid
*/
private Updateable retrieveCurrentTimestampsFromWeb()
throws MalformedURLException, DownloadFailedException, InvalidDataException, InvalidSettingException {
final Updateable updates = new Updateable();
updates.add(BATCH, Settings.getString(Settings.KEYS.BATCH_UPDATE_URL),
null, false);
final String url = Settings.getString(Settings.KEYS.CVE_MODIFIED_20_URL, "");
if (!url.isEmpty()) {
updates.add(MODIFIED, url,
Settings.getString(Settings.KEYS.CVE_MODIFIED_12_URL),
false);
}
return updates;
}
}

2
dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/DatabaseUpdater.java
View File

@@ -46,7 +46,7 @@ public class DatabaseUpdater implements CachedWebDataSource {
@Override
public void update() throws UpdateException {
try {
final UpdateTask task = UpdateTaskFactory.getUpdateTask();
final StandardUpdateTask task = new StandardUpdateTask();
if (task.isUpdateNeeded()) {
if (task.shouldDeleteAndRecreate()) {
try {

188
dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/StandardUpdateTask.java
View File

@@ -29,6 +29,8 @@ import java.sql.SQLException;
import java.util.Calendar;
import java.util.Date;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
@@ -36,38 +38,89 @@ import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.logging.Level;
import java.util.logging.Logger;
import javax.xml.parsers.SAXParser;
import javax.xml.parsers.SAXParserFactory;
import org.owasp.dependencycheck.data.UpdateException;
import org.owasp.dependencycheck.data.nvdcve.CveDB;
import org.owasp.dependencycheck.utils.DownloadFailedException;
import org.owasp.dependencycheck.utils.Settings;
import org.owasp.dependencycheck.data.nvdcve.DatabaseException;
import org.owasp.dependencycheck.data.nvdcve.NvdCve12Handler;
import org.owasp.dependencycheck.data.nvdcve.NvdCve20Handler;
import org.owasp.dependencycheck.utils.InvalidSettingException;
import static org.owasp.dependencycheck.data.update.DataStoreMetaInfo.MODIFIED;
import org.owasp.dependencycheck.dependency.VulnerableSoftware;
import org.owasp.dependencycheck.utils.FileUtils;
/**
* Class responsible for updating the CPE and NVDCVE data stores.
*
* @author Jeremy Long (jeremy.long@owasp.org)
*/
public class StandardUpdateTask extends AbstractUpdateTask {
public class StandardUpdateTask {
/**
* The max thread pool size to use when downloading files.
*/
public static final int MAX_THREAD_POOL_SIZE = Settings.getInt(Settings.KEYS.MAX_DOWNLOAD_THREAD_POOL_SIZE, 3);
/**
* Information about the timestamps and URLs for data that needs to be
* updated.
*/
private DataStoreMetaInfo properties;
/**
* A collection of updateable NVD CVE items.
*/
private Updateable updateable;
/**
* A flag indicating whether or not the current data store should be
* deleted.
*/
private boolean deleteAndRecreate = false;
/**
* Reference to the Cve Database.
*/
private CveDB cveDB = null;
/**
* Gets whether or not an update is needed.
*
* @return true or false depending on whether an update is needed
*/
public boolean isUpdateNeeded() {
return updateable.isUpdateNeeded();
}
/**
* Set the value of deleteAndRecreate.
*
* @param deleteAndRecreate new value of deleteAndRecreate
*/
protected void setDeleteAndRecreate(boolean deleteAndRecreate) {
this.deleteAndRecreate = deleteAndRecreate;
}
/**
* Get the value of deleteAndRecreate.
*
* @return the value of deleteAndRecreate
*/
public boolean shouldDeleteAndRecreate() {
return deleteAndRecreate;
}
/**
* Constructs a new Standard Update Task.
*
* @param properties information about the data store
* @throws MalformedURLException thrown if a configured URL is malformed
* @throws DownloadFailedException thrown if a timestamp cannot be checked
* on a configured URL
* @throws UpdateException thrown if there is an exception generating the
* update task
*/
public StandardUpdateTask(DataStoreMetaInfo properties) throws MalformedURLException, DownloadFailedException, UpdateException {
super(properties);
public StandardUpdateTask() throws MalformedURLException, DownloadFailedException, UpdateException {
properties = new DataStoreMetaInfo();
updateable = updatesNeeded();
}
/**
@@ -77,11 +130,10 @@ public class StandardUpdateTask extends AbstractUpdateTask {
* @throws UpdateException is thrown if there is an error updating the
* database
*/
@Override
public void update() throws UpdateException {
int maxUpdates = 0;
try {
for (NvdCveInfo cve : getUpdateable()) {
for (NvdCveInfo cve : updateable) {
if (cve.getNeedsUpdate()) {
maxUpdates += 1;
}
@@ -101,7 +153,7 @@ public class StandardUpdateTask extends AbstractUpdateTask {
final ExecutorService executorService = Executors.newFixedThreadPool(poolSize);
final Set<Future<CallableDownloadTask>> futures = new HashSet<Future<CallableDownloadTask>>(maxUpdates);
for (NvdCveInfo cve : getUpdateable()) {
for (NvdCveInfo cve : updateable) {
if (cve.getNeedsUpdate()) {
final File file1;
final File file2;
@@ -123,8 +175,8 @@ public class StandardUpdateTask extends AbstractUpdateTask {
Logger.getLogger(StandardUpdateTask.class.getName()).log(Level.INFO, msg);
try {
importXML(filePair.getFirst(), filePair.getSecond());
getCveDB().commit();
getProperties().save(filePair.getNvdCveInfo());
cveDB.commit();
properties.save(filePair.getNvdCveInfo());
} catch (FileNotFoundException ex) {
throw new UpdateException(ex);
} catch (ParserConfigurationException ex) {
@@ -159,8 +211,8 @@ public class StandardUpdateTask extends AbstractUpdateTask {
}
if (maxUpdates >= 1) { //ensure the modified file date gets written
getProperties().save(getUpdateable().get(MODIFIED));
getCveDB().cleanupDatabase();
properties.save(updateable.get(MODIFIED));
cveDB.cleanupDatabase();
}
} finally {
closeDataStores();
@@ -279,7 +331,6 @@ public class StandardUpdateTask extends AbstractUpdateTask {
* @throws UpdateException Is thrown if there is an issue with the last
* updated properties file
*/
@Override
protected Updateable updatesNeeded() throws MalformedURLException, DownloadFailedException, UpdateException {
Updateable updates = null;
try {
@@ -300,26 +351,25 @@ public class StandardUpdateTask extends AbstractUpdateTask {
if (updates == null) {
throw new DownloadFailedException("Unable to retrieve the timestamps of the currently published NVD CVE data");
}
final DataStoreMetaInfo properties = getProperties();
if (!properties.isEmpty()) {
try {
float version;
if (properties.getProperty("version") == null) {
setDeleteAndRecreate(true);
deleteAndRecreate = true;
} else {
try {
version = Float.parseFloat(properties.getProperty("version"));
final float currentVersion = Float.parseFloat(CveDB.DB_SCHEMA_VERSION);
if (currentVersion > version) {
setDeleteAndRecreate(true);
deleteAndRecreate = true;
}
} catch (NumberFormatException ex) {
setDeleteAndRecreate(true);
deleteAndRecreate = true;
}
}
if (shouldDeleteAndRecreate()) {
if (deleteAndRecreate) {
return updates;
}
@@ -401,4 +451,108 @@ public class StandardUpdateTask extends AbstractUpdateTask {
return updates;
}
/**
* Deletes the existing data directories.
*
* @throws IOException thrown if the directory cannot be deleted
*/
protected void deleteExistingData() throws IOException {
File data = Settings.getDataFile(Settings.KEYS.CVE_DATA_DIRECTORY);
if (data.exists()) {
FileUtils.delete(data);
}
data = DataStoreMetaInfo.getPropertiesFile();
if (data.exists()) {
FileUtils.delete(data);
}
}
/**
* Closes the CVE and CPE data stores.
*/
protected void closeDataStores() {
if (cveDB != null) {
try {
cveDB.close();
} catch (Exception ignore) {
Logger.getLogger(StandardUpdateTask.class.getName()).log(Level.FINEST, "Error closing the cveDB", ignore);
}
}
}
/**
* Opens the CVE and CPE data stores.
*
* @throws UpdateException thrown if a data store cannot be opened
*/
protected void openDataStores() throws UpdateException {
//open the cve and cpe data stores
try {
cveDB = new CveDB();
cveDB.open();
} catch (IOException ex) {
closeDataStores();
Logger.getLogger(StandardUpdateTask.class.getName()).log(Level.FINE, "IO Error opening databases", ex);
throw new UpdateException("Error updating the CPE/CVE data, please see the log file for more details.");
} catch (SQLException ex) {
closeDataStores();
Logger.getLogger(StandardUpdateTask.class.getName()).log(Level.FINE, "SQL Exception opening databases", ex);
throw new UpdateException("Error updating the CPE/CVE data, please see the log file for more details.");
} catch (DatabaseException ex) {
closeDataStores();
Logger.getLogger(StandardUpdateTask.class.getName()).log(Level.FINE, "Database Exception opening databases", ex);
throw new UpdateException("Error updating the CPE/CVE data, please see the log file for more details.");
} catch (ClassNotFoundException ex) {
closeDataStores();
Logger.getLogger(StandardUpdateTask.class.getName()).log(Level.FINE, "Class not found exception opening databases", ex);
throw new UpdateException("Error updating the CPE/CVE data, please see the log file for more details.");
}
}
/**
* Determines if the epoch date is within the range specified of the
* compareTo epoch time. This takes the (compareTo-date)/1000/60/60/24 to
* get the number of days. If the calculated days is less then the range the
* date is considered valid.
*
* @param date the date to be checked.
* @param compareTo the date to compare to.
* @param range the range in days to be considered valid.
* @return whether or not the date is within the range.
*/
protected boolean withinRange(long date, long compareTo, int range) {
final double differenceInDays = (compareTo - date) / 1000.0 / 60.0 / 60.0 / 24.0;
return differenceInDays < range;
}
/**
* Imports the NVD CVE XML File into the Lucene Index.
*
* @param file the file containing the NVD CVE XML
* @param oldVersion contains the file containing the NVD CVE XML 1.2
* @throws ParserConfigurationException is thrown if there is a parser
* configuration exception
* @throws SAXException is thrown if there is a SAXException
* @throws IOException is thrown if there is a IO Exception
* @throws SQLException is thrown if there is a SQL exception
* @throws DatabaseException is thrown if there is a database exception
* @throws ClassNotFoundException thrown if the h2 database driver cannot be
* loaded
*/
protected void importXML(File file, File oldVersion) throws ParserConfigurationException,
SAXException, IOException, SQLException, DatabaseException, ClassNotFoundException {
final SAXParserFactory factory = SAXParserFactory.newInstance();
final SAXParser saxParser = factory.newSAXParser();
final NvdCve12Handler cve12Handler = new NvdCve12Handler();
saxParser.parse(oldVersion, cve12Handler);
final Map<String, List<VulnerableSoftware>> prevVersionVulnMap = cve12Handler.getVulnerabilities();
final NvdCve20Handler cve20Handler = new NvdCve20Handler();
cve20Handler.setCveDB(cveDB);
cve20Handler.setPrevVersionVulnMap(prevVersionVulnMap);
saxParser.parse(file, cve20Handler);
}
}

51
dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/UpdateTask.java
View File

@@ -1,51 +0,0 @@
/*
* This file is part of dependency-check-core.
*
* Dependency-check-core is free software: you can redistribute it and/or modify it
* under the terms of the GNU General Public License as published by the Free
* Software Foundation, either version 3 of the License, or (at your option) any
* later version.
*
* Dependency-check-core is distributed in the hope that it will be useful, but
* WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
* details.
*
* You should have received a copy of the GNU General Public License along with
* dependency-check-core. If not, see http://www.gnu.org/licenses/.
*
* Copyright (c) 2012 Jeremy Long. All Rights Reserved.
*/
package org.owasp.dependencycheck.data.update;
import org.owasp.dependencycheck.data.UpdateException;
/**
* An interface defining an update task.
*
* @author Jeremy Long (jeremy.long@owasp.org)
*/
public interface UpdateTask {
/**
* <p>Updates the data store to the latest version.</p>
*
* @throws UpdateException is thrown if there is an error updating the
* database
*/
void update() throws UpdateException;
/**
* Get the value of deleteAndRecreate.
*
* @return the value of deleteAndRecreate
*/
boolean shouldDeleteAndRecreate();
/**
* Gets whether or not an update is needed.
*
* @return true or false depending on whether an update is needed
*/
boolean isUpdateNeeded();
}

60
dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/UpdateTaskFactory.java
View File

@@ -1,60 +0,0 @@
/*
* This file is part of dependency-check-core.
*
* Dependency-check-core is free software: you can redistribute it and/or modify it
* under the terms of the GNU General Public License as published by the Free
* Software Foundation, either version 3 of the License, or (at your option) any
* later version.
*
* Dependency-check-core is distributed in the hope that it will be useful, but
* WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
* details.
*
* You should have received a copy of the GNU General Public License along with
* dependency-check-core. If not, see http://www.gnu.org/licenses/.
*
* Copyright (c) 2012 Jeremy Long. All Rights Reserved.
*/
package org.owasp.dependencycheck.data.update;
import java.net.MalformedURLException;
import org.owasp.dependencycheck.data.UpdateException;
import org.owasp.dependencycheck.utils.DownloadFailedException;
/**
* An UpdateTask Factory that instantiates the correct UpdateTask based on the
* given configuration.
*
* @author Jeremy Long (jeremy.long@owasp.org)
*/
public final class UpdateTaskFactory {
/**
* private constructor for a utility class.
*/
private UpdateTaskFactory() {
//empty contrusctor for utility class
}
/**
* Constructs the appropriate update task based on configuration.
*
* @return an UpdateTask
* @throws MalformedURLException thrown if a configured URL is malformed
* @throws DownloadFailedException thrown if a timestamp cannot be checked
* on a configured URL
* @throws UpdateException thrown if there is an exception generating the
* update task
*/
public static UpdateTask getUpdateTask() throws MalformedURLException, DownloadFailedException, UpdateException {
final UpdateTask task;
final DataStoreMetaInfo properties = new DataStoreMetaInfo();
// if (properties.isBatchUpdateMode()) {
// task = new BatchUpdateTask(properties);
// } else {
task = new StandardUpdateTask(properties);
// }
return task;
}
}

140
dependency-check-core/src/test/java/org/owasp/dependencycheck/data/update/AbstractUpdateTaskTest.java
View File

@@ -1,140 +0,0 @@
/*
* This file is part of dependency-check-core.
*
* Dependency-check-core is free software: you can redistribute it and/or modify it
* under the terms of the GNU General Public License as published by the Free
* Software Foundation, either version 3 of the License, or (at your option) any
* later version.
*
* Dependency-check-core is distributed in the hope that it will be useful, but
* WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
* details.
*
* You should have received a copy of the GNU General Public License along with
* dependency-check-core. If not, see http://www.gnu.org/licenses/.
*
* Copyright (c) 2013 Jeremy Long. All Rights Reserved.
*/
package org.owasp.dependencycheck.data.update;
import java.io.File;
import java.io.IOException;
import java.net.MalformedURLException;
import java.text.DateFormat;
import java.util.Calendar;
import java.util.logging.Level;
import java.util.logging.Logger;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
import static org.junit.Assert.*;
import org.owasp.dependencycheck.data.UpdateException;
import org.owasp.dependencycheck.utils.DownloadFailedException;
/**
*
* @author Jeremy Long (jeremy.long@owasp.org)
*/
public class AbstractUpdateTaskTest {
public AbstractUpdateTaskTest() {
}
@BeforeClass
public static void setUpClass() {
}
@AfterClass
public static void tearDownClass() {
}
@Before
public void setUp() {
}
@After
public void tearDown() {
}
public AbstractUpdateTask getAbstractUpdateImpl() throws Exception {
DataStoreMetaInfo props = new DataStoreMetaInfo();
AbstractUpdateTask instance = new AbstractUpdateImpl(props);
return instance;
}
/**
* Test of setDeleteAndRecreate method, of class AbstractUpdateTask.
*/
@Test
public void testSetDeleteAndRecreate() throws Exception {
boolean deleteAndRecreate = false;
boolean expResult = false;
AbstractUpdateTask instance = getAbstractUpdateImpl();
instance.setDeleteAndRecreate(deleteAndRecreate);
boolean result = instance.shouldDeleteAndRecreate();
assertEquals(expResult, result);
}
/**
* Test of deleteExistingData method, of class AbstractUpdateTask.
*/
@Test
public void testDeleteExistingData() throws Exception {
AbstractUpdateTask instance = getAbstractUpdateImpl();
Exception result = null;
try {
instance.deleteExistingData();
} catch (IOException ex) {
result = ex;
}
assertNull(result);
}
/**
* Test of openDataStores method, of class AbstractUpdateTask.
*/
@Test
public void testOpenDataStores() throws Exception {
AbstractUpdateTask instance = getAbstractUpdateImpl();
instance.openDataStores();
instance.closeDataStores();
}
/**
* Test of withinRange method, of class AbstractUpdateTask.
*/
@Test
public void testWithinRange() throws Exception {
Calendar c = Calendar.getInstance();
long current = c.getTimeInMillis();
long lastRun = c.getTimeInMillis() - (3 * (1000 * 60 * 60 * 24));
int range = 7; // 7 days
AbstractUpdateTask instance = getAbstractUpdateImpl();
boolean expResult = true;
boolean result = instance.withinRange(lastRun, current, range);
assertEquals(expResult, result);
lastRun = c.getTimeInMillis() - (8 * (1000 * 60 * 60 * 24));
expResult = false;
result = instance.withinRange(lastRun, current, range);
assertEquals(expResult, result);
}
public class AbstractUpdateImpl extends AbstractUpdateTask {
public AbstractUpdateImpl(DataStoreMetaInfo props) throws Exception {
super(props);
}
public Updateable updatesNeeded() throws MalformedURLException, DownloadFailedException, UpdateException {
return null;
}
public void update() throws UpdateException {
}
}
}

121
dependency-check-core/src/test/java/org/owasp/dependencycheck/data/update/BatchUpdateTaskTest.java
View File

@@ -1,121 +0,0 @@
/*
* This file is part of dependency-check-core.
*
* Dependency-check-core is free software: you can redistribute it and/or modify it
* under the terms of the GNU General Public License as published by the Free
* Software Foundation, either version 3 of the License, or (at your option) any
* later version.
*
* Dependency-check-core is distributed in the hope that it will be useful, but
* WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
* details.
*
* You should have received a copy of the GNU General Public License along with
* dependency-check-core. If not, see http://www.gnu.org/licenses/.
*
* Copyright (c) 2013 Jeremy Long. All Rights Reserved.
*/
package org.owasp.dependencycheck.data.update;
import java.io.File;
import java.net.MalformedURLException;
import org.apache.commons.io.FileUtils;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
import static org.junit.Assert.*;
import org.owasp.dependencycheck.data.UpdateException;
import org.owasp.dependencycheck.utils.DownloadFailedException;
import org.owasp.dependencycheck.utils.Settings;
/**
*
* @author Jeremy Long (jeremy.long@owasp.org)
*/
public class BatchUpdateTaskTest {
public BatchUpdateTaskTest() {
}
@BeforeClass
public static void setUpClass() {
}
@AfterClass
public static void tearDownClass() {
}
private String old12;
private String old20;
@Before
public void setUp() throws Exception {
old12 = Settings.getString(Settings.KEYS.CVE_MODIFIED_12_URL);
old20 = Settings.getString(Settings.KEYS.CVE_MODIFIED_20_URL);
File tmp = Settings.getTempDirectory();
if (!tmp.exists()) {
tmp.mkdirs();
}
File dest = new File(tmp, "data.zip");
File file = new File(this.getClass().getClassLoader().getResource("data.zip").toURI());
FileUtils.copyFile(file, dest);
String path = "file:///" + dest.getCanonicalPath();
Settings.setString(Settings.KEYS.BATCH_UPDATE_URL, path);
dest = new File(tmp, "nvdcve-2012.xml");
file = new File(this.getClass().getClassLoader().getResource("nvdcve-2012.xml").toURI());
FileUtils.copyFile(file, dest);
path = "file:///" + dest.getCanonicalPath();
Settings.setString(Settings.KEYS.CVE_MODIFIED_12_URL, path);
dest = new File(tmp, "nvdcve-2.0-2012.xml");
file = new File(this.getClass().getClassLoader().getResource("nvdcve-2.0-2012.xml").toURI());
FileUtils.copyFile(file, dest);
path = "file:///" + dest.getCanonicalPath();
Settings.setString(Settings.KEYS.CVE_MODIFIED_20_URL, path);
}
@After
public void tearDown() {
Settings.setString(Settings.KEYS.CVE_MODIFIED_12_URL, old12);
Settings.setString(Settings.KEYS.CVE_MODIFIED_20_URL, old20);
Settings.setString(Settings.KEYS.BATCH_UPDATE_URL, "");
}
public BatchUpdateTask getBatchUpdateTask() throws MalformedURLException, DownloadFailedException, UpdateException {
DataStoreMetaInfo props = new DataStoreMetaInfo();
BatchUpdateTask instance = new BatchUpdateTask(props);
return instance;
}
/**
* Test of setDoBatchUpdate method, of class BatchUpdateTask.
*/
@Test
public void testSetDoBatchUpdate() throws DownloadFailedException, MalformedURLException, UpdateException {
boolean expected = false;
BatchUpdateTask instance = getBatchUpdateTask();
instance.setDoBatchUpdate(expected);
boolean results = instance.isDoBatchUpdate();
assertEquals(results, expected);
}
/**
* Test of update method, of class BatchUpdateTask.
*/
@Test
public void testUpdate() throws Exception {
BatchUpdateTask instance = getBatchUpdateTask();
//do some setup
instance.setDoBatchUpdate(true);
instance.deleteExistingData();
instance.update(); //no exceptions it worked?
//todo add some actual asserts to check things.
}
}

64
dependency-check-core/src/test/java/org/owasp/dependencycheck/data/update/StandardUpdateTaskIntegrationTest.java
View File

@@ -18,7 +18,9 @@
*/
package org.owasp.dependencycheck.data.update;
import java.io.IOException;
import java.net.MalformedURLException;
import java.util.Calendar;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.Before;
@@ -54,11 +56,69 @@ public class StandardUpdateTaskIntegrationTest {
}
public StandardUpdateTask getStandardUpdateTask() throws MalformedURLException, DownloadFailedException, UpdateException {
DataStoreMetaInfo props = new DataStoreMetaInfo();
StandardUpdateTask instance = new StandardUpdateTask(props);
StandardUpdateTask instance = new StandardUpdateTask();
return instance;
}
/**
* Test of setDeleteAndRecreate method, of class StandardUpdateTask.
*/
@Test
public void testSetDeleteAndRecreate() throws Exception {
boolean deleteAndRecreate = false;
boolean expResult = false;
StandardUpdateTask instance = getStandardUpdateTask();
instance.setDeleteAndRecreate(deleteAndRecreate);
boolean result = instance.shouldDeleteAndRecreate();
assertEquals(expResult, result);
}
/**
* Test of deleteExistingData method, of class StandardUpdateTask.
*/
@Test
public void testDeleteExistingData() throws Exception {
StandardUpdateTask instance = getStandardUpdateTask();
Exception result = null;
try {
instance.deleteExistingData();
} catch (IOException ex) {
result = ex;
}
assertNull(result);
}
/**
* Test of openDataStores method, of class StandardUpdateTask.
*/
@Test
public void testOpenDataStores() throws Exception {
StandardUpdateTask instance = getStandardUpdateTask();
instance.openDataStores();
instance.closeDataStores();
}
/**
* Test of withinRange method, of class StandardUpdateTask.
*/
@Test
public void testWithinRange() throws Exception {
Calendar c = Calendar.getInstance();
long current = c.getTimeInMillis();
long lastRun = c.getTimeInMillis() - (3 * (1000 * 60 * 60 * 24));
int range = 7; // 7 days
StandardUpdateTask instance = getStandardUpdateTask();
boolean expResult = true;
boolean result = instance.withinRange(lastRun, current, range);
assertEquals(expResult, result);
lastRun = c.getTimeInMillis() - (8 * (1000 * 60 * 60 * 24));
expResult = false;
result = instance.withinRange(lastRun, current, range);
assertEquals(expResult, result);
}
/**
* Test of update method, of class StandardUpdateTask.
*/