mirror of
https://github.com/ysoftdevs/DependencyCheck.git
synced 2026-01-14 07:43:40 +01:00
seperated functionality to make the update procedure easier to understand
Former-commit-id: 15e86b665c007af38bf58b47097f94f7ec82bb5a
This commit is contained in:
Jeremy Long
@@ -0,0 +1,241 @@
|
||||
/*
|
||||
* This file is part of dependency-check-core.
|
||||
*
|
||||
* Dependency-check-core is free software: you can redistribute it and/or modify it
|
||||
* under the terms of the GNU General Public License as published by the Free
|
||||
* Software Foundation, either version 3 of the License, or (at your option) any
|
||||
* later version.
|
||||
*
|
||||
* Dependency-check-core is distributed in the hope that it will be useful, but
|
||||
* WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
|
||||
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
|
||||
* details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License along with
|
||||
* dependency-check-core. If not, see http://www.gnu.org/licenses/.
|
||||
*
|
||||
* Copyright (c) 2012 Jeremy Long. All Rights Reserved.
|
||||
*/
|
||||
package org.owasp.dependencycheck.data.update;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.net.MalformedURLException;
|
||||
import org.owasp.dependencycheck.data.CachedWebDataSource;
|
||||
import java.sql.SQLException;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.logging.Level;
|
||||
import java.util.logging.Logger;
|
||||
import javax.xml.parsers.ParserConfigurationException;
|
||||
import javax.xml.parsers.SAXParser;
|
||||
import javax.xml.parsers.SAXParserFactory;
|
||||
import org.owasp.dependencycheck.data.UpdateException;
|
||||
import org.owasp.dependencycheck.data.cpe.CpeIndexWriter;
|
||||
import org.owasp.dependencycheck.data.nvdcve.CveDB;
|
||||
import org.owasp.dependencycheck.utils.FileUtils;
|
||||
import org.owasp.dependencycheck.utils.Settings;
|
||||
import org.owasp.dependencycheck.data.nvdcve.DatabaseException;
|
||||
import org.owasp.dependencycheck.data.nvdcve.NvdCve12Handler;
|
||||
import org.owasp.dependencycheck.data.nvdcve.NvdCve20Handler;
|
||||
import org.owasp.dependencycheck.dependency.VulnerableSoftware;
|
||||
import org.owasp.dependencycheck.utils.DownloadFailedException;
|
||||
import org.xml.sax.SAXException;
|
||||
|
||||
/**
|
||||
* Class responsible for updating the CPE and NVDCVE data stores.
|
||||
*
|
||||
* @author Jeremy Long (jeremy.long@owasp.org)
|
||||
*/
|
||||
public abstract class AbstractUpdate {
|
||||
|
||||
private Updateable updateable;
|
||||
/**
|
||||
* Utility to read and write meta-data about the data.
|
||||
*/
|
||||
protected DataStoreMetaInfo properties = null;
|
||||
/**
|
||||
* Reference to the Cve Database.
|
||||
*/
|
||||
protected CveDB cveDB = null;
|
||||
/**
|
||||
* Reference to the Cpe Index.
|
||||
*/
|
||||
protected CpeIndexWriter cpeIndex = null;
|
||||
|
||||
public AbstractUpdate() throws MalformedURLException, DownloadFailedException, UpdateException {
|
||||
this.updateable = updatesNeeded();
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets whether or not an update is needed.
|
||||
*
|
||||
* @return true or false depending on whether an update is needed
|
||||
*/
|
||||
public boolean isUpdateNeeded() {
|
||||
return updateable.isUpdateNeeded();
|
||||
}
|
||||
|
||||
/**
|
||||
* Determines if the index needs to be updated.
|
||||
*
|
||||
* @return a collection of updateable resources.
|
||||
* @throws MalformedURLException is thrown if the URL for the NVD CVE Meta
|
||||
* data is incorrect.
|
||||
* @throws DownloadFailedException is thrown if there is an error.
|
||||
* downloading the NVD CVE download data file.
|
||||
* @throws UpdateException Is thrown if there is an issue with the last
|
||||
* updated properties file.
|
||||
*/
|
||||
protected abstract Updateable updatesNeeded() throws MalformedURLException, DownloadFailedException, UpdateException;
|
||||
|
||||
/**
|
||||
* <p>Updates the data store to the latest version.</p>
|
||||
*
|
||||
* @throws UpdateException is thrown if there is an error updating the
|
||||
* database
|
||||
*/
|
||||
public abstract void update() throws UpdateException;
|
||||
/**
|
||||
* A flag indicating whether or not the current data store should be
|
||||
* deleted.
|
||||
*/
|
||||
private boolean deleteAndRecreate = false;
|
||||
protected Updateable updatesNeeded = null;
|
||||
|
||||
/**
|
||||
* Get the value of deleteAndRecreate
|
||||
*
|
||||
* @return the value of deleteAndRecreate
|
||||
*/
|
||||
public boolean shouldDeleteAndRecreate() {
|
||||
return deleteAndRecreate;
|
||||
}
|
||||
|
||||
/**
|
||||
* Set the value of deleteAndRecreate
|
||||
*
|
||||
* @param deleteAndRecreate new value of deleteAndRecreate
|
||||
*/
|
||||
public void setDeleteAndRecreate(boolean deleteAndRecreate) {
|
||||
this.deleteAndRecreate = deleteAndRecreate;
|
||||
}
|
||||
|
||||
/**
|
||||
* Deletes the existing data directories.
|
||||
*
|
||||
* @throws IOException thrown if the directory cannot be deleted
|
||||
*/
|
||||
protected void deleteExistingData() throws IOException {
|
||||
File data = Settings.getFile(Settings.KEYS.CVE_DATA_DIRECTORY);
|
||||
if (data.exists()) {
|
||||
FileUtils.delete(data);
|
||||
}
|
||||
data = Settings.getFile(Settings.KEYS.CPE_DATA_DIRECTORY);
|
||||
if (data.exists()) {
|
||||
FileUtils.delete(data);
|
||||
}
|
||||
data = DataStoreMetaInfo.getPropertiesFile();
|
||||
if (data.exists()) {
|
||||
FileUtils.delete(data);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Closes the CVE and CPE data stores.
|
||||
*/
|
||||
protected void closeDataStores() {
|
||||
if (cveDB != null) {
|
||||
try {
|
||||
cveDB.close();
|
||||
} catch (Exception ignore) {
|
||||
Logger.getLogger(AbstractUpdate.class.getName()).log(Level.FINEST, "Error closing the cveDB", ignore);
|
||||
}
|
||||
}
|
||||
if (cpeIndex != null) {
|
||||
try {
|
||||
cpeIndex.close();
|
||||
} catch (Exception ignore) {
|
||||
Logger.getLogger(AbstractUpdate.class.getName()).log(Level.FINEST, "Error closing the cpeIndex", ignore);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Opens the CVE and CPE data stores.
|
||||
*
|
||||
* @throws UpdateException thrown if a data store cannot be opened
|
||||
*/
|
||||
protected void openDataStores() throws UpdateException {
|
||||
//open the cve and cpe data stores
|
||||
try {
|
||||
cveDB = new CveDB();
|
||||
cveDB.open();
|
||||
cpeIndex = new CpeIndexWriter();
|
||||
cpeIndex.open();
|
||||
} catch (IOException ex) {
|
||||
closeDataStores();
|
||||
Logger.getLogger(AbstractUpdate.class.getName()).log(Level.FINE, "IO Error opening databases", ex);
|
||||
throw new UpdateException("Error updating the CPE/CVE data, please see the log file for more details.");
|
||||
} catch (SQLException ex) {
|
||||
closeDataStores();
|
||||
Logger.getLogger(AbstractUpdate.class.getName()).log(Level.FINE, "SQL Exception opening databases", ex);
|
||||
throw new UpdateException("Error updating the CPE/CVE data, please see the log file for more details.");
|
||||
} catch (DatabaseException ex) {
|
||||
closeDataStores();
|
||||
Logger.getLogger(AbstractUpdate.class.getName()).log(Level.FINE, "Database Exception opening databases", ex);
|
||||
throw new UpdateException("Error updating the CPE/CVE data, please see the log file for more details.");
|
||||
} catch (ClassNotFoundException ex) {
|
||||
closeDataStores();
|
||||
Logger.getLogger(AbstractUpdate.class.getName()).log(Level.FINE, "Class not found exception opening databases", ex);
|
||||
throw new UpdateException("Error updating the CPE/CVE data, please see the log file for more details.");
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Determines if the epoch date is within the range specified of the
|
||||
* compareTo epoch time. This takes the (compareTo-date)/1000/60/60/24 to
|
||||
* get the number of days. If the calculated days is less then the range the
|
||||
* date is considered valid.
|
||||
*
|
||||
* @param date the date to be checked.
|
||||
* @param compareTo the date to compare to.
|
||||
* @param range the range in days to be considered valid.
|
||||
* @return whether or not the date is within the range.
|
||||
*/
|
||||
protected boolean withinRange(long date, long compareTo, int range) {
|
||||
final double differenceInDays = (compareTo - date) / 1000.0 / 60.0 / 60.0 / 24.0;
|
||||
return differenceInDays < range;
|
||||
}
|
||||
|
||||
/**
|
||||
* Imports the NVD CVE XML File into the Lucene Index.
|
||||
*
|
||||
* @param file the file containing the NVD CVE XML
|
||||
* @param oldVersion contains the file containing the NVD CVE XML 1.2
|
||||
* @throws ParserConfigurationException is thrown if there is a parser
|
||||
* configuration exception
|
||||
* @throws SAXException is thrown if there is a SAXException
|
||||
* @throws IOException is thrown if there is a IO Exception
|
||||
* @throws SQLException is thrown if there is a SQL exception
|
||||
* @throws DatabaseException is thrown if there is a database exception
|
||||
* @throws ClassNotFoundException thrown if the h2 database driver cannot be
|
||||
* loaded
|
||||
*/
|
||||
protected void importXML(File file, File oldVersion)
|
||||
throws ParserConfigurationException, SAXException, IOException, SQLException, DatabaseException, ClassNotFoundException {
|
||||
|
||||
final SAXParserFactory factory = SAXParserFactory.newInstance();
|
||||
final SAXParser saxParser = factory.newSAXParser();
|
||||
|
||||
final NvdCve12Handler cve12Handler = new NvdCve12Handler();
|
||||
saxParser.parse(oldVersion, cve12Handler);
|
||||
final Map<String, List<VulnerableSoftware>> prevVersionVulnMap = cve12Handler.getVulnerabilities();
|
||||
|
||||
final NvdCve20Handler cve20Handler = new NvdCve20Handler();
|
||||
cve20Handler.setCveDB(cveDB);
|
||||
cve20Handler.setPrevVersionVulnMap(prevVersionVulnMap);
|
||||
cve20Handler.setCpeIndex(cpeIndex);
|
||||
saxParser.parse(file, cve20Handler);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,266 @@
|
||||
/*
|
||||
* This file is part of dependency-check-core.
|
||||
*
|
||||
* Dependency-check-core is free software: you can redistribute it and/or modify it
|
||||
* under the terms of the GNU General Public License as published by the Free
|
||||
* Software Foundation, either version 3 of the License, or (at your option) any
|
||||
* later version.
|
||||
*
|
||||
* Dependency-check-core is distributed in the hope that it will be useful, but
|
||||
* WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
|
||||
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
|
||||
* details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License along with
|
||||
* dependency-check-core. If not, see http://www.gnu.org/licenses/.
|
||||
*
|
||||
* Copyright (c) 2012 Jeremy Long. All Rights Reserved.
|
||||
*/
|
||||
package org.owasp.dependencycheck.data.update;
|
||||
|
||||
import org.owasp.dependencycheck.data.nvdcve.InvalidDataException;
|
||||
import java.io.File;
|
||||
import java.io.FileNotFoundException;
|
||||
import java.io.IOException;
|
||||
import javax.xml.parsers.ParserConfigurationException;
|
||||
import org.xml.sax.SAXException;
|
||||
import java.net.MalformedURLException;
|
||||
import java.net.URISyntaxException;
|
||||
import java.net.URL;
|
||||
import java.sql.SQLException;
|
||||
import java.util.Calendar;
|
||||
import java.util.Date;
|
||||
import java.util.logging.Level;
|
||||
import java.util.logging.Logger;
|
||||
import org.owasp.dependencycheck.data.UpdateException;
|
||||
import org.owasp.dependencycheck.data.nvdcve.CveDB;
|
||||
import org.owasp.dependencycheck.utils.DownloadFailedException;
|
||||
import org.owasp.dependencycheck.utils.Downloader;
|
||||
import org.owasp.dependencycheck.utils.FileUtils;
|
||||
import org.owasp.dependencycheck.utils.Settings;
|
||||
import org.owasp.dependencycheck.data.nvdcve.DatabaseException;
|
||||
import org.owasp.dependencycheck.utils.InvalidSettingException;
|
||||
import static org.owasp.dependencycheck.data.update.DataStoreMetaInfo.BATCH;
|
||||
import static org.owasp.dependencycheck.data.update.DataStoreMetaInfo.MODIFIED;
|
||||
|
||||
/**
|
||||
* Class responsible for updating the CPE and NVDCVE data stores.
|
||||
*
|
||||
* @author Jeremy Long (jeremy.long@owasp.org)
|
||||
*/
|
||||
public class BatchUpdate extends AbstractUpdate {
|
||||
|
||||
public BatchUpdate() throws MalformedURLException, DownloadFailedException, UpdateException {
|
||||
super();
|
||||
}
|
||||
/**
|
||||
* A flag indicating whether or not the batch update should be performed.
|
||||
*/
|
||||
private boolean doBatchUpdate;
|
||||
|
||||
/**
|
||||
* Get the value of doBatchUpdate
|
||||
*
|
||||
* @return the value of doBatchUpdate
|
||||
*/
|
||||
protected boolean isDoBatchUpdate() {
|
||||
return doBatchUpdate;
|
||||
}
|
||||
|
||||
/**
|
||||
* Set the value of doBatchUpdate
|
||||
*
|
||||
* @param doBatchUpdate new value of doBatchUpdate
|
||||
*/
|
||||
protected void setDoBatchUpdate(boolean doBatchUpdate) {
|
||||
this.doBatchUpdate = doBatchUpdate;
|
||||
}
|
||||
|
||||
/**
|
||||
* <p>Downloads the latest NVD CVE XML file from the web and imports it into
|
||||
* the current CVE Database.</p>
|
||||
*
|
||||
* @throws UpdateException is thrown if there is an error updating the
|
||||
* database
|
||||
*/
|
||||
@Override
|
||||
public void update() throws UpdateException {
|
||||
if (properties.isBatchUpdateMode() && doBatchUpdate) {
|
||||
final String batchSrc = Settings.getString(Settings.KEYS.BATCH_UPDATE_URL);
|
||||
File tmp = null;
|
||||
try {
|
||||
deleteExistingData();
|
||||
final File dataDirectory = CveDB.getDataDirectory().getParentFile();
|
||||
final URL batchUrl = new URL(batchSrc);
|
||||
if ("file".equals(batchUrl.getProtocol())) {
|
||||
try {
|
||||
tmp = new File(batchUrl.toURI());
|
||||
} catch (URISyntaxException ex) {
|
||||
final String msg = String.format("Invalid batch update URI: %s", batchSrc);
|
||||
throw new UpdateException(msg, ex);
|
||||
}
|
||||
} else if ("http".equals(batchUrl.getProtocol())
|
||||
|| "https".equals(batchUrl.getProtocol())) {
|
||||
tmp = File.createTempFile("batch_", ".zip");
|
||||
Downloader.fetchFile(batchUrl, tmp);
|
||||
}
|
||||
//TODO add FTP?
|
||||
FileUtils.extractFiles(tmp, dataDirectory);
|
||||
|
||||
} catch (IOException ex) {
|
||||
final String msg = String.format("IO Exception Occured performing batch update using: %s", batchSrc);
|
||||
throw new UpdateException(msg, ex);
|
||||
} finally {
|
||||
if (tmp != null && !tmp.delete()) {
|
||||
tmp.deleteOnExit();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Determines if the index needs to be updated. This is done by fetching the
|
||||
* NVD CVE meta data and checking the last update date. If the data needs to
|
||||
* be refreshed this method will return the NvdCveUrl for the files that
|
||||
* need to be updated.
|
||||
*
|
||||
* @return the collection of files that need to be updated
|
||||
* @throws MalformedURLException is thrown if the URL for the NVD CVE Meta
|
||||
* data is incorrect
|
||||
* @throws DownloadFailedException is thrown if there is an error.
|
||||
* downloading the NVD CVE download data file
|
||||
* @throws UpdateException Is thrown if there is an issue with the last
|
||||
* updated properties file
|
||||
*/
|
||||
@Override
|
||||
public Updateable updatesNeeded() throws MalformedURLException, DownloadFailedException, UpdateException {
|
||||
Updateable updates = null;
|
||||
try {
|
||||
updates = retrieveCurrentTimestampsFromWeb();
|
||||
} catch (InvalidDataException ex) {
|
||||
final String msg = "Unable to retrieve valid timestamp from nvd cve downloads page";
|
||||
Logger.getLogger(BatchUpdate.class.getName()).log(Level.FINE, msg, ex);
|
||||
throw new DownloadFailedException(msg, ex);
|
||||
} catch (InvalidSettingException ex) {
|
||||
Logger.getLogger(BatchUpdate.class.getName()).log(Level.FINE, "Invalid setting found when retrieving timestamps", ex);
|
||||
throw new DownloadFailedException("Invalid settings", ex);
|
||||
}
|
||||
|
||||
if (updates == null) {
|
||||
throw new DownloadFailedException("Unable to retrieve the timestamps of the currently published NVD CVE data");
|
||||
}
|
||||
|
||||
if (!properties.isEmpty()) {
|
||||
try {
|
||||
boolean deleteAndRecreate = false;
|
||||
float version;
|
||||
|
||||
if (properties.getProperty("version") == null) {
|
||||
deleteAndRecreate = true;
|
||||
} else {
|
||||
try {
|
||||
version = Float.parseFloat(properties.getProperty("version"));
|
||||
final float currentVersion = Float.parseFloat(CveDB.DB_SCHEMA_VERSION);
|
||||
if (currentVersion > version) {
|
||||
deleteAndRecreate = true;
|
||||
}
|
||||
} catch (NumberFormatException ex) {
|
||||
deleteAndRecreate = true;
|
||||
}
|
||||
}
|
||||
|
||||
final NvdCveInfo batchInfo = updates.get(BATCH);
|
||||
if (properties.isBatchUpdateMode() && batchInfo != null) {
|
||||
final long lastUpdated = Long.parseLong(properties.getProperty(DataStoreMetaInfo.BATCH, "0"));
|
||||
if (lastUpdated != batchInfo.getTimestamp()) {
|
||||
deleteAndRecreate = true;
|
||||
}
|
||||
}
|
||||
|
||||
if (deleteAndRecreate) {
|
||||
setDoBatchUpdate(properties.isBatchUpdateMode());
|
||||
try {
|
||||
deleteExistingData();
|
||||
} catch (IOException ex) {
|
||||
final String msg = "Unable to delete existing data";
|
||||
Logger.getLogger(BatchUpdate.class.getName()).log(Level.WARNING, msg);
|
||||
Logger.getLogger(BatchUpdate.class.getName()).log(Level.FINE, null, ex);
|
||||
}
|
||||
return updates;
|
||||
}
|
||||
|
||||
final long lastUpdated = Long.parseLong(properties.getProperty(DataStoreMetaInfo.LAST_UPDATED, "0"));
|
||||
final Date now = new Date();
|
||||
final int days = Settings.getInt(Settings.KEYS.CVE_MODIFIED_VALID_FOR_DAYS, 7);
|
||||
final int start = Settings.getInt(Settings.KEYS.CVE_START_YEAR, 2002);
|
||||
final int end = Calendar.getInstance().get(Calendar.YEAR);
|
||||
if (lastUpdated == updates.get(MODIFIED).getTimestamp()) {
|
||||
updates.clear(); //we don't need to update anything.
|
||||
setDoBatchUpdate(properties.isBatchUpdateMode());
|
||||
} else if (withinRange(lastUpdated, now.getTime(), days)) {
|
||||
updates.get(MODIFIED).setNeedsUpdate(true);
|
||||
if (properties.isBatchUpdateMode()) {
|
||||
setDoBatchUpdate(false);
|
||||
} else {
|
||||
for (int i = start; i <= end; i++) {
|
||||
updates.get(String.valueOf(i)).setNeedsUpdate(false);
|
||||
}
|
||||
}
|
||||
} else if (properties.isBatchUpdateMode()) {
|
||||
updates.get(MODIFIED).setNeedsUpdate(true);
|
||||
setDoBatchUpdate(true);
|
||||
} else { //we figure out which of the several XML files need to be downloaded.
|
||||
updates.get(MODIFIED).setNeedsUpdate(false);
|
||||
for (int i = start; i <= end; i++) {
|
||||
final NvdCveInfo cve = updates.get(String.valueOf(i));
|
||||
long currentTimestamp = 0;
|
||||
try {
|
||||
currentTimestamp = Long.parseLong(properties.getProperty(DataStoreMetaInfo.LAST_UPDATED_BASE + String.valueOf(i), "0"));
|
||||
} catch (NumberFormatException ex) {
|
||||
final String msg = String.format("Error parsing '%s' '%s' from nvdcve.lastupdated",
|
||||
DataStoreMetaInfo.LAST_UPDATED_BASE, String.valueOf(i));
|
||||
Logger.getLogger(BatchUpdate.class.getName()).log(Level.FINE, msg, ex);
|
||||
}
|
||||
if (currentTimestamp == cve.getTimestamp()) {
|
||||
cve.setNeedsUpdate(false); //they default to true.
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (NumberFormatException ex) {
|
||||
final String msg = "An invalid schema version or timestamp exists in the data.properties file.";
|
||||
Logger.getLogger(BatchUpdate.class.getName()).log(Level.WARNING, msg);
|
||||
Logger.getLogger(BatchUpdate.class.getName()).log(Level.FINE, null, ex);
|
||||
setDoBatchUpdate(properties.isBatchUpdateMode());
|
||||
}
|
||||
}
|
||||
return updates;
|
||||
}
|
||||
|
||||
/**
|
||||
* Retrieves the timestamps from the NVD CVE meta data file.
|
||||
*
|
||||
* @return the timestamp from the currently published nvdcve downloads page
|
||||
* @throws MalformedURLException thrown if the URL for the NVD CCE Meta data
|
||||
* is incorrect.
|
||||
* @throws DownloadFailedException thrown if there is an error downloading
|
||||
* the nvd cve meta data file
|
||||
* @throws InvalidDataException thrown if there is an exception parsing the
|
||||
* timestamps
|
||||
* @throws InvalidSettingException thrown if the settings are invalid
|
||||
*/
|
||||
private Updateable retrieveCurrentTimestampsFromWeb()
|
||||
throws MalformedURLException, DownloadFailedException, InvalidDataException, InvalidSettingException {
|
||||
Updateable updates = new Updateable();
|
||||
updates.add(BATCH, Settings.getString(Settings.KEYS.BATCH_UPDATE_URL),
|
||||
null, false);
|
||||
|
||||
String url = Settings.getString(Settings.KEYS.CVE_MODIFIED_20_URL, "");
|
||||
if (!url.isEmpty()) {
|
||||
final NvdCveInfo item = new NvdCveInfo();
|
||||
updates.add(MODIFIED, url,
|
||||
Settings.getString(Settings.KEYS.CVE_MODIFIED_12_URL),
|
||||
false);
|
||||
}
|
||||
return updates;
|
||||
}
|
||||
}
|
||||
@@ -18,40 +18,19 @@
|
||||
*/
|
||||
package org.owasp.dependencycheck.data.update;
|
||||
|
||||
import org.owasp.dependencycheck.data.nvdcve.NvdCve12Handler;
|
||||
import org.owasp.dependencycheck.data.nvdcve.NvdCve20Handler;
|
||||
import org.owasp.dependencycheck.data.nvdcve.InvalidDataException;
|
||||
import java.io.File;
|
||||
import java.io.FileNotFoundException;
|
||||
import java.io.IOException;
|
||||
import javax.xml.parsers.ParserConfigurationException;
|
||||
import org.xml.sax.SAXException;
|
||||
import org.owasp.dependencycheck.data.CachedWebDataSource;
|
||||
import java.net.MalformedURLException;
|
||||
import java.net.URISyntaxException;
|
||||
import java.net.URL;
|
||||
import java.sql.SQLException;
|
||||
import java.util.Calendar;
|
||||
import java.util.Date;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.TreeMap;
|
||||
import java.util.logging.Level;
|
||||
import java.util.logging.Logger;
|
||||
import javax.xml.parsers.SAXParser;
|
||||
import javax.xml.parsers.SAXParserFactory;
|
||||
import org.owasp.dependencycheck.concurrency.DirectoryLockException;
|
||||
import org.owasp.dependencycheck.concurrency.DirectorySpinLock;
|
||||
import org.owasp.dependencycheck.concurrency.InvalidDirectoryException;
|
||||
import org.owasp.dependencycheck.data.UpdateException;
|
||||
import org.owasp.dependencycheck.data.cpe.CpeIndexWriter;
|
||||
import org.owasp.dependencycheck.data.nvdcve.CveDB;
|
||||
import org.owasp.dependencycheck.dependency.VulnerableSoftware;
|
||||
import org.owasp.dependencycheck.utils.DownloadFailedException;
|
||||
import org.owasp.dependencycheck.utils.Downloader;
|
||||
import org.owasp.dependencycheck.utils.FileUtils;
|
||||
import org.owasp.dependencycheck.utils.Settings;
|
||||
import org.owasp.dependencycheck.data.nvdcve.DatabaseException;
|
||||
import org.owasp.dependencycheck.utils.InvalidSettingException;
|
||||
import static org.owasp.dependencycheck.data.update.DataStoreMetaInfo.BATCH;
|
||||
import static org.owasp.dependencycheck.data.update.DataStoreMetaInfo.MODIFIED;
|
||||
|
||||
/**
|
||||
* Class responsible for updating the CPE and NVDCVE data stores.
|
||||
@@ -63,37 +42,7 @@ public class DatabaseUpdater implements CachedWebDataSource {
|
||||
/**
|
||||
* Utility to read and write meta-data about the data.
|
||||
*/
|
||||
private DataStoreMetaInfo properties = null;
|
||||
/**
|
||||
* Reference to the Cve Database.
|
||||
*/
|
||||
private CveDB cveDB = null;
|
||||
/**
|
||||
* Reference to the Cpe Index.
|
||||
*/
|
||||
private CpeIndexWriter cpeIndex = null;
|
||||
/**
|
||||
* A flag indicating whether or not the batch update should be performed.
|
||||
*/
|
||||
private boolean doBatchUpdate;
|
||||
|
||||
/**
|
||||
* Get the value of doBatchUpdate
|
||||
*
|
||||
* @return the value of doBatchUpdate
|
||||
*/
|
||||
protected boolean isDoBatchUpdate() {
|
||||
return doBatchUpdate;
|
||||
}
|
||||
|
||||
/**
|
||||
* Set the value of doBatchUpdate
|
||||
*
|
||||
* @param doBatchUpdate new value of doBatchUpdate
|
||||
*/
|
||||
protected void setDoBatchUpdate(boolean doBatchUpdate) {
|
||||
this.doBatchUpdate = doBatchUpdate;
|
||||
}
|
||||
protected DataStoreMetaInfo properties = null;
|
||||
|
||||
/**
|
||||
* <p>Downloads the latest NVD CVE XML file from the web and imports it into
|
||||
@@ -104,149 +53,57 @@ public class DatabaseUpdater implements CachedWebDataSource {
|
||||
*/
|
||||
@Override
|
||||
public void update() throws UpdateException {
|
||||
doBatchUpdate = false;
|
||||
properties = new DataStoreMetaInfo();
|
||||
AbstractUpdate store = null;
|
||||
File dataDir = Settings.getFile(Settings.KEYS.DATA_DIRECTORY);
|
||||
DirectorySpinLock lock = null;
|
||||
try {
|
||||
final Map<String, NvdCveInfo> update = updateNeeded();
|
||||
int maxUpdates = 0;
|
||||
for (NvdCveInfo cve : update.values()) {
|
||||
if (cve.getNeedsUpdate()) {
|
||||
maxUpdates += 1;
|
||||
}
|
||||
}
|
||||
if (maxUpdates > 3 && !properties.isBatchUpdateMode()) {
|
||||
Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.INFO,
|
||||
"NVD CVE requires several updates; this could take a couple of minutes.");
|
||||
}
|
||||
if (maxUpdates > 0 && !isDoBatchUpdate()) {
|
||||
openDataStores();
|
||||
}
|
||||
lock = new DirectorySpinLock(dataDir);
|
||||
} catch (InvalidDirectoryException ex) {
|
||||
throw new UpdateException("Unable to obtain lock on the data directory", ex);
|
||||
} catch (DirectoryLockException ex) {
|
||||
throw new UpdateException("Unable to obtain exclusive lock on the data directory", ex);
|
||||
}
|
||||
|
||||
if (properties.isBatchUpdateMode() && isDoBatchUpdate()) {
|
||||
try {
|
||||
performBatchUpdate();
|
||||
openDataStores();
|
||||
} catch (IOException ex) {
|
||||
throw new UpdateException("Unable to perform batch update", ex);
|
||||
}
|
||||
try {
|
||||
lock.obtainSharedLock();
|
||||
if (properties.isBatchUpdateMode()) {
|
||||
store = new BatchUpdate();
|
||||
} else {
|
||||
store = new StandardUpdate();
|
||||
}
|
||||
|
||||
int count = 0;
|
||||
for (NvdCveInfo cve : update.values()) {
|
||||
if (cve.getNeedsUpdate()) {
|
||||
count += 1;
|
||||
Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.INFO,
|
||||
"Updating NVD CVE ({0} of {1})", new Object[]{count, maxUpdates});
|
||||
URL url = new URL(cve.getUrl());
|
||||
File outputPath = null;
|
||||
File outputPath12 = null;
|
||||
if (store.isUpdateNeeded()) {
|
||||
lock.release();
|
||||
lock.obtainExclusiveLock();
|
||||
if (store.shouldDeleteAndRecreate()) {
|
||||
try {
|
||||
Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.INFO,
|
||||
"Downloading {0}", cve.getUrl());
|
||||
outputPath = File.createTempFile("cve" + cve.getId() + "_", ".xml");
|
||||
Downloader.fetchFile(url, outputPath);
|
||||
|
||||
url = new URL(cve.getOldSchemaVersionUrl());
|
||||
outputPath12 = File.createTempFile("cve_1_2_" + cve.getId() + "_", ".xml");
|
||||
Downloader.fetchFile(url, outputPath12);
|
||||
|
||||
Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.INFO,
|
||||
"Processing {0}", cve.getUrl());
|
||||
|
||||
importXML(outputPath, outputPath12);
|
||||
|
||||
cveDB.commit();
|
||||
cpeIndex.commit();
|
||||
|
||||
properties.save(cve);
|
||||
|
||||
Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.INFO,
|
||||
"Completed update {0} of {1}", new Object[]{count, maxUpdates});
|
||||
} catch (FileNotFoundException ex) {
|
||||
throw new UpdateException(ex);
|
||||
} catch (ParserConfigurationException ex) {
|
||||
throw new UpdateException(ex);
|
||||
} catch (SAXException ex) {
|
||||
throw new UpdateException(ex);
|
||||
deleteExistingData();
|
||||
} catch (IOException ex) {
|
||||
throw new UpdateException(ex);
|
||||
} catch (SQLException ex) {
|
||||
throw new UpdateException(ex);
|
||||
} catch (DatabaseException ex) {
|
||||
throw new UpdateException(ex);
|
||||
} catch (ClassNotFoundException ex) {
|
||||
throw new UpdateException(ex);
|
||||
} finally {
|
||||
boolean deleted = false;
|
||||
try {
|
||||
if (outputPath != null && outputPath.exists()) {
|
||||
deleted = outputPath.delete();
|
||||
}
|
||||
} finally {
|
||||
if (outputPath != null && (outputPath.exists() || !deleted)) {
|
||||
outputPath.deleteOnExit();
|
||||
}
|
||||
}
|
||||
try {
|
||||
deleted = false;
|
||||
if (outputPath12 != null && outputPath12.exists()) {
|
||||
deleted = outputPath12.delete();
|
||||
}
|
||||
} finally {
|
||||
if (outputPath12 != null && (outputPath12.exists() || !deleted)) {
|
||||
outputPath12.deleteOnExit();
|
||||
}
|
||||
}
|
||||
Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.WARNING, "Unable to delete the existing data directory");
|
||||
Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.FINE, null, ex);
|
||||
}
|
||||
}
|
||||
store.update();
|
||||
}
|
||||
if (maxUpdates >= 1) { //ensure the modified file date gets written
|
||||
properties.save(update.get(MODIFIED));
|
||||
cveDB.cleanupDatabase();
|
||||
}
|
||||
if (update.get(BATCH) != null) {
|
||||
properties.save(update.get(BATCH));
|
||||
}
|
||||
} catch (DirectoryLockException ex) {
|
||||
Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.WARNING,
|
||||
"Unable to obtain lock on data directory, unable to update the data to use the most current data.");
|
||||
Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.FINE, null, ex);
|
||||
} catch (MalformedURLException ex) {
|
||||
throw new UpdateException(ex);
|
||||
Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.WARNING,
|
||||
"NVD CVE properties files contain an invalid URL, unable to update the data to use the most current data.");
|
||||
Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.FINE, null, ex);
|
||||
} catch (DownloadFailedException ex) {
|
||||
throw new UpdateException(ex);
|
||||
Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.WARNING,
|
||||
"Unable to download the NVD CVE data, unable to update the data to use the most current data.");
|
||||
Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.FINE, null, ex);
|
||||
} finally {
|
||||
closeDataStores();
|
||||
if (lock != null) {
|
||||
lock.release();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Imports the NVD CVE XML File into the Lucene Index.
|
||||
*
|
||||
* @param file the file containing the NVD CVE XML
|
||||
* @param oldVersion contains the file containing the NVD CVE XML 1.2
|
||||
* @throws ParserConfigurationException is thrown if there is a parser
|
||||
* configuration exception
|
||||
* @throws SAXException is thrown if there is a SAXException
|
||||
* @throws IOException is thrown if there is a IO Exception
|
||||
* @throws SQLException is thrown if there is a SQL exception
|
||||
* @throws DatabaseException is thrown if there is a database exception
|
||||
* @throws ClassNotFoundException thrown if the h2 database driver cannot be
|
||||
* loaded
|
||||
*/
|
||||
private void importXML(File file, File oldVersion)
|
||||
throws ParserConfigurationException, SAXException, IOException, SQLException, DatabaseException, ClassNotFoundException {
|
||||
|
||||
final SAXParserFactory factory = SAXParserFactory.newInstance();
|
||||
final SAXParser saxParser = factory.newSAXParser();
|
||||
|
||||
final NvdCve12Handler cve12Handler = new NvdCve12Handler();
|
||||
saxParser.parse(oldVersion, cve12Handler);
|
||||
final Map<String, List<VulnerableSoftware>> prevVersionVulnMap = cve12Handler.getVulnerabilities();
|
||||
|
||||
final NvdCve20Handler cve20Handler = new NvdCve20Handler();
|
||||
cve20Handler.setCveDB(cveDB);
|
||||
cve20Handler.setPrevVersionVulnMap(prevVersionVulnMap);
|
||||
cve20Handler.setCpeIndex(cpeIndex);
|
||||
saxParser.parse(file, cve20Handler);
|
||||
}
|
||||
|
||||
/**
|
||||
* Deletes the existing data directories.
|
||||
*
|
||||
@@ -266,307 +123,4 @@ public class DatabaseUpdater implements CachedWebDataSource {
|
||||
FileUtils.delete(data);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Performs the batch update based on the configured batch update URL.
|
||||
*
|
||||
* @throws UpdateException thrown if there is an exception during the update
|
||||
* process
|
||||
*/
|
||||
private void performBatchUpdate() throws UpdateException {
|
||||
if (properties.isBatchUpdateMode() && doBatchUpdate) {
|
||||
final String batchSrc = Settings.getString(Settings.KEYS.BATCH_UPDATE_URL);
|
||||
File tmp = null;
|
||||
try {
|
||||
deleteExistingData();
|
||||
final File dataDirectory = CveDB.getDataDirectory().getParentFile();
|
||||
final URL batchUrl = new URL(batchSrc);
|
||||
if ("file".equals(batchUrl.getProtocol())) {
|
||||
try {
|
||||
tmp = new File(batchUrl.toURI());
|
||||
} catch (URISyntaxException ex) {
|
||||
final String msg = String.format("Invalid batch update URI: %s", batchSrc);
|
||||
throw new UpdateException(msg, ex);
|
||||
}
|
||||
} else if ("http".equals(batchUrl.getProtocol())
|
||||
|| "https".equals(batchUrl.getProtocol())) {
|
||||
tmp = File.createTempFile("batch_", ".zip");
|
||||
Downloader.fetchFile(batchUrl, tmp);
|
||||
}
|
||||
//TODO add FTP?
|
||||
FileUtils.extractFiles(tmp, dataDirectory);
|
||||
|
||||
} catch (IOException ex) {
|
||||
final String msg = String.format("IO Exception Occured performing batch update using: %s", batchSrc);
|
||||
throw new UpdateException(msg, ex);
|
||||
} finally {
|
||||
if (tmp != null && !tmp.delete()) {
|
||||
tmp.deleteOnExit();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Closes the CVE and CPE data stores.
|
||||
*/
|
||||
private void closeDataStores() {
|
||||
if (cveDB != null) {
|
||||
try {
|
||||
cveDB.close();
|
||||
} catch (Exception ignore) {
|
||||
Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.FINEST, "Error closing the cveDB", ignore);
|
||||
}
|
||||
}
|
||||
if (cpeIndex != null) {
|
||||
try {
|
||||
cpeIndex.close();
|
||||
} catch (Exception ignore) {
|
||||
Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.FINEST, "Error closing the cpeIndex", ignore);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Opens the CVE and CPE data stores.
|
||||
*
|
||||
* @throws UpdateException thrown if a data store cannot be opened
|
||||
*/
|
||||
private void openDataStores() throws UpdateException {
|
||||
//open the cve and cpe data stores
|
||||
try {
|
||||
cveDB = new CveDB();
|
||||
cveDB.open();
|
||||
cpeIndex = new CpeIndexWriter();
|
||||
cpeIndex.open();
|
||||
} catch (IOException ex) {
|
||||
closeDataStores();
|
||||
Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.FINE, "IO Error opening databases", ex);
|
||||
throw new UpdateException("Error updating the CPE/CVE data, please see the log file for more details.");
|
||||
} catch (SQLException ex) {
|
||||
closeDataStores();
|
||||
Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.FINE, "SQL Exception opening databases", ex);
|
||||
throw new UpdateException("Error updating the CPE/CVE data, please see the log file for more details.");
|
||||
} catch (DatabaseException ex) {
|
||||
closeDataStores();
|
||||
Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.FINE, "Database Exception opening databases", ex);
|
||||
throw new UpdateException("Error updating the CPE/CVE data, please see the log file for more details.");
|
||||
} catch (ClassNotFoundException ex) {
|
||||
closeDataStores();
|
||||
Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.FINE, "Class not found exception opening databases", ex);
|
||||
throw new UpdateException("Error updating the CPE/CVE data, please see the log file for more details.");
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Determines if the index needs to be updated. This is done by fetching the
|
||||
* NVD CVE meta data and checking the last update date. If the data needs to
|
||||
* be refreshed this method will return the NvdCveUrl for the files that
|
||||
* need to be updated.
|
||||
*
|
||||
* @return the NvdCveUrl of the files that need to be updated.
|
||||
* @throws MalformedURLException is thrown if the URL for the NVD CVE Meta
|
||||
* data is incorrect.
|
||||
* @throws DownloadFailedException is thrown if there is an error.
|
||||
* downloading the NVD CVE download data file.
|
||||
* @throws UpdateException Is thrown if there is an issue with the last
|
||||
* updated properties file.
|
||||
*/
|
||||
private Map<String, NvdCveInfo> updateNeeded() throws MalformedURLException, DownloadFailedException, UpdateException {
|
||||
|
||||
Map<String, NvdCveInfo> currentlyPublished;
|
||||
try {
|
||||
currentlyPublished = retrieveCurrentTimestampsFromWeb();
|
||||
} catch (InvalidDataException ex) {
|
||||
final String msg = "Unable to retrieve valid timestamp from nvd cve downloads page";
|
||||
Logger.getLogger(DataStoreMetaInfo.class.getName()).log(Level.FINE, msg, ex);
|
||||
throw new DownloadFailedException(msg, ex);
|
||||
} catch (InvalidSettingException ex) {
|
||||
Logger.getLogger(DataStoreMetaInfo.class.getName()).log(Level.FINE, "Invalid setting found when retrieving timestamps", ex);
|
||||
throw new DownloadFailedException("Invalid settings", ex);
|
||||
}
|
||||
|
||||
if (currentlyPublished == null) {
|
||||
throw new DownloadFailedException("Unable to retrieve the timestamps of the currently published NVD CVE data");
|
||||
}
|
||||
|
||||
// final File cpeDataDirectory;
|
||||
// try {
|
||||
// cpeDataDirectory = CveDB.getDataDirectory();
|
||||
// } catch (IOException ex) {
|
||||
// String msg;
|
||||
// try {
|
||||
// msg = String.format("Unable to create the CVE Data Directory '%s'",
|
||||
// Settings.getFile(Settings.KEYS.CVE_DATA_DIRECTORY).getCanonicalPath());
|
||||
// } catch (IOException ex1) {
|
||||
// msg = String.format("Unable to create the CVE Data Directory, this is likely a configuration issue: '%s%s%s'",
|
||||
// Settings.getString(Settings.KEYS.DATA_DIRECTORY, ""),
|
||||
// File.separator,
|
||||
// Settings.getString(Settings.KEYS.CVE_DATA_DIRECTORY, ""));
|
||||
// }
|
||||
// throw new UpdateException(msg, ex);
|
||||
// }
|
||||
|
||||
if (!properties.isEmpty()) {
|
||||
try {
|
||||
boolean deleteAndRecreate = false;
|
||||
float version;
|
||||
|
||||
if (properties.getProperty("version") == null) {
|
||||
deleteAndRecreate = true;
|
||||
} else {
|
||||
try {
|
||||
version = Float.parseFloat(properties.getProperty("version"));
|
||||
final float currentVersion = Float.parseFloat(CveDB.DB_SCHEMA_VERSION);
|
||||
if (currentVersion > version) {
|
||||
deleteAndRecreate = true;
|
||||
}
|
||||
} catch (NumberFormatException ex) {
|
||||
deleteAndRecreate = true;
|
||||
}
|
||||
}
|
||||
|
||||
final NvdCveInfo batchInfo = currentlyPublished.get(BATCH);
|
||||
if (properties.isBatchUpdateMode() && batchInfo != null) {
|
||||
final long lastUpdated = Long.parseLong(properties.getProperty(DataStoreMetaInfo.BATCH, "0"));
|
||||
if (lastUpdated != batchInfo.getTimestamp()) {
|
||||
deleteAndRecreate = true;
|
||||
}
|
||||
}
|
||||
|
||||
if (deleteAndRecreate) {
|
||||
setDoBatchUpdate(properties.isBatchUpdateMode());
|
||||
try {
|
||||
deleteExistingData();
|
||||
} catch (IOException ex) {
|
||||
final String msg = "Unable to delete existing data";
|
||||
Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.WARNING, msg);
|
||||
Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.FINE, null, ex);
|
||||
}
|
||||
return currentlyPublished;
|
||||
}
|
||||
|
||||
final long lastUpdated = Long.parseLong(properties.getProperty(DataStoreMetaInfo.LAST_UPDATED, "0"));
|
||||
final Date now = new Date();
|
||||
final int days = Settings.getInt(Settings.KEYS.CVE_MODIFIED_VALID_FOR_DAYS, 7);
|
||||
final int start = Settings.getInt(Settings.KEYS.CVE_START_YEAR, 2002);
|
||||
final int end = Calendar.getInstance().get(Calendar.YEAR);
|
||||
if (lastUpdated == currentlyPublished.get(MODIFIED).getTimestamp()) {
|
||||
currentlyPublished.clear(); //we don't need to update anything.
|
||||
setDoBatchUpdate(properties.isBatchUpdateMode());
|
||||
} else if (withinRange(lastUpdated, now.getTime(), days)) {
|
||||
currentlyPublished.get(MODIFIED).setNeedsUpdate(true);
|
||||
if (properties.isBatchUpdateMode()) {
|
||||
setDoBatchUpdate(false);
|
||||
} else {
|
||||
for (int i = start; i <= end; i++) {
|
||||
currentlyPublished.get(String.valueOf(i)).setNeedsUpdate(false);
|
||||
}
|
||||
}
|
||||
} else if (properties.isBatchUpdateMode()) {
|
||||
currentlyPublished.get(MODIFIED).setNeedsUpdate(true);
|
||||
setDoBatchUpdate(true);
|
||||
} else { //we figure out which of the several XML files need to be downloaded.
|
||||
currentlyPublished.get(MODIFIED).setNeedsUpdate(false);
|
||||
for (int i = start; i <= end; i++) {
|
||||
final NvdCveInfo cve = currentlyPublished.get(String.valueOf(i));
|
||||
long currentTimestamp = 0;
|
||||
try {
|
||||
currentTimestamp = Long.parseLong(properties.getProperty(DataStoreMetaInfo.LAST_UPDATED_BASE + String.valueOf(i), "0"));
|
||||
} catch (NumberFormatException ex) {
|
||||
final String msg = String.format("Error parsing '%s' '%s' from nvdcve.lastupdated",
|
||||
DataStoreMetaInfo.LAST_UPDATED_BASE, String.valueOf(i));
|
||||
Logger.getLogger(DataStoreMetaInfo.class.getName()).log(Level.FINE, msg, ex);
|
||||
}
|
||||
if (currentTimestamp == cve.getTimestamp()) {
|
||||
cve.setNeedsUpdate(false); //they default to true.
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (NumberFormatException ex) {
|
||||
final String msg = "An invalid schema version or timestamp exists in the data.properties file.";
|
||||
Logger.getLogger(DataStoreMetaInfo.class.getName()).log(Level.WARNING, msg);
|
||||
Logger.getLogger(DataStoreMetaInfo.class.getName()).log(Level.FINE, null, ex);
|
||||
setDoBatchUpdate(properties.isBatchUpdateMode());
|
||||
}
|
||||
} else {
|
||||
setDoBatchUpdate(properties.isBatchUpdateMode());
|
||||
}
|
||||
return currentlyPublished;
|
||||
}
|
||||
|
||||
/**
|
||||
* Determines if the epoch date is within the range specified of the
|
||||
* compareTo epoch time. This takes the (compareTo-date)/1000/60/60/24 to
|
||||
* get the number of days. If the calculated days is less then the range the
|
||||
* date is considered valid.
|
||||
*
|
||||
* @param date the date to be checked.
|
||||
* @param compareTo the date to compare to.
|
||||
* @param range the range in days to be considered valid.
|
||||
* @return whether or not the date is within the range.
|
||||
*/
|
||||
private boolean withinRange(long date, long compareTo, int range) {
|
||||
final double differenceInDays = (compareTo - date) / 1000.0 / 60.0 / 60.0 / 24.0;
|
||||
return differenceInDays < range;
|
||||
}
|
||||
|
||||
/**
|
||||
* Retrieves the timestamps from the NVD CVE meta data file.
|
||||
*
|
||||
* @return the timestamp from the currently published nvdcve downloads page
|
||||
* @throws MalformedURLException thrown if the URL for the NVD CCE Meta data
|
||||
* is incorrect.
|
||||
* @throws DownloadFailedException thrown if there is an error downloading
|
||||
* the nvd cve meta data file
|
||||
* @throws InvalidDataException thrown if there is an exception parsing the
|
||||
* timestamps
|
||||
* @throws InvalidSettingException thrown if the settings are invalid
|
||||
*/
|
||||
private Map<String, NvdCveInfo> retrieveCurrentTimestampsFromWeb()
|
||||
throws MalformedURLException, DownloadFailedException, InvalidDataException, InvalidSettingException {
|
||||
|
||||
final Map<String, NvdCveInfo> map = new TreeMap<String, NvdCveInfo>();
|
||||
String retrieveUrl = Settings.getString(Settings.KEYS.CVE_MODIFIED_20_URL);
|
||||
if (retrieveUrl == null && properties.isBatchUpdateMode()) {
|
||||
final NvdCveInfo item = new NvdCveInfo();
|
||||
retrieveUrl = Settings.getString(Settings.KEYS.BATCH_UPDATE_URL);
|
||||
if (retrieveUrl == null) {
|
||||
final String msg = "Invalid configuration - neither the modified or batch update URLs are specified in the configuration.";
|
||||
Logger.getLogger(DataStoreMetaInfo.class.getName()).log(Level.SEVERE, msg);
|
||||
throw new InvalidSettingException(msg);
|
||||
}
|
||||
item.setTimestamp(Downloader.getLastModified(new URL(retrieveUrl)));
|
||||
item.setId(BATCH);
|
||||
item.setNeedsUpdate(false);
|
||||
map.put(BATCH, item);
|
||||
} else {
|
||||
NvdCveInfo item = new NvdCveInfo();
|
||||
item.setNeedsUpdate(false); //the others default to true, to make life easier later this should default to false.
|
||||
item.setId(MODIFIED);
|
||||
item.setUrl(retrieveUrl);
|
||||
item.setOldSchemaVersionUrl(Settings.getString(Settings.KEYS.CVE_MODIFIED_12_URL));
|
||||
|
||||
item.setTimestamp(Downloader.getLastModified(new URL(retrieveUrl)));
|
||||
map.put(MODIFIED, item);
|
||||
|
||||
//only add these urls if we are not in batch mode
|
||||
if (!properties.isBatchUpdateMode()) {
|
||||
final int start = Settings.getInt(Settings.KEYS.CVE_START_YEAR);
|
||||
final int end = Calendar.getInstance().get(Calendar.YEAR);
|
||||
final String baseUrl20 = Settings.getString(Settings.KEYS.CVE_SCHEMA_2_0);
|
||||
final String baseUrl12 = Settings.getString(Settings.KEYS.CVE_SCHEMA_1_2);
|
||||
for (int i = start; i <= end; i++) {
|
||||
retrieveUrl = String.format(baseUrl20, i);
|
||||
item = new NvdCveInfo();
|
||||
item.setId(Integer.toString(i));
|
||||
item.setUrl(retrieveUrl);
|
||||
item.setOldSchemaVersionUrl(String.format(baseUrl12, i));
|
||||
item.setTimestamp(Downloader.getLastModified(new URL(retrieveUrl)));
|
||||
map.put(item.getId(), item);
|
||||
}
|
||||
}
|
||||
}
|
||||
return map;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,292 @@
|
||||
/*
|
||||
* This file is part of dependency-check-core.
|
||||
*
|
||||
* Dependency-check-core is free software: you can redistribute it and/or modify it
|
||||
* under the terms of the GNU General Public License as published by the Free
|
||||
* Software Foundation, either version 3 of the License, or (at your option) any
|
||||
* later version.
|
||||
*
|
||||
* Dependency-check-core is distributed in the hope that it will be useful, but
|
||||
* WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
|
||||
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
|
||||
* details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License along with
|
||||
* dependency-check-core. If not, see http://www.gnu.org/licenses/.
|
||||
*
|
||||
* Copyright (c) 2012 Jeremy Long. All Rights Reserved.
|
||||
*/
|
||||
package org.owasp.dependencycheck.data.update;
|
||||
|
||||
import org.owasp.dependencycheck.data.nvdcve.InvalidDataException;
|
||||
import java.io.File;
|
||||
import java.io.FileNotFoundException;
|
||||
import java.io.IOException;
|
||||
import javax.xml.parsers.ParserConfigurationException;
|
||||
import org.xml.sax.SAXException;
|
||||
import java.net.MalformedURLException;
|
||||
import java.net.URL;
|
||||
import java.sql.SQLException;
|
||||
import java.util.Calendar;
|
||||
import java.util.Date;
|
||||
import java.util.logging.Level;
|
||||
import java.util.logging.Logger;
|
||||
import org.owasp.dependencycheck.data.UpdateException;
|
||||
import org.owasp.dependencycheck.data.nvdcve.CveDB;
|
||||
import org.owasp.dependencycheck.utils.DownloadFailedException;
|
||||
import org.owasp.dependencycheck.utils.Downloader;
|
||||
import org.owasp.dependencycheck.utils.Settings;
|
||||
import org.owasp.dependencycheck.data.nvdcve.DatabaseException;
|
||||
import org.owasp.dependencycheck.utils.InvalidSettingException;
|
||||
import static org.owasp.dependencycheck.data.update.DataStoreMetaInfo.MODIFIED;
|
||||
|
||||
/**
|
||||
* Class responsible for updating the CPE and NVDCVE data stores.
|
||||
*
|
||||
* @author Jeremy Long (jeremy.long@owasp.org)
|
||||
*/
|
||||
public class StandardUpdate extends AbstractUpdate {
|
||||
|
||||
public StandardUpdate() throws MalformedURLException, DownloadFailedException, UpdateException {
|
||||
super();
|
||||
}
|
||||
|
||||
/**
|
||||
* <p>Downloads the latest NVD CVE XML file from the web and imports it into
|
||||
* the current CVE Database.</p>
|
||||
*
|
||||
* @param updatesNeeded a collection of NvdCveInfo containing information
|
||||
* about needed updates.
|
||||
* @throws UpdateException is thrown if there is an error updating the
|
||||
* database
|
||||
*/
|
||||
@Override
|
||||
public void update() throws UpdateException {
|
||||
try {
|
||||
properties = new DataStoreMetaInfo();
|
||||
int maxUpdates = 0;
|
||||
for (NvdCveInfo cve : updatesNeeded) {
|
||||
if (cve.getNeedsUpdate()) {
|
||||
maxUpdates += 1;
|
||||
}
|
||||
}
|
||||
if (maxUpdates > 3) {
|
||||
Logger.getLogger(StandardUpdate.class.getName()).log(Level.INFO,
|
||||
"NVD CVE requires several updates; this could take a couple of minutes.");
|
||||
}
|
||||
if (maxUpdates > 0) {
|
||||
openDataStores();
|
||||
}
|
||||
|
||||
int count = 0;
|
||||
for (NvdCveInfo cve : updatesNeeded) {
|
||||
if (cve.getNeedsUpdate()) {
|
||||
count += 1;
|
||||
Logger.getLogger(StandardUpdate.class.getName()).log(Level.INFO,
|
||||
"Updating NVD CVE ({0} of {1})", new Object[]{count, maxUpdates});
|
||||
URL url = new URL(cve.getUrl());
|
||||
File outputPath = null;
|
||||
File outputPath12 = null;
|
||||
try {
|
||||
Logger.getLogger(StandardUpdate.class.getName()).log(Level.INFO,
|
||||
"Downloading {0}", cve.getUrl());
|
||||
outputPath = File.createTempFile("cve" + cve.getId() + "_", ".xml");
|
||||
Downloader.fetchFile(url, outputPath);
|
||||
|
||||
url = new URL(cve.getOldSchemaVersionUrl());
|
||||
outputPath12 = File.createTempFile("cve_1_2_" + cve.getId() + "_", ".xml");
|
||||
Downloader.fetchFile(url, outputPath12);
|
||||
|
||||
Logger.getLogger(StandardUpdate.class.getName()).log(Level.INFO,
|
||||
"Processing {0}", cve.getUrl());
|
||||
|
||||
importXML(outputPath, outputPath12);
|
||||
|
||||
cveDB.commit();
|
||||
cpeIndex.commit();
|
||||
|
||||
properties.save(cve);
|
||||
|
||||
Logger.getLogger(StandardUpdate.class.getName()).log(Level.INFO,
|
||||
"Completed update {0} of {1}", new Object[]{count, maxUpdates});
|
||||
} catch (FileNotFoundException ex) {
|
||||
throw new UpdateException(ex);
|
||||
} catch (ParserConfigurationException ex) {
|
||||
throw new UpdateException(ex);
|
||||
} catch (SAXException ex) {
|
||||
throw new UpdateException(ex);
|
||||
} catch (IOException ex) {
|
||||
throw new UpdateException(ex);
|
||||
} catch (SQLException ex) {
|
||||
throw new UpdateException(ex);
|
||||
} catch (DatabaseException ex) {
|
||||
throw new UpdateException(ex);
|
||||
} catch (ClassNotFoundException ex) {
|
||||
throw new UpdateException(ex);
|
||||
} finally {
|
||||
boolean deleted = false;
|
||||
try {
|
||||
if (outputPath != null && outputPath.exists()) {
|
||||
deleted = outputPath.delete();
|
||||
}
|
||||
} finally {
|
||||
if (outputPath != null && (outputPath.exists() || !deleted)) {
|
||||
outputPath.deleteOnExit();
|
||||
}
|
||||
}
|
||||
try {
|
||||
deleted = false;
|
||||
if (outputPath12 != null && outputPath12.exists()) {
|
||||
deleted = outputPath12.delete();
|
||||
}
|
||||
} finally {
|
||||
if (outputPath12 != null && (outputPath12.exists() || !deleted)) {
|
||||
outputPath12.deleteOnExit();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if (maxUpdates >= 1) { //ensure the modified file date gets written
|
||||
properties.save(updatesNeeded.get(MODIFIED));
|
||||
cveDB.cleanupDatabase();
|
||||
}
|
||||
} catch (MalformedURLException ex) {
|
||||
throw new UpdateException(ex);
|
||||
} finally {
|
||||
closeDataStores();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Determines if the index needs to be updated. This is done by fetching the
|
||||
* NVD CVE meta data and checking the last update date. If the data needs to
|
||||
* be refreshed this method will return the NvdCveUrl for the files that
|
||||
* need to be updated.
|
||||
*
|
||||
* @return the collection of files that need to be updated
|
||||
* @throws MalformedURLException is thrown if the URL for the NVD CVE Meta
|
||||
* data is incorrect
|
||||
* @throws DownloadFailedException is thrown if there is an error.
|
||||
* downloading the NVD CVE download data file
|
||||
* @throws UpdateException Is thrown if there is an issue with the last
|
||||
* updated properties file
|
||||
*/
|
||||
@Override
|
||||
protected Updateable updatesNeeded() throws MalformedURLException, DownloadFailedException, UpdateException {
|
||||
Updateable updates = null;
|
||||
try {
|
||||
updates = retrieveCurrentTimestampsFromWeb();
|
||||
} catch (InvalidDataException ex) {
|
||||
final String msg = "Unable to retrieve valid timestamp from nvd cve downloads page";
|
||||
Logger.getLogger(StandardUpdate.class.getName()).log(Level.FINE, msg, ex);
|
||||
throw new DownloadFailedException(msg, ex);
|
||||
} catch (InvalidSettingException ex) {
|
||||
Logger.getLogger(StandardUpdate.class.getName()).log(Level.FINE, "Invalid setting found when retrieving timestamps", ex);
|
||||
throw new DownloadFailedException("Invalid settings", ex);
|
||||
}
|
||||
|
||||
if (updates == null) {
|
||||
throw new DownloadFailedException("Unable to retrieve the timestamps of the currently published NVD CVE data");
|
||||
}
|
||||
|
||||
if (!properties.isEmpty()) {
|
||||
try {
|
||||
float version;
|
||||
|
||||
if (properties.getProperty("version") == null) {
|
||||
setDeleteAndRecreate(true);
|
||||
} else {
|
||||
try {
|
||||
version = Float.parseFloat(properties.getProperty("version"));
|
||||
final float currentVersion = Float.parseFloat(CveDB.DB_SCHEMA_VERSION);
|
||||
if (currentVersion > version) {
|
||||
setDeleteAndRecreate(true);
|
||||
}
|
||||
} catch (NumberFormatException ex) {
|
||||
setDeleteAndRecreate(true);
|
||||
}
|
||||
}
|
||||
|
||||
if (shouldDeleteAndRecreate()) {
|
||||
return updates;
|
||||
}
|
||||
|
||||
final long lastUpdated = Long.parseLong(properties.getProperty(DataStoreMetaInfo.LAST_UPDATED, "0"));
|
||||
final Date now = new Date();
|
||||
final int days = Settings.getInt(Settings.KEYS.CVE_MODIFIED_VALID_FOR_DAYS, 7);
|
||||
final int start = Settings.getInt(Settings.KEYS.CVE_START_YEAR, 2002);
|
||||
final int end = Calendar.getInstance().get(Calendar.YEAR);
|
||||
if (lastUpdated == updates.getTimeStamp(MODIFIED)) {
|
||||
updates.clear(); //we don't need to update anything.
|
||||
} else if (withinRange(lastUpdated, now.getTime(), days)) {
|
||||
for (NvdCveInfo entry : updates) {
|
||||
if (MODIFIED.equals(entry.getId())) {
|
||||
entry.setNeedsUpdate(true);
|
||||
} else {
|
||||
entry.setNeedsUpdate(false);
|
||||
}
|
||||
}
|
||||
} else { //we figure out which of the several XML files need to be downloaded.
|
||||
for (NvdCveInfo entry : updates) {
|
||||
if (MODIFIED.equals(entry.getId())) {
|
||||
entry.setNeedsUpdate(true);
|
||||
} else {
|
||||
long currentTimestamp = 0;
|
||||
try {
|
||||
currentTimestamp = Long.parseLong(properties.getProperty(DataStoreMetaInfo.LAST_UPDATED_BASE + entry.getId(), "0"));
|
||||
} catch (NumberFormatException ex) {
|
||||
final String msg = String.format("Error parsing '%s' '%s' from nvdcve.lastupdated",
|
||||
DataStoreMetaInfo.LAST_UPDATED_BASE, entry.getId());
|
||||
Logger.getLogger(StandardUpdate.class.getName()).log(Level.FINE, msg, ex);
|
||||
}
|
||||
if (currentTimestamp == entry.getTimestamp()) {
|
||||
entry.setNeedsUpdate(false);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (NumberFormatException ex) {
|
||||
final String msg = "An invalid schema version or timestamp exists in the data.properties file.";
|
||||
Logger.getLogger(StandardUpdate.class.getName()).log(Level.WARNING, msg);
|
||||
Logger.getLogger(StandardUpdate.class.getName()).log(Level.FINE, null, ex);
|
||||
}
|
||||
}
|
||||
return updates;
|
||||
}
|
||||
|
||||
/**
|
||||
* Retrieves the timestamps from the NVD CVE meta data file.
|
||||
*
|
||||
* @return the timestamp from the currently published nvdcve downloads page
|
||||
* @throws MalformedURLException thrown if the URL for the NVD CCE Meta data
|
||||
* is incorrect.
|
||||
* @throws DownloadFailedException thrown if there is an error downloading
|
||||
* the nvd cve meta data file
|
||||
* @throws InvalidDataException thrown if there is an exception parsing the
|
||||
* timestamps
|
||||
* @throws InvalidSettingException thrown if the settings are invalid
|
||||
*/
|
||||
private Updateable retrieveCurrentTimestampsFromWeb()
|
||||
throws MalformedURLException, DownloadFailedException, InvalidDataException, InvalidSettingException {
|
||||
|
||||
Updateable updates = new Updateable();
|
||||
updates.add(MODIFIED, Settings.getString(Settings.KEYS.CVE_MODIFIED_20_URL),
|
||||
Settings.getString(Settings.KEYS.CVE_MODIFIED_12_URL),
|
||||
false);
|
||||
|
||||
//only add these urls if we are not in batch mode
|
||||
if (!properties.isBatchUpdateMode()) {
|
||||
final int start = Settings.getInt(Settings.KEYS.CVE_START_YEAR);
|
||||
final int end = Calendar.getInstance().get(Calendar.YEAR);
|
||||
final String baseUrl20 = Settings.getString(Settings.KEYS.CVE_SCHEMA_2_0);
|
||||
final String baseUrl12 = Settings.getString(Settings.KEYS.CVE_SCHEMA_1_2);
|
||||
for (int i = start; i <= end; i++) {
|
||||
updates.add(Integer.toString(i), String.format(baseUrl20, i),
|
||||
String.format(baseUrl12, i),
|
||||
true);
|
||||
}
|
||||
}
|
||||
return updates;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,152 @@
|
||||
/*
|
||||
* To change this template, choose Tools | Templates
|
||||
* and open the template in the editor.
|
||||
*/
|
||||
package org.owasp.dependencycheck.data.update;
|
||||
|
||||
import java.net.MalformedURLException;
|
||||
import java.net.URL;
|
||||
import java.util.Iterator;
|
||||
import java.util.Map;
|
||||
import java.util.Map.Entry;
|
||||
import java.util.TreeMap;
|
||||
import org.owasp.dependencycheck.utils.DownloadFailedException;
|
||||
import org.owasp.dependencycheck.utils.Downloader;
|
||||
|
||||
/**
|
||||
* Contains a collection of updateable NvdCveInfo objects. This is used to
|
||||
* determine which files need to be downloaded and processed.
|
||||
*
|
||||
* @author Jeremy Long (jeremy.long@owasp.org)
|
||||
*/
|
||||
public class Updateable implements java.lang.Iterable<NvdCveInfo>, Iterator<NvdCveInfo> {
|
||||
|
||||
/**
|
||||
* A collection of sources of data.
|
||||
*/
|
||||
Map<String, NvdCveInfo> collection = new TreeMap<String, NvdCveInfo>();
|
||||
|
||||
/**
|
||||
* Gets whether or not an update is needed.
|
||||
*
|
||||
* @return true or false depending on whether an update is needed
|
||||
*/
|
||||
public boolean isUpdateNeeded() {
|
||||
for (NvdCveInfo item : this) {
|
||||
if (item.getNeedsUpdate()) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Adds a new entry of updateable information to the contained collection.
|
||||
*
|
||||
* @param id the key for the item to be added
|
||||
* @param url the URL to download the item
|
||||
* @param oldUrl the URL for the old version of the item (the NVD CVE old
|
||||
* schema still contains useful data we need).
|
||||
* @throws MalformedURLException thrown if the URL provided is invalid
|
||||
* @throws DownloadFailedException thrown if the download fails.
|
||||
*/
|
||||
public void add(String id, String url, String oldUrl) throws MalformedURLException, DownloadFailedException {
|
||||
add(id, url, oldUrl, false);
|
||||
}
|
||||
|
||||
/**
|
||||
* Adds a new entry of updateable information to the contained collection.
|
||||
*
|
||||
* @param id the key for the item to be added
|
||||
* @param url the URL to download the item
|
||||
* @param oldUrl the URL for the old version of the item (the NVD CVE old
|
||||
* schema still contains useful data we need).
|
||||
* @param needsUpdate whether or not the data needs to be updated
|
||||
* @throws MalformedURLException thrown if the URL provided is invalid
|
||||
* @throws DownloadFailedException thrown if the download fails.
|
||||
*/
|
||||
public void add(String id, String url, String oldUrl, boolean needsUpdate) throws MalformedURLException, DownloadFailedException {
|
||||
NvdCveInfo item = new NvdCveInfo();
|
||||
item.setNeedsUpdate(needsUpdate); //the others default to true, to make life easier later this should default to false.
|
||||
item.setId(id);
|
||||
item.setUrl(url);
|
||||
item.setOldSchemaVersionUrl(oldUrl);
|
||||
item.setTimestamp(Downloader.getLastModified(new URL(url)));
|
||||
collection.put(id, item);
|
||||
}
|
||||
|
||||
/**
|
||||
* Clears the contained collection of NvdCveInfo entries.
|
||||
*/
|
||||
public void clear() {
|
||||
collection.clear();
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the timestamp for the given entry.
|
||||
*
|
||||
* @param key the key to lookup in the collection of NvdCveInfo items
|
||||
* @return the timestamp for the given entry
|
||||
*/
|
||||
public long getTimeStamp(String key) {
|
||||
return collection.get(key).getTimestamp();
|
||||
}
|
||||
/**
|
||||
* An internal iterator used to implement iterable.
|
||||
*/
|
||||
Iterator<Entry<String, NvdCveInfo>> iterableContent = null;
|
||||
|
||||
/**
|
||||
* <p>Returns an iterator for the NvdCveInfo contained.</p>
|
||||
* <p><b>This method is not thread safe.</b></p>
|
||||
*
|
||||
* @return an NvdCveInfo Iterator
|
||||
*/
|
||||
@Override
|
||||
public Iterator<NvdCveInfo> iterator() {
|
||||
iterableContent = collection.entrySet().iterator();
|
||||
return this;
|
||||
}
|
||||
|
||||
/**
|
||||
* <p>Returns whether or not there is another item in the collection.</p>
|
||||
* <p><b>This method is not thread safe.</b></p>
|
||||
*
|
||||
* @return true or false depending on whether or not another item exists in
|
||||
* the collection
|
||||
*/
|
||||
@Override
|
||||
public boolean hasNext() {
|
||||
return iterableContent.hasNext();
|
||||
}
|
||||
|
||||
/**
|
||||
* <p>Returns the next item in the collection.</p>
|
||||
* <p><b>This method is not thread safe.</b></p>
|
||||
*
|
||||
* @return the next NvdCveInfo item in the collection
|
||||
*/
|
||||
@Override
|
||||
public NvdCveInfo next() {
|
||||
return iterableContent.next().getValue();
|
||||
}
|
||||
|
||||
/**
|
||||
* <p>Removes the current NvdCveInfo object from the collection.</p>
|
||||
* <p><b>This method is not thread safe.</b></p>
|
||||
*/
|
||||
@Override
|
||||
public void remove() {
|
||||
iterableContent.remove();
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the specified item from the collection.
|
||||
*
|
||||
* @param key the key to lookup the return value
|
||||
* @return the NvdCveInfo object stored using the specified key
|
||||
*/
|
||||
NvdCveInfo get(String key) {
|
||||
return collection.get(key);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user