github-mirrors/DependencyCheck
1
0
Fork 0
mirror of https://github.com/ysoftdevs/DependencyCheck.git synced 2026-01-19 10:07:12 +01:00
Code Issues Packages Projects Releases Wiki Activity

changes to resolve issues with multiple connections to the embedded H2 database

Browse Source
This commit is contained in:
Jeremy Long
2017-09-25 06:55:22 -04:00
parent 165170e5d4
commit 04dc5f8491
25 changed files with 449 additions and 377 deletions
Download Patch File Download Diff File Expand all files Collapse all files

116
dependency-check-core/src/main/java/org/owasp/dependencycheck/Engine.java
View File

@@ -42,6 +42,9 @@ import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.FileFilter;
import java.io.IOException;
import java.nio.file.CopyOption;
import java.nio.file.Files;
import java.nio.file.StandardCopyOption;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
@@ -60,6 +63,9 @@ import java.util.concurrent.TimeUnit;
import javax.annotation.concurrent.NotThreadSafe;
import static org.owasp.dependencycheck.analyzer.AnalysisPhase.*;
import org.owasp.dependencycheck.exception.H2DBLockException;
import org.owasp.dependencycheck.utils.FileUtils;
import org.owasp.dependencycheck.utils.H2DBLock;
/**
* Scans files, directories, etc. for Dependencies. Analyzers are loaded and
@@ -710,7 +716,7 @@ public class Engine implements FileFilter, AutoCloseable {
}
if (autoUpdate) {
try {
doUpdates();
doUpdates(true);
} catch (UpdateException ex) {
exceptions.add(ex);
LOGGER.warn("Unable to update Cached Web DataSource, using local "
@@ -724,7 +730,7 @@ public class Engine implements FileFilter, AutoCloseable {
if (ConnectionFactory.isH2Connection(settings) && !ConnectionFactory.h2DataFileExists(settings)) {
throw new ExceptionCollection(new NoDataException("Autoupdate is disabled and the database does not exist"), true);
} else {
openDatabase();
openDatabase(true, true);
}
} catch (IOException ex) {
throw new ExceptionCollection(new DatabaseException("Autoupdate is disabled and unable to connect to the database"), true);
@@ -854,28 +860,114 @@ public class Engine implements FileFilter, AutoCloseable {
* @throws UpdateException thrown if the operation fails
*/
public void doUpdates() throws UpdateException {
doUpdates(false);
}
/**
* Cycles through the cached web data sources and calls update on all of
* them.
*
* @param remainOpen whether or not the database connection should remain
* open
* @throws UpdateException thrown if the operation fails
*/
public void doUpdates(boolean remainOpen) throws UpdateException {
if (mode.isDatabseRequired()) {
openDatabase();
LOGGER.info("Checking for updates");
final long updateStart = System.currentTimeMillis();
final UpdateService service = new UpdateService(serviceClassLoader);
final Iterator<CachedWebDataSource> iterator = service.getDataSources();
while (iterator.hasNext()) {
final CachedWebDataSource source = iterator.next();
source.update(this);
H2DBLock dblock = null;
try {
if (ConnectionFactory.isH2Connection(settings)) {
dblock = new H2DBLock(settings);
LOGGER.debug("locking for update");
dblock.lock();
}
openDatabase(false, false);
LOGGER.info("Checking for updates");
final long updateStart = System.currentTimeMillis();
final UpdateService service = new UpdateService(serviceClassLoader);
final Iterator<CachedWebDataSource> iterator = service.getDataSources();
while (iterator.hasNext()) {
final CachedWebDataSource source = iterator.next();
source.update(this);
}
database.close();
database = null;
LOGGER.info("Check for updates complete ({} ms)", System.currentTimeMillis() - updateStart);
if (remainOpen) {
openDatabase(true, false);
}
} catch (H2DBLockException ex) {
throw new UpdateException("Unable to obtain an exclusive lock on the H2 database to perform updates", ex);
} finally {
if (dblock != null) {
dblock.release();
}
}
LOGGER.info("Check for updates complete ({} ms)", System.currentTimeMillis() - updateStart);
} else {
LOGGER.info("Skipping update check in evidence collection mode.");
}
}
/**
* Opens the database connection.
* <p>
* This method is only public for unit/integration testing. This method
* should not be called by any integration that uses
* dependency-check-core.</p>
* <p>
* Opens the database connection.</p>
*/
public void openDatabase() {
openDatabase(false, true);
}
/**
* <p>
* This method is only public for unit/integration testing. This method
* should not be called by any integration that uses
* dependency-check-core.</p>
* <p>
* Opens the database connection; if readOnly is true a copy of the database
* will be made.</p>
*
* @param readOnly whether or not the database connection should be readonly
* @param lockRequired whether or not a lock needs to be acquired when
* opening the database
*/
public void openDatabase(boolean readOnly, boolean lockRequired) {
if (mode.isDatabseRequired() && database == null) {
//needed to update schema any required schema changes
database = new CveDB(settings);
if (readOnly
&& ConnectionFactory.isH2Connection(settings)
&& settings.getString(Settings.KEYS.DB_CONNECTION_STRING).contains("file:%s")) {
H2DBLock lock = null;
try {
File db = ConnectionFactory.getH2DataFile(settings);
if (db.isFile()) {
database.close();
if (lockRequired) {
lock = new H2DBLock(settings);
lock.lock();
}
LOGGER.debug("copying database");
File temp = settings.getTempDirectory();
File tempDB = new File(temp, db.getName());
Files.copy(db.toPath(), tempDB.toPath());
LOGGER.debug("copying complete '{}'", temp.toPath());
settings.setString(Settings.KEYS.DATA_DIRECTORY, temp.getPath());
String connStr = settings.getString(Settings.KEYS.DB_CONNECTION_STRING);
settings.setString(Settings.KEYS.DB_CONNECTION_STRING, connStr + "ACCESS_MODE_DATA=r");
database = new CveDB(settings);
}
} catch (IOException ex) {
LOGGER.debug("Unable to open db in read only mode", ex);
} catch (H2DBLockException ex) {
LOGGER.debug("Failed to obtain lock - unable to open db in read only mode", ex);
} finally {
if (lock != null) {
lock.release();
}
}
}
}
}

12
dependency-check-core/src/main/java/org/owasp/dependencycheck/agent/DependencyCheckScanAgent.java
View File

@@ -105,8 +105,8 @@ public class DependencyCheckScanAgent {
*/
private boolean autoUpdate = true;
/**
* Sets whether the data directory should be updated without performing a scan.
* Default is false.
* Sets whether the data directory should be updated without performing a
* scan. Default is false.
*/
private boolean updateOnly = false;
/**
@@ -221,8 +221,8 @@ public class DependencyCheckScanAgent {
*/
private Settings settings;
/**
* The path to optional dependency-check properties file. This will be
* used to side-load additional user-defined properties.
* The path to optional dependency-check properties file. This will be used
* to side-load additional user-defined properties.
* {@link Settings#mergeProperties(String)}
*/
private String propertiesFilePath;
@@ -937,7 +937,7 @@ public class DependencyCheckScanAgent {
LOGGER.error("Continuing execution");
}
}
settings.setBoolean(Settings.KEYS.AUTO_UPDATE, autoUpdate);
settings.setStringIfNotEmpty(Settings.KEYS.PROXY_SERVER, proxyServer);
settings.setStringIfNotEmpty(Settings.KEYS.PROXY_PORT, proxyPort);
@@ -1028,7 +1028,7 @@ public class DependencyCheckScanAgent {
final String msg = String.format("%n%nDependency-Check Failure:%n"
+ "One or more dependencies were identified with vulnerabilities that have a CVSS score greater than '%.1f': %s%n"
+ "See the dependency-check report for more details.%n%n", failBuildOnCVSS, ids.toString());
throw new ScanAgentException(msg);
}
}

43
dependency-check-core/src/main/java/org/owasp/dependencycheck/data/nvdcve/ConnectionFactory.java
View File

@@ -29,13 +29,10 @@ import java.sql.SQLException;
import java.sql.Statement;
import javax.annotation.concurrent.ThreadSafe;
import org.apache.commons.io.IOUtils;
import org.owasp.dependencycheck.data.update.exception.UpdateException;
import org.owasp.dependencycheck.exception.H2DBLockException;
import org.owasp.dependencycheck.utils.DBUtils;
import org.owasp.dependencycheck.utils.DependencyVersion;
import org.owasp.dependencycheck.utils.DependencyVersionUtil;
import org.owasp.dependencycheck.utils.FileUtils;
import org.owasp.dependencycheck.utils.H2DBLock;
import org.owasp.dependencycheck.utils.Settings;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -115,26 +112,15 @@ public final class ConnectionFactory {
return;
}
Connection conn = null;
H2DBLock dblock = null;
try {
if (isH2Connection()) {
dblock = new H2DBLock(settings);
LOGGER.debug("locking for init");
dblock.lock();
}
//load the driver if necessary
final String driverName = settings.getString(Settings.KEYS.DB_DRIVER_NAME, "");
if (!driverName.isEmpty()) {
LOGGER.debug("Loading driver: {}", driverName);
final String driverPath = settings.getString(Settings.KEYS.DB_DRIVER_PATH, "");
final String driverPath = settings.getString(Settings.KEYS.DB_DRIVER_PATH, "");
if (!driverPath.isEmpty()) {
LOGGER.debug("Loading driver '{}' from '{}'", driverName, driverPath);
try {
if (!driverPath.isEmpty()) {
LOGGER.debug("Loading driver from: {}", driverPath);
driver = DriverLoader.load(driverName, driverPath);
} else {
driver = DriverLoader.load(driverName);
}
LOGGER.debug("Loading driver from: {}", driverPath);
driver = DriverLoader.load(driverName, driverPath);
} catch (DriverLoadException ex) {
LOGGER.debug("Unable to load database driver", ex);
throw new DatabaseException("Unable to load database driver", ex);
@@ -198,8 +184,6 @@ public final class ConnectionFactory {
LOGGER.debug("", dex);
throw new DatabaseException("Database schema does not match this version of dependency-check", dex);
}
} catch (H2DBLockException ex) {
throw new DatabaseException("Unable to obtain an exclusive lock on the H2 database to perform initializataion", ex);
} finally {
if (conn != null) {
try {
@@ -208,9 +192,6 @@ public final class ConnectionFactory {
LOGGER.debug("An error occurred closing the connection", ex);
}
}
if (dblock != null) {
dblock.release();
}
}
}
@@ -272,10 +253,22 @@ public final class ConnectionFactory {
* cannot be created
*/
public static boolean h2DataFileExists(Settings configuration) throws IOException {
File file = getH2DataFile(configuration);
return file.exists();
}
/**
* Returns a reference to the H2 database file.
*
* @param configuration the configured settings
* @return the path to the H2 database file
* @throws IOException thrown if there is an error
*/
public static File getH2DataFile(Settings configuration) throws IOException {
final File dir = configuration.getDataDirectory();
final String fileName = configuration.getString(Settings.KEYS.DB_FILE_NAME);
final File file = new File(dir, fileName);
return file.exists();
return file;
}
/**

17
dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/NvdCveUpdater.java
View File

@@ -33,7 +33,6 @@ import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import javax.annotation.concurrent.ThreadSafe;
import org.owasp.dependencycheck.Engine;
import org.owasp.dependencycheck.data.nvdcve.ConnectionFactory;
import org.owasp.dependencycheck.data.nvdcve.CveDB;
import org.owasp.dependencycheck.data.nvdcve.DatabaseException;
import org.owasp.dependencycheck.data.nvdcve.DatabaseProperties;
@@ -44,11 +43,9 @@ import org.owasp.dependencycheck.data.update.nvd.DownloadTask;
import org.owasp.dependencycheck.data.update.nvd.NvdCveInfo;
import org.owasp.dependencycheck.data.update.nvd.ProcessTask;
import org.owasp.dependencycheck.data.update.nvd.UpdateableNvdCve;
import org.owasp.dependencycheck.exception.H2DBLockException;
import org.owasp.dependencycheck.utils.DateUtil;
import org.owasp.dependencycheck.utils.Downloader;
import org.owasp.dependencycheck.utils.DownloadFailedException;
import org.owasp.dependencycheck.utils.H2DBLock;
import org.owasp.dependencycheck.utils.InvalidSettingException;
import org.owasp.dependencycheck.utils.Settings;
import org.slf4j.Logger;
@@ -113,17 +110,10 @@ public class NvdCveUpdater implements CachedWebDataSource {
if (isUpdateConfiguredFalse()) {
return;
}
H2DBLock dblock = null;
try {
if (ConnectionFactory.isH2Connection(settings)) {
dblock = new H2DBLock(settings);
LOGGER.debug("locking for update");
dblock.lock();
}
initializeExecutorServices();
dbProperties = cveDb.getDatabaseProperties();
if (checkUpdate()) {
initializeExecutorServices();
final UpdateableNvdCve updateable = getUpdatesNeeded();
if (updateable.isUpdateNeeded()) {
performUpdate(updateable);
@@ -140,12 +130,7 @@ public class NvdCveUpdater implements CachedWebDataSource {
throw new UpdateException("Unable to download the NVD CVE data.", ex);
} catch (DatabaseException ex) {
throw new UpdateException("Database Exception, unable to update the data to use the most current data.", ex);
} catch (H2DBLockException ex) {
throw new UpdateException("Unable to obtain an exclusive lock on the H2 database to perform updates", ex);
} finally {
if (dblock != null) {
dblock.release();
}
shutdownExecutorServices();
}
}

87
dependency-check-core/src/main/java/org/owasp/dependencycheck/utils/H2DBLock.java
View File

@@ -18,9 +18,13 @@
package org.owasp.dependencycheck.utils;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.RandomAccessFile;
import java.nio.channels.FileLock;
import java.security.SecureRandom;
import java.sql.Timestamp;
import java.util.Date;
import javax.annotation.concurrent.NotThreadSafe;
import org.owasp.dependencycheck.exception.H2DBLockException;
@@ -38,6 +42,14 @@ public class H2DBLock {
* The logger.
*/
private static final Logger LOGGER = LoggerFactory.getLogger(H2DBLock.class);
/**
* How long to sleep waiting for the lock.
*/
public static final int SLEEP_DURATION = 10000;
/**
* Max attempts to obtain a lock.
*/
public static final int MAX_SLEEP_COUNT = 120;
/**
* The file lock.
*/
@@ -54,6 +66,10 @@ public class H2DBLock {
* The configured settings.
*/
private final Settings settings;
/**
* A random string used to validate the lock.
*/
private final String magic;
/**
* Constructs a new H2DB Lock object with the configured settings.
@@ -62,6 +78,10 @@ public class H2DBLock {
*/
public H2DBLock(Settings settings) {
this.settings = settings;
byte[] random = new byte[16];
SecureRandom gen = new SecureRandom();
gen.nextBytes(random);
magic = Checksum.getHex(random);
}
/**
@@ -85,9 +105,12 @@ public class H2DBLock {
if (!lockFile.getParentFile().isDirectory() && !lockFile.mkdir()) {
throw new H2DBLockException("Unable to create path to data directory.");
}
if (lockFile.isFile() && getFileAge(lockFile) > 5 && !lockFile.delete()) {
LOGGER.warn("An old db update lock file was found but the system was unable to delete "
+ "the file. Consider manually deleting {}", lockFile.getAbsolutePath());
if (lockFile.isFile() && getFileAge(lockFile) > 30) {
LOGGER.debug("An old db update lock file was found: {}", lockFile.getAbsolutePath());
if (!lockFile.delete()) {
LOGGER.warn("An old db update lock file was found but the system was unable to delete "
+ "the file. Consider manually deleting {}", lockFile.getAbsolutePath());
}
}
int ctr = 0;
do {
@@ -95,37 +118,50 @@ public class H2DBLock {
if (!lockFile.exists() && lockFile.createNewFile()) {
file = new RandomAccessFile(lockFile, "rw");
lock = file.getChannel().lock();
LOGGER.debug("Lock file created ({})", Thread.currentThread().getName());
file.writeBytes(magic);
file.getChannel().force(true);
Thread.sleep(20);
file.seek(0);
String current = file.readLine();
if (current != null && !current.equals(magic)) {
lock.close();
lock = null;
LOGGER.debug("Another process obtained a lock first ({})", Thread.currentThread().getName());
} else {
Timestamp timestamp = new Timestamp(System.currentTimeMillis());
LOGGER.debug("Lock file created ({}) {} @ {}", Thread.currentThread().getName(), magic, timestamp.toString());
}
}
} catch (IOException ex) {
} catch (IOException | InterruptedException ex) {
LOGGER.trace("Expected error as another thread has likely locked the file", ex);
} finally {
if (lock == null && file != null) {
try {
file.close();
file = null;
} catch (IOException ex) {
LOGGER.trace("Unable to close the ulFile", ex);
LOGGER.trace("Unable to close the lock file", ex);
}
}
}
if (lock == null || !lock.isValid()) {
try {
LOGGER.debug("Sleeping thread {} for 5 seconds because an exclusive lock on the database could not be obtained",
Thread.currentThread().getName());
Thread.sleep(5000);
Timestamp timestamp = new Timestamp(System.currentTimeMillis());
LOGGER.debug("Sleeping thread {} ({}) for 10 seconds because an exclusive lock on the database could not be obtained ({})",
Thread.currentThread().getName(), magic, timestamp.toString());
Thread.sleep(SLEEP_DURATION);
} catch (InterruptedException ex) {
LOGGER.trace("ignorable error, sleep was interrupted.", ex);
LOGGER.debug("sleep was interrupted.", ex);
Thread.currentThread().interrupt();
}
}
} while (++ctr < 60 && (lock == null || !lock.isValid()));
} while (++ctr < MAX_SLEEP_COUNT && (lock == null || !lock.isValid()));
if (lock == null || !lock.isValid()) {
throw new H2DBLockException("Unable to obtain the update lock, skipping the database update. Skippinig the database update.");
}
} catch (IOException ex) {
throw new H2DBLockException(ex.getMessage(), ex);
}
}
/**
@@ -137,7 +173,7 @@ public class H2DBLock {
lock.release();
lock = null;
} catch (IOException ex) {
LOGGER.trace("Ignorable exception", ex);
LOGGER.debug("Failed to release lock", ex);
}
}
if (file != null) {
@@ -145,15 +181,24 @@ public class H2DBLock {
file.close();
file = null;
} catch (IOException ex) {
LOGGER.trace("Ignorable exception", ex);
LOGGER.debug("Unable to delete lock file", ex);
}
}
if (lockFile != null && lockFile.isFile() && !lockFile.delete()) {
LOGGER.error("Lock file '{}' was unable to be deleted. Please manually delete this file.", lockFile.toString());
lockFile.deleteOnExit();
if (lockFile != null && lockFile.isFile()) {
try (RandomAccessFile f = new RandomAccessFile(lockFile, "rw")) {
String m = f.readLine();
f.close();
if (m != null && m.equals(magic) && !lockFile.delete()) {
LOGGER.error("Lock file '{}' was unable to be deleted. Please manually delete this file.", lockFile.toString());
lockFile.deleteOnExit();
}
} catch (IOException ex) {
LOGGER.debug("Error deleting lock file", ex);
}
}
lockFile = null;
LOGGER.debug("Lock released ({})", Thread.currentThread().getName());
Timestamp timestamp = new Timestamp(System.currentTimeMillis());
LOGGER.debug("Lock released ({}) {} @ {}", Thread.currentThread().getName(), magic, timestamp.toString());
}
/**
@@ -162,9 +207,11 @@ public class H2DBLock {
* @param file the file to calculate the age
* @return the age of the file
*/
private long getFileAge(File file) {
private double getFileAge(File file) {
final Date d = new Date();
final long modified = file.lastModified();
return (d.getTime() - modified) / 1000 / 60;
final double time = (d.getTime() - modified) / 1000 / 60;
LOGGER.debug("Lock file age is {} minutes", time);
return time;
}
}

2
dependency-check-core/src/main/resources/dependencycheck.properties
View File

@@ -23,7 +23,7 @@ data.file_name=dc.h2.db
### the gradle PurgeDataExtension.
data.version=3.0
data.connection_string=jdbc:h2:file:%s;MV_STORE=FALSE;AUTOCOMMIT=ON;LOCK_MODE=0;FILE_LOCK=NO
data.connection_string=jdbc:h2:file:%s;MV_STORE=FALSE;AUTOCOMMIT=ON;
#data.connection_string=jdbc:mysql://localhost:3306/dependencycheck
# user name and password for the database connection. The inherent case is to use H2.