github-mirrors/DependencyCheck
1
0
Fork 0
mirror of https://github.com/ysoftdevs/DependencyCheck.git synced 2026-01-14 07:43:40 +01:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #839 from jeremylong/h2upgrade

Browse Source 
H2 Upgrade
This commit is contained in:
Jeremy Long
2017-08-20 11:02:17 -04:00
committed by GitHub
parent 631c10f8b6 c0aca39d31
commit bcae8d2015
13 changed files with 247 additions and 87 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
dependency-check-cli/src/main/java/org/owasp/dependencycheck/App.java
View File

@@ -112,7 +112,7 @@ public class App {
}
File db;
try {
db = new File(Settings.getDataDirectory(), "dc.h2.db");
db = new File(Settings.getDataDirectory(), Settings.getString(Settings.KEYS.DB_FILE_NAME, "dc.h2.db"));
if (db.exists()) {
if (db.delete()) {
LOGGER.info("Database file purged; local copy of the NVD has been removed");

11
dependency-check-core/src/main/java/org/owasp/dependencycheck/data/nvdcve/ConnectionFactory.java
View File

@@ -82,6 +82,11 @@ public final class ConnectionFactory {
* The password for the database.
*/
private static String password = null;
/**
* Counter to ensure that calls to ensureSchemaVersion does not end up in an
* endless loop.
*/
private static int callDepth = 0;
/**
* Private constructor for this factory class; no instance is ever needed.
@@ -369,12 +374,6 @@ public final class ConnectionFactory {
}
}
/**
* Counter to ensure that calls to ensureSchemaVersion does not end up in an
* endless loop.
*/
private static int callDepth = 0;
/**
* Uses the provided connection to check the specified schema version within
* the database.

79
dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/NvdCveUpdater.java
View File

@@ -47,9 +47,11 @@ import org.owasp.dependencycheck.data.update.nvd.DownloadTask;
import org.owasp.dependencycheck.data.update.nvd.NvdCveInfo;
import org.owasp.dependencycheck.data.update.nvd.ProcessTask;
import org.owasp.dependencycheck.data.update.nvd.UpdateableNvdCve;
import org.owasp.dependencycheck.exception.H2DBLockException;
import org.owasp.dependencycheck.utils.DateUtil;
import org.owasp.dependencycheck.utils.Downloader;
import org.owasp.dependencycheck.utils.DownloadFailedException;
import org.owasp.dependencycheck.utils.H2DBLock;
import org.owasp.dependencycheck.utils.InvalidSettingException;
import org.owasp.dependencycheck.utils.Settings;
import org.slf4j.Logger;
@@ -107,46 +109,9 @@ public class NvdCveUpdater implements CachedWebDataSource {
if (isUpdateConfiguredFalse()) {
return;
}
FileLock lock = null;
RandomAccessFile ulFile = null;
File lockFile = null;
H2DBLock dbupdate = new H2DBLock();
try {
if (ConnectionFactory.isH2Connection()) {
final File dir = Settings.getDataDirectory();
lockFile = new File(dir, "odc.update.lock");
if (lockFile.isFile() && getFileAge(lockFile) > 5 && !lockFile.delete()) {
LOGGER.warn("An old db update lock file was found but the system was unable to delete "
+ "the file. Consider manually deleting {}", lockFile.getAbsolutePath());
}
int ctr = 0;
do {
try {
if (!lockFile.exists() && lockFile.createNewFile()) {
ulFile = new RandomAccessFile(lockFile, "rw");
lock = ulFile.getChannel().lock();
}
} catch (IOException ex) {
LOGGER.trace("Expected error as another thread has likely locked the file", ex);
} finally {
if (lock == null && ulFile != null) {
ulFile.close();
}
}
if (lock == null || !lock.isValid()) {
try {
LOGGER.debug("Sleeping thread {} for 5 seconds because we could not obtain the update lock.",
Thread.currentThread().getName());
Thread.sleep(5000);
} catch (InterruptedException ex) {
LOGGER.trace("ignorable error, sleep was interrupted.", ex);
Thread.currentThread().interrupt();
}
}
} while (++ctr < 60 && (lock == null || !lock.isValid()));
if (lock == null || !lock.isValid()) {
throw new UpdateException("Unable to obtain the update lock, skipping the database update. Skippinig the database update.");
}
}
dbupdate.lock();
initializeExecutorServices();
cveDb = CveDB.getInstance();
dbProperties = cveDb.getDatabaseProperties();
@@ -168,30 +133,14 @@ public class NvdCveUpdater implements CachedWebDataSource {
throw new UpdateException("Unable to download the NVD CVE data.", ex);
} catch (DatabaseException ex) {
throw new UpdateException("Database Exception, unable to update the data to use the most current data.", ex);
} catch (IOException ex) {
throw new UpdateException("Database Exception", ex);
} catch (H2DBLockException ex) {
throw new UpdateException("Unable to obtain an exclusive lock on the H2 database to perform updates", ex);
} finally {
shutdownExecutorServices();
if (cveDb != null) {
cveDb.close();
}
if (lock != null) {
try {
lock.release();
} catch (IOException ex) {
LOGGER.trace("Ignorable exception", ex);
}
}
if (ulFile != null) {
try {
ulFile.close();
} catch (IOException ex) {
LOGGER.trace("Ignorable exception", ex);
}
}
if (lockFile != null && lockFile.isFile() && !lockFile.delete()) {
LOGGER.error("Lock file '{}' was unable to be deleted. Please manually delete this file.", lockFile.toString());
}
dbupdate.release();
shutdownExecutorServices();
}
}
@@ -218,18 +167,6 @@ public class NvdCveUpdater implements CachedWebDataSource {
return !autoUpdate;
}
/**
* Returns the age of the file in minutes.
*
* @param file the file to calculate the age
* @return the age of the file
*/
private long getFileAge(File file) {
final Date d = new Date();
final long modified = file.lastModified();
return (d.getTime() - modified) / 1000 / 60;
}
/**
* Initialize the executor services for download and processing of the NVD
* CVE XML data.

66
dependency-check-core/src/main/java/org/owasp/dependencycheck/exception/H2DBLockException.java Normal file
View File

@@ -0,0 +1,66 @@
/*
* This file is part of dependency-check-core.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
* Copyright (c) 2017 Jeremy Long. All Rights Reserved.
*/
package org.owasp.dependencycheck.exception;
/**
* An exception used when trying to obtain a lock on the H2 database.
*
* @author Jeremy Long
*/
public class H2DBLockException extends Exception {
/**
* The serial version uid.
*/
private static final long serialVersionUID = 1L;
/**
* Creates a new H2DBLockException.
*/
public H2DBLockException() {
super();
}
/**
* Creates a new H2DBLockException.
*
* @param msg a message for the exception.
*/
public H2DBLockException(String msg) {
super(msg);
}
/**
* Creates a new H2DBLockException.
*
* @param ex the cause of the exception.
*/
public H2DBLockException(Throwable ex) {
super(ex);
}
/**
* Creates a new H2DBLockException.
*
* @param msg a message for the exception.
* @param ex the cause of the exception.
*/
public H2DBLockException(String msg, Throwable ex) {
super(msg, ex);
}
}

152
dependency-check-core/src/main/java/org/owasp/dependencycheck/utils/H2DBLock.java Normal file
View File

@@ -0,0 +1,152 @@
/*
* This file is part of dependency-check-core.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
* Copyright (c) 2017 Jeremy Long. All Rights Reserved.
*/
package org.owasp.dependencycheck.utils;
import java.io.File;
import java.io.IOException;
import java.io.RandomAccessFile;
import java.nio.channels.FileLock;
import java.util.Date;
import org.owasp.dependencycheck.data.nvdcve.ConnectionFactory;
import org.owasp.dependencycheck.exception.H2DBLockException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
*
* @author Jeremy Long
*/
public class H2DBLock {
/**
* The logger.
*/
private static final Logger LOGGER = LoggerFactory.getLogger(H2DBLock.class);
/**
* The file lock.
*/
private FileLock lock = null;
/**
* Reference to the file that we are locking.
*/
private RandomAccessFile file = null;
/**
* The lock file.
*/
private File lockFile = null;
/**
* Determine if the lock is currently held.
*
* @return true if the lock is currently held
*/
public boolean isLocked() {
return lock != null && lock.isValid();
}
/**
* Obtains a lock on the H2 database.
*
* @throws H2DBLockException thrown if a lock could not be obtained
*/
public void lock() throws H2DBLockException {
if (ConnectionFactory.isH2Connection()) {
try {
final File dir = Settings.getDataDirectory();
lockFile = new File(dir, "dc.update.lock");
if (lockFile.isFile() && getFileAge(lockFile) > 5 && !lockFile.delete()) {
LOGGER.warn("An old db update lock file was found but the system was unable to delete "
+ "the file. Consider manually deleting {}", lockFile.getAbsolutePath());
}
int ctr = 0;
do {
try {
if (!lockFile.exists() && lockFile.createNewFile()) {
file = new RandomAccessFile(lockFile, "rw");
lock = file.getChannel().lock();
}
} catch (IOException ex) {
LOGGER.trace("Expected error as another thread has likely locked the file", ex);
} finally {
if (lock == null && file != null) {
try {
file.close();
} catch (IOException ex) {
LOGGER.trace("Unable to close the ulFile", ex);
}
}
}
if (lock == null || !lock.isValid()) {
try {
LOGGER.debug("Sleeping thread {} for 5 seconds because we could not obtain the update lock.",
Thread.currentThread().getName());
Thread.sleep(5000);
} catch (InterruptedException ex) {
LOGGER.trace("ignorable error, sleep was interrupted.", ex);
Thread.currentThread().interrupt();
}
}
} while (++ctr < 60 && (lock == null || !lock.isValid()));
if (lock == null || !lock.isValid()) {
throw new H2DBLockException("Unable to obtain the update lock, skipping the database update. Skippinig the database update.");
}
} catch (IOException ex) {
throw new H2DBLockException(ex.getMessage(), ex);
}
}
}
/**
* Releases the lock on the H2 database.
*/
public void release() {
if (lock != null) {
try {
lock.release();
lock = null;
} catch (IOException ex) {
LOGGER.trace("Ignorable exception", ex);
}
}
if (file != null) {
try {
file.close();
file = null;
} catch (IOException ex) {
LOGGER.trace("Ignorable exception", ex);
}
}
if (lockFile != null && lockFile.isFile() && !lockFile.delete()) {
LOGGER.error("Lock file '{}' was unable to be deleted. Please manually delete this file.", lockFile.toString());
lockFile.deleteOnExit();
}
lockFile = null;
}
/**
* Returns the age of the file in minutes.
*
* @param file the file to calculate the age
* @return the age of the file
*/
private long getFileAge(File file) {
final Date d = new Date();
final long modified = file.lastModified();
return (d.getTime() - modified) / 1000 / 60;
}
}

2
dependency-check-core/src/main/resources/dependencycheck.properties
View File

@@ -23,7 +23,7 @@ data.file_name=dc.h2.db
### the gradle PurgeDataExtension.
data.version=3.0
data.connection_string=jdbc:h2:file:%s;FILE_LOCK=SERIALIZED;AUTOCOMMIT=ON;
data.connection_string=jdbc:h2:file:%s;MV_STORE=FALSE;AUTOCOMMIT=ON;LOCK_MODE=0;FILE_LOCK=NO
#data.connection_string=jdbc:mysql://localhost:3306/dependencycheck
# user name and password for the database connection. The inherent case is to use H2.

7
dependency-check-core/src/test/java/org/owasp/dependencycheck/BaseDBTestCase.java
View File

@@ -70,9 +70,10 @@ public abstract class BaseDBTestCase extends BaseTest {
d.mkdir();
continue;
}
File o = new File(dataPath, entry.getName());
o.createNewFile();
try (FileOutputStream fos = new FileOutputStream(o, false);
//File o = new File(dataPath, entry.getName());
//o.createNewFile();
dataFile.createNewFile();
try (FileOutputStream fos = new FileOutputStream(dataFile, false);
BufferedOutputStream dest = new BufferedOutputStream(fos, BUFFER_SIZE)) {
IOUtils.copy(zin, dest);
} catch (Throwable ex) {

5
dependency-check-core/src/test/java/org/owasp/dependencycheck/EngineModeIT.java
View File

@@ -116,6 +116,11 @@ public class EngineModeIT extends BaseTest {
assertThat(Files.exists(directory), is(true));
assertThat(Files.isDirectory(directory), is(true));
Path database = directory.resolve(Settings.getString(Settings.KEYS.DB_FILE_NAME));
System.err.println(database.toString());
for (String f : directory.toFile().list()) {
System.err.println(f);
}
assertThat(Files.exists(database), is(exists));
}
}

2
dependency-check-core/src/test/resources/dependencycheck.properties
View File

@@ -18,7 +18,7 @@ data.directory=[JAR]/data
#if the filename has a %s it will be replaced with the current expected version
data.file_name=dc.h2.db
data.version=3.0
data.connection_string=jdbc:h2:file:%s;FILE_LOCK=SERIALIZED;AUTOCOMMIT=ON;
data.connection_string=jdbc:h2:file:%s;MV_STORE=FALSE;AUTOCOMMIT=ON;LOCK_MODE=0;FILE_LOCK=NO
#data.connection_string=jdbc:mysql://localhost:3306/dependencycheck
# user name and password for the database connection. The inherent case is to use H2.

2
dependency-check-maven/src/main/java/org/owasp/dependencycheck/maven/PurgeMojo.java
View File

@@ -73,7 +73,7 @@ public class PurgeMojo extends BaseDependencyCheckMojo {
populateSettings();
File db;
try {
db = new File(Settings.getDataDirectory(), "dc.h2.db");
db = new File(Settings.getDataDirectory(), Settings.getString(Settings.KEYS.DB_FILE_NAME, "dc.h2.db"));
if (db.exists()) {
if (db.delete()) {
getLog().info("Database file purged; local copy of the NVD has been removed");

2
dependency-check-utils/src/main/java/org/owasp/dependencycheck/utils/Settings.java
View File

@@ -1010,7 +1010,7 @@ public final class Settings {
// yes, for H2 this path won't actually exists - but this is sufficient to get the value needed
final File dbFile = new File(directory, fileName);
final String cString = String.format(connStr, dbFile.getCanonicalPath());
LOGGER.debug("Connection String: '{}'", cString);
LOGGER.error("Connection String: '{}'", cString);
return cString;
}
return connStr;

2
dependency-check-utils/src/test/resources/dependencycheck.properties
View File

@@ -17,7 +17,7 @@ engine.version.url=http://jeremylong.github.io/DependencyCheck/current.txt
data.directory=[JAR]/data
data.file_name=dc.h2.db
data.version=3.0
data.connection_string=jdbc:h2:file:%s;FILE_LOCK=SERIALIZED;AUTOCOMMIT=ON;
data.connection_string=jdbc:h2:file:%s;MV_STORE=FALSE;AUTOCOMMIT=ON;LOCK_MODE=0;FILE_LOCK=NO
#data.connection_string=jdbc:h2:file:%s;AUTO_SERVER=TRUE;AUTOCOMMIT=ON;
#data.connection_string=jdbc:mysql://localhost:3306/dependencycheck

2
pom.xml
View File

@@ -624,7 +624,7 @@ Copyright (c) 2012 - Jeremy Long
<dependency>
<groupId>com.h2database</groupId>
<artifactId>h2</artifactId>
<version>1.3.176</version>
<version>1.4.196</version>
</dependency>
<dependency>
<groupId>commons-cli</groupId>