From f473e63a61a2613681a0733f2232564d4326066e Mon Sep 17 00:00:00 2001 From: Jeremy Long Date: Sun, 23 Apr 2017 07:05:31 -0400 Subject: [PATCH] added test case and added locking mechanism so only one update can run at any given time --- .../data/update/NvdCveUpdater.java | 73 ++++++++++++++++++- dependency-check-maven/pom.xml | 3 + .../invoker.properties | 2 +- .../src/it/690-threadsafety/first/pom.xml | 35 +++++++++ .../src/it/690-threadsafety/fourth/pom.xml | 35 +++++++++ .../fourth/src/main/webapp/WEB-INF/web.xml | 26 +++++++ .../it/690-threadsafety/invoker.properties | 19 +++++ .../src/it/690-threadsafety/pom.xml | 31 ++++++++ .../src/it/690-threadsafety/postbuild.groovy | 28 +++++++ .../src/it/690-threadsafety/prebuild.groovy | 28 +++++++ .../src/it/690-threadsafety/second/pom.xml | 35 +++++++++ .../second/src/main/webapp/WEB-INF/web.xml | 26 +++++++ .../src/it/690-threadsafety/third/pom.xml | 35 +++++++++ .../third/src/main/webapp/WEB-INF/web.xml | 26 +++++++ 14 files changed, 399 insertions(+), 3 deletions(-) create mode 100644 dependency-check-maven/src/it/690-threadsafety/first/pom.xml create mode 100644 dependency-check-maven/src/it/690-threadsafety/fourth/pom.xml create mode 100644 dependency-check-maven/src/it/690-threadsafety/fourth/src/main/webapp/WEB-INF/web.xml create mode 100644 dependency-check-maven/src/it/690-threadsafety/invoker.properties create mode 100644 dependency-check-maven/src/it/690-threadsafety/pom.xml create mode 100644 dependency-check-maven/src/it/690-threadsafety/postbuild.groovy create mode 100644 dependency-check-maven/src/it/690-threadsafety/prebuild.groovy create mode 100644 dependency-check-maven/src/it/690-threadsafety/second/pom.xml create mode 100644 dependency-check-maven/src/it/690-threadsafety/second/src/main/webapp/WEB-INF/web.xml create mode 100644 dependency-check-maven/src/it/690-threadsafety/third/pom.xml create mode 100644 dependency-check-maven/src/it/690-threadsafety/third/src/main/webapp/WEB-INF/web.xml diff --git a/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/NvdCveUpdater.java b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/NvdCveUpdater.java index 2ae21797c..c6b20c12b 100644 --- a/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/NvdCveUpdater.java +++ b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/NvdCveUpdater.java @@ -17,6 +17,9 @@ */ package org.owasp.dependencycheck.data.update; +import java.io.File; +import java.io.IOException; +import java.io.RandomAccessFile; import java.net.MalformedURLException; import java.util.Calendar; import java.util.HashMap; @@ -24,6 +27,8 @@ import java.util.HashSet; import java.util.Map; import java.util.Set; import java.net.URL; +import java.nio.channels.FileLock; +import java.util.Date; import java.util.concurrent.Callable; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; @@ -31,6 +36,7 @@ import java.util.concurrent.Executors; import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; +import org.owasp.dependencycheck.data.nvdcve.ConnectionFactory; import org.owasp.dependencycheck.data.nvdcve.CveDB; import org.owasp.dependencycheck.data.nvdcve.DatabaseException; import org.owasp.dependencycheck.data.nvdcve.DatabaseProperties; @@ -95,7 +101,7 @@ public class NvdCveUpdater implements CachedWebDataSource { * database */ @Override - public void update() throws UpdateException { + public synchronized void update() throws UpdateException { try { if (!Settings.getBoolean(Settings.KEYS.UPDATE_NVDCVE_ENABLED, true)) { return; @@ -113,8 +119,40 @@ public class NvdCveUpdater implements CachedWebDataSource { if (!autoUpdate) { return; } - initializeExecutorServices(); + FileLock lock = null; + RandomAccessFile ulFile = null; + File lockFile = null; try { + if (ConnectionFactory.isH2Connection()) { + final File dir = Settings.getDataDirectory(); + lockFile = new File(dir, "odc.update.lock"); + if (lockFile.isFile() && getFileAge(lockFile) > 5 && !lockFile.delete()) { + LOGGER.warn("An old db update lock file was found but the system was unable to delete the file. Consider manually deleting " + lockFile.getAbsolutePath()); + } + int ctr = 0; + do { + try { + if (!lockFile.exists() && lockFile.createNewFile()) { + ulFile = new RandomAccessFile(lockFile, "rw"); + lock = ulFile.getChannel().lock(); + } + } catch (IOException ex) { + LOGGER.trace("Expected error as another thread has likely locked the file", ex); + } + if (lock == null || !lock.isValid()) { + try { + LOGGER.debug(String.format("Sleeping thread %s for 5 seconds because we could not obtain the update lock.", Thread.currentThread().getName())); + Thread.sleep(5000); + } catch (InterruptedException ex) { + LOGGER.trace("ignorable error, sleep was interrupted.", ex); + } + } + } while (++ctr < 100 && (lock == null || !lock.isValid())); + if (lock == null || !lock.isValid()) { + throw new UpdateException("Unable to obtain the update lock, skipping the database update. Skippinig the database update."); + } + } + initializeExecutorServices(); cveDb = CveDB.getInstance(); dbProperties = cveDb.getDatabaseProperties(); @@ -137,12 +175,43 @@ public class NvdCveUpdater implements CachedWebDataSource { throw new UpdateException("Unable to download the NVD CVE data.", ex); } catch (DatabaseException ex) { throw new UpdateException("Database Exception, unable to update the data to use the most current data.", ex); + } catch (IOException ex) { + throw new UpdateException("Database Exception", ex); } finally { shutdownExecutorServices(); cveDb.close(); + if (lock != null) { + try { + lock.release(); + } catch (IOException ex) { + LOGGER.trace("Ignorable exception", ex); + } + } + if (ulFile != null) { + try { + ulFile.close(); + } catch (IOException ex) { + LOGGER.trace("Ignorable exception", ex); + } + } + if (lockFile != null) { + lockFile.delete(); + } } } + /** + * Returns the age of the file in minutes. + * + * @param file the file to calculate the age + * @return the age of the file + */ + private long getFileAge(File file) { + final Date d = new Date(); + final long modified = file.lastModified(); + return (d.getTime() - modified) / 1000 / 60; + } + /** * Initialize the executor services for download and processing of the NVD * CVE XML data. diff --git a/dependency-check-maven/pom.xml b/dependency-check-maven/pom.xml index 498bdcb94..905f53374 100644 --- a/dependency-check-maven/pom.xml +++ b/dependency-check-maven/pom.xml @@ -245,6 +245,9 @@ Copyright (c) 2013 Jeremy Long. All Rights Reserved. maven-invoker-plugin 2.0.0 + + 690-threadsafety/pom.xml + ${project.build.directory}/it target/local-repo diff --git a/dependency-check-maven/src/it/617-hierarchical-cross-deps/invoker.properties b/dependency-check-maven/src/it/617-hierarchical-cross-deps/invoker.properties index 1368a7b72..317af6f63 100644 --- a/dependency-check-maven/src/it/617-hierarchical-cross-deps/invoker.properties +++ b/dependency-check-maven/src/it/617-hierarchical-cross-deps/invoker.properties @@ -16,4 +16,4 @@ # Copyright (c) 2014 Jeremy Long. All Rights Reserved. # -invoker.goals = install ${project.groupId}:${project.artifactId}:${project.version}:check -e -T 2 +invoker.goals = install ${project.groupId}:${project.artifactId}:${project.version}:check -e diff --git a/dependency-check-maven/src/it/690-threadsafety/first/pom.xml b/dependency-check-maven/src/it/690-threadsafety/first/pom.xml new file mode 100644 index 000000000..8cddd68e2 --- /dev/null +++ b/dependency-check-maven/src/it/690-threadsafety/first/pom.xml @@ -0,0 +1,35 @@ + + + + 4.0.0 + + org.owasp.test + threaded-parent + 1.0.0-SNAPSHOT + + first + jar + + + log4j + log4j + 1.2.17 + + + diff --git a/dependency-check-maven/src/it/690-threadsafety/fourth/pom.xml b/dependency-check-maven/src/it/690-threadsafety/fourth/pom.xml new file mode 100644 index 000000000..9d000b344 --- /dev/null +++ b/dependency-check-maven/src/it/690-threadsafety/fourth/pom.xml @@ -0,0 +1,35 @@ + + + + 4.0.0 + + org.owasp.test + threaded-parent + 1.0.0-SNAPSHOT + + fourth + jar + + + log4j + log4j + 1.2.17 + + + \ No newline at end of file diff --git a/dependency-check-maven/src/it/690-threadsafety/fourth/src/main/webapp/WEB-INF/web.xml b/dependency-check-maven/src/it/690-threadsafety/fourth/src/main/webapp/WEB-INF/web.xml new file mode 100644 index 000000000..65c96051c --- /dev/null +++ b/dependency-check-maven/src/it/690-threadsafety/fourth/src/main/webapp/WEB-INF/web.xml @@ -0,0 +1,26 @@ + + + + test-app + + index.html + + + diff --git a/dependency-check-maven/src/it/690-threadsafety/invoker.properties b/dependency-check-maven/src/it/690-threadsafety/invoker.properties new file mode 100644 index 000000000..75f5eb6d8 --- /dev/null +++ b/dependency-check-maven/src/it/690-threadsafety/invoker.properties @@ -0,0 +1,19 @@ +# +# This file is part of dependency-check-maven. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Copyright (c) 2014 Jeremy Long. All Rights Reserved. +# + +invoker.goals = install ${project.groupId}:${project.artifactId}:${project.version}:check -e -T 4 diff --git a/dependency-check-maven/src/it/690-threadsafety/pom.xml b/dependency-check-maven/src/it/690-threadsafety/pom.xml new file mode 100644 index 000000000..9872389f0 --- /dev/null +++ b/dependency-check-maven/src/it/690-threadsafety/pom.xml @@ -0,0 +1,31 @@ + + + + 4.0.0 + org.owasp.test + threaded-parent + 1.0.0-SNAPSHOT + pom + + first + second + third + fourth + + \ No newline at end of file diff --git a/dependency-check-maven/src/it/690-threadsafety/postbuild.groovy b/dependency-check-maven/src/it/690-threadsafety/postbuild.groovy new file mode 100644 index 000000000..09ea6f264 --- /dev/null +++ b/dependency-check-maven/src/it/690-threadsafety/postbuild.groovy @@ -0,0 +1,28 @@ +/* + * This file is part of dependency-check-maven. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * Copyright (c) 2014 Jeremy Long. All Rights Reserved. + */ + +import org.apache.commons.io.FileUtils; +import org.apache.commons.lang.StringUtils; + +// Save NVD-CVE for next IT (if not already done) +File datasDwl = new File("target/local-repo/org/owasp/dependency-check-data/3.0", "dc.h2.db"); +File datasSave = new File("target/nvd-cve-backup", "dc.h2.db"); +if (datasDwl.exists() && !datasSave.exists()){ + System.out.println("Save NVD-CVE into backup"); + FileUtils.copyFile(datasDwl, datasSave); +} diff --git a/dependency-check-maven/src/it/690-threadsafety/prebuild.groovy b/dependency-check-maven/src/it/690-threadsafety/prebuild.groovy new file mode 100644 index 000000000..c1e9eda11 --- /dev/null +++ b/dependency-check-maven/src/it/690-threadsafety/prebuild.groovy @@ -0,0 +1,28 @@ +/* + * This file is part of dependency-check-maven. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * Copyright (c) 2014 Jeremy Long. All Rights Reserved. + */ + +import org.apache.commons.io.FileUtils; + +// Load NVD-CVE if not exist and had been saved in a previous IT +File datasDwl = new File("target/local-repo/org/owasp/dependency-check-data/3.0", "dc.h2.db"); +File datasSave = new File("target/nvd-cve-backup", "dc.h2.db"); + +if (!datasDwl.exists() && datasSave.exists()){ + System.out.println("Load NVD-CVE from backup"); + FileUtils.copyFile(datasSave, datasDwl); +} diff --git a/dependency-check-maven/src/it/690-threadsafety/second/pom.xml b/dependency-check-maven/src/it/690-threadsafety/second/pom.xml new file mode 100644 index 000000000..ed04073ef --- /dev/null +++ b/dependency-check-maven/src/it/690-threadsafety/second/pom.xml @@ -0,0 +1,35 @@ + + + + 4.0.0 + + org.owasp.test + threaded-parent + 1.0.0-SNAPSHOT + + second + jar + + + log4j + log4j + 1.2.17 + + + \ No newline at end of file diff --git a/dependency-check-maven/src/it/690-threadsafety/second/src/main/webapp/WEB-INF/web.xml b/dependency-check-maven/src/it/690-threadsafety/second/src/main/webapp/WEB-INF/web.xml new file mode 100644 index 000000000..65c96051c --- /dev/null +++ b/dependency-check-maven/src/it/690-threadsafety/second/src/main/webapp/WEB-INF/web.xml @@ -0,0 +1,26 @@ + + + + test-app + + index.html + + + diff --git a/dependency-check-maven/src/it/690-threadsafety/third/pom.xml b/dependency-check-maven/src/it/690-threadsafety/third/pom.xml new file mode 100644 index 000000000..062992fbb --- /dev/null +++ b/dependency-check-maven/src/it/690-threadsafety/third/pom.xml @@ -0,0 +1,35 @@ + + + + 4.0.0 + + org.owasp.test + threaded-parent + 1.0.0-SNAPSHOT + + third + jar + + + log4j + log4j + 1.2.17 + + + \ No newline at end of file diff --git a/dependency-check-maven/src/it/690-threadsafety/third/src/main/webapp/WEB-INF/web.xml b/dependency-check-maven/src/it/690-threadsafety/third/src/main/webapp/WEB-INF/web.xml new file mode 100644 index 000000000..65c96051c --- /dev/null +++ b/dependency-check-maven/src/it/690-threadsafety/third/src/main/webapp/WEB-INF/web.xml @@ -0,0 +1,26 @@ + + + + test-app + + index.html + + +