**/*IntegrationTest.java
diff --git a/dependency-check-core/src/main/java/org/owasp/dependencycheck/analyzer/ArchiveAnalyzer.java b/dependency-check-core/src/main/java/org/owasp/dependencycheck/analyzer/ArchiveAnalyzer.java
index 80f926369..9614c7142 100644
--- a/dependency-check-core/src/main/java/org/owasp/dependencycheck/analyzer/ArchiveAnalyzer.java
+++ b/dependency-check-core/src/main/java/org/owasp/dependencycheck/analyzer/ArchiveAnalyzer.java
@@ -143,8 +143,10 @@ public class ArchiveAnalyzer extends AbstractAnalyzer implements Analyzer {
*/
@Override
public void initialize() throws Exception {
- final String tmpDir = Settings.getString(Settings.KEYS.TEMP_DIRECTORY, System.getProperty("java.io.tmpdir"));
- final File baseDir = new File(tmpDir);
+ final File baseDir = Settings.getTempDirectory();
+ if (!baseDir.exists()) {
+ baseDir.mkdirs();
+ }
tempFileLocation = File.createTempFile("check", "tmp", baseDir);
if (!tempFileLocation.delete()) {
throw new AnalysisException("Unable to delete temporary file '" + tempFileLocation.getAbsolutePath() + "'.");
diff --git a/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/cpe/CPEAnalyzer.java b/dependency-check-core/src/main/java/org/owasp/dependencycheck/analyzer/CPEAnalyzer.java
similarity index 97%
rename from dependency-check-core/src/main/java/org/owasp/dependencycheck/data/cpe/CPEAnalyzer.java
rename to dependency-check-core/src/main/java/org/owasp/dependencycheck/analyzer/CPEAnalyzer.java
index a458990c8..7280cb932 100644
--- a/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/cpe/CPEAnalyzer.java
+++ b/dependency-check-core/src/main/java/org/owasp/dependencycheck/analyzer/CPEAnalyzer.java
@@ -16,7 +16,7 @@
*
* Copyright (c) 2012 Jeremy Long. All Rights Reserved.
*/
-package org.owasp.dependencycheck.data.cpe;
+package org.owasp.dependencycheck.analyzer;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
@@ -35,14 +35,14 @@ import org.apache.lucene.queryparser.classic.ParseException;
import org.apache.lucene.search.ScoreDoc;
import org.apache.lucene.search.TopDocs;
import org.owasp.dependencycheck.Engine;
-import org.owasp.dependencycheck.analyzer.AnalysisException;
-import org.owasp.dependencycheck.analyzer.AnalysisPhase;
import org.owasp.dependencycheck.data.lucene.LuceneUtils;
import org.owasp.dependencycheck.dependency.Dependency;
import org.owasp.dependencycheck.dependency.Evidence;
import org.owasp.dependencycheck.dependency.Evidence.Confidence;
import org.owasp.dependencycheck.dependency.EvidenceCollection;
-import org.owasp.dependencycheck.analyzer.Analyzer;
+import org.owasp.dependencycheck.data.cpe.CpeIndexReader;
+import org.owasp.dependencycheck.data.cpe.Fields;
+import org.owasp.dependencycheck.data.cpe.IndexEntry;
import org.owasp.dependencycheck.data.nvdcve.CveDB;
import org.owasp.dependencycheck.data.nvdcve.DatabaseException;
import org.owasp.dependencycheck.dependency.Identifier;
@@ -83,9 +83,9 @@ public class CPEAnalyzer implements Analyzer {
*/
static final int STRING_BUILDER_BUFFER = 20;
/**
- * The CPE Index.
+ * The CPE Index Reader.
*/
- private Index cpe;
+ private CpeIndexReader cpe;
/**
* The CVE Database.
*/
@@ -100,7 +100,7 @@ public class CPEAnalyzer implements Analyzer {
* usually occurs when the database is in use by another process.
*/
public void open() throws IOException, DatabaseException {
- cpe = new Index();
+ cpe = new CpeIndexReader();
cpe.open();
cve = new CveDB();
try {
@@ -119,8 +119,12 @@ public class CPEAnalyzer implements Analyzer {
*/
@Override
public void close() {
- cpe.close();
- cve.close();
+ if (cpe != null) {
+ cpe.close();
+ }
+ if (cve != null) {
+ cve.close();
+ }
}
/**
@@ -162,7 +166,6 @@ public class CPEAnalyzer implements Analyzer {
String vendors = addEvidenceWithoutDuplicateTerms("", dependency.getVendorEvidence(), vendorConf);
String products = addEvidenceWithoutDuplicateTerms("", dependency.getProductEvidence(), productConf);
- //boolean found = false;
int ctr = 0;
do {
if (!vendors.isEmpty() && !products.isEmpty()) {
@@ -171,27 +174,20 @@ public class CPEAnalyzer implements Analyzer {
for (IndexEntry e : entries) {
if (verifyEntry(e, dependency)) {
- //found = true; // we found a vendor/product pair. Now find version from the cve db.
final String vendor = e.getVendor();
final String product = e.getProduct();
- // cve.getVersions(vendor, product);
determineIdentifiers(dependency, vendor, product);
}
}
}
- //if (!found) {
vendorConf = reduceConfidence(vendorConf);
if (dependency.getVendorEvidence().contains(vendorConf)) {
- //vendors += " " + dependency.getVendorEvidence().toString(vendorConf);
vendors = addEvidenceWithoutDuplicateTerms(vendors, dependency.getVendorEvidence(), vendorConf);
}
productConf = reduceConfidence(productConf);
if (dependency.getProductEvidence().contains(productConf)) {
- //products += " " + dependency.getProductEvidence().toString(productConf);
products = addEvidenceWithoutDuplicateTerms(products, dependency.getProductEvidence(), productConf);
}
- //}
- //} while (!found && (++ctr) < 4);
} while ((++ctr) < 4);
}
diff --git a/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/nvdcve/NvdCveAnalyzer.java b/dependency-check-core/src/main/java/org/owasp/dependencycheck/analyzer/NvdCveAnalyzer.java
similarity index 97%
rename from dependency-check-core/src/main/java/org/owasp/dependencycheck/data/nvdcve/NvdCveAnalyzer.java
rename to dependency-check-core/src/main/java/org/owasp/dependencycheck/analyzer/NvdCveAnalyzer.java
index 9e6285dbe..7c6be8a04 100644
--- a/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/nvdcve/NvdCveAnalyzer.java
+++ b/dependency-check-core/src/main/java/org/owasp/dependencycheck/analyzer/NvdCveAnalyzer.java
@@ -16,19 +16,18 @@
*
* Copyright (c) 2012 Jeremy Long. All Rights Reserved.
*/
-package org.owasp.dependencycheck.data.nvdcve;
+package org.owasp.dependencycheck.analyzer;
import java.io.IOException;
import java.sql.SQLException;
import java.util.List;
import java.util.Set;
import org.owasp.dependencycheck.Engine;
-import org.owasp.dependencycheck.analyzer.AnalysisException;
-import org.owasp.dependencycheck.analyzer.AnalysisPhase;
import org.owasp.dependencycheck.dependency.Dependency;
import org.owasp.dependencycheck.dependency.Vulnerability;
import org.owasp.dependencycheck.dependency.Identifier;
-import org.owasp.dependencycheck.analyzer.Analyzer;
+import org.owasp.dependencycheck.data.nvdcve.CveDB;
+import org.owasp.dependencycheck.data.nvdcve.DatabaseException;
/**
* NvdCveAnalyzer is a utility class that takes a project dependency and
diff --git a/dependency-check-core/src/main/java/org/owasp/dependencycheck/concurrency/DirectoryLockException.java b/dependency-check-core/src/main/java/org/owasp/dependencycheck/concurrency/DirectoryLockException.java
new file mode 100644
index 000000000..3fbc7ae95
--- /dev/null
+++ b/dependency-check-core/src/main/java/org/owasp/dependencycheck/concurrency/DirectoryLockException.java
@@ -0,0 +1,67 @@
+/*
+ * This file is part of dependency-check-core.
+ *
+ * Dependency-check-core is free software: you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation, either version 3 of the License, or (at your option) any
+ * later version.
+ *
+ * Dependency-check-core is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+ * details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * dependency-check-core. If not, see http://www.gnu.org/licenses/.
+ *
+ * Copyright (c) 2013 Jeremy Long. All Rights Reserved.
+ */
+package org.owasp.dependencycheck.concurrency;
+
+/**
+ * If thrown, indicates that a problem occurred when locking a directory.
+ *
+ * @author Jeremy Long (jeremy.long@owasp.org)
+ */
+public class DirectoryLockException extends Exception {
+
+ /**
+ * Default serial version UID.
+ */
+ private static final long serialVersionUID = 1L;
+
+ /**
+ * Constructs a new Directory Lock Exception.
+ */
+ public DirectoryLockException() {
+ super();
+ }
+
+ /**
+ * Constructs a new Directory Lock Exception.
+ *
+ * @param msg the message describing the exception
+ */
+ public DirectoryLockException(String msg) {
+ super(msg);
+ }
+
+ /**
+ * Constructs a new Directory Lock Exception.
+ *
+ * @param ex the cause of the exception
+ */
+ public DirectoryLockException(Throwable ex) {
+ super(ex);
+ }
+
+ /**
+ * Constructs a new Directory Lock Exception.
+ *
+ * @param msg the message describing the exception
+ * @param ex the cause of the exception
+ */
+ public DirectoryLockException(String msg, Throwable ex) {
+ super(msg, ex);
+ }
+}
diff --git a/dependency-check-core/src/main/java/org/owasp/dependencycheck/concurrency/DirectorySpinLock.java b/dependency-check-core/src/main/java/org/owasp/dependencycheck/concurrency/DirectorySpinLock.java
new file mode 100644
index 000000000..2261846ed
--- /dev/null
+++ b/dependency-check-core/src/main/java/org/owasp/dependencycheck/concurrency/DirectorySpinLock.java
@@ -0,0 +1,267 @@
+/*
+ * This file is part of dependency-check-core.
+ *
+ * Dependency-check-core is free software: you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation, either version 3 of the License, or (at your option) any
+ * later version.
+ *
+ * Dependency-check-core is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+ * details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * dependency-check-core. If not, see http://www.gnu.org/licenses/.
+ *
+ * Copyright (c) 2013 Jeremy Long. All Rights Reserved.
+ */
+package org.owasp.dependencycheck.concurrency;
+
+import java.io.Closeable;
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.RandomAccessFile;
+import java.nio.channels.AsynchronousCloseException;
+import java.nio.channels.ClosedChannelException;
+import java.nio.channels.FileChannel;
+import java.nio.channels.FileLock;
+import java.nio.channels.FileLockInterruptionException;
+import java.nio.channels.NonWritableChannelException;
+import java.nio.channels.OverlappingFileLockException;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+/**
+ * Implements a spin lock on a given directory. If the lock cannot be obtained,
+ * the process will "spin" waiting for an opportunity to obtain the lock
+ * requested.
+ *
+ * @author Jeremy Long (jeremy.long@owasp.org)
+ */
+public class DirectorySpinLock implements Closeable /*, AutoCloseable*/ {
+
+ /**
+ * The name of the lock file.
+ */
+ public static final String LOCK_NAME = "data.lock";
+ /**
+ * The maximum wait period used when attempting to obtain a lock.
+ */
+ public static final int MAX_SPIN = 100;
+ /**
+ * The file channel used to perform the lock.
+ */
+ private FileChannel channel = null;
+ /**
+ * The file used to perform the lock.
+ */
+ private File lockFile = null;
+ /**
+ * The lock object.
+ */
+ private FileLock lock = null;
+ /**
+ * The maximum number of seconds that the spin lock will wait while trying
+ * to obtain a lock.
+ */
+ private long maxWait = MAX_SPIN;
+
+ /**
+ * Get the maximum wait time, in seconds, that the spin lock will wait while
+ * trying to obtain a lock.
+ *
+ * @return the number of seconds the spin lock will wait
+ */
+ public long getMaxWait() {
+ return maxWait / 2; //sleep is for 500, so / 2
+ }
+
+ /**
+ * Set the maximum wait time, in seconds, that the spin lock will wait while
+ * trying to obtain a lock.
+ *
+ * @param maxWait the number of seconds the spin lock will wait
+ */
+ public void setMaxWait(long maxWait) {
+ this.maxWait = maxWait * 2; //sleep is for 500, so * 2
+ }
+
+ /**
+ * Constructs a new spin lock on the given directory.
+ *
+ * @param directory the directory to monitor/lock
+ * @throws InvalidDirectoryException thrown if there is an issue with the
+ * directory provided
+ * @throws DirectoryLockException thrown there is an issue obtaining a
+ * handle to the lock file
+ */
+ public DirectorySpinLock(File directory) throws InvalidDirectoryException, DirectoryLockException {
+ checkDirectory(directory);
+ lockFile = new File(directory, LOCK_NAME);
+ RandomAccessFile file = null;
+ try {
+ file = new RandomAccessFile(lockFile, "rw");
+ } catch (FileNotFoundException ex) {
+ throw new DirectoryLockException("Lock file not found", ex);
+ }
+ channel = file.getChannel();
+ }
+
+ /**
+ * Attempts to obtain an exclusive lock; an exception is thrown if the lock
+ * could not be obtained. This method may block for a few seconds if a lock
+ * cannot be obtained.
+ *
+ * @throws DirectoryLockException thrown if there is an exception obtaining
+ * the lock
+ */
+ public void obtainSharedLock() throws DirectoryLockException {
+ obtainLock(true);
+ }
+
+ /**
+ * Attempts to obtain an exclusive lock; an exception is thrown if the lock
+ * could not be obtained. This method may block for a few seconds if a lock
+ * cannot be obtained.
+ *
+ * @throws DirectoryLockException thrown if there is an exception obtaining
+ * the lock
+ */
+ public void obtainExclusiveLock() throws DirectoryLockException {
+ obtainLock(false);
+ }
+
+ /**
+ * Attempts to obtain a lock; an exception is thrown if the lock could not
+ * be obtained. This method may block for a few seconds if a lock cannot be
+ * obtained.
+ *
+ * @param shared true if the lock is shared, otherwise false
+ * @param maxWait the maximum time to wait, in seconds, while trying to
+ * obtain the lock
+ * @throws DirectoryLockException thrown if there is an exception obtaining
+ * the lock
+ */
+ protected void obtainLock(boolean shared, long maxWait) throws DirectoryLockException {
+ setMaxWait(maxWait);
+ obtainLock(shared);
+ }
+
+ /**
+ * Attempts to obtain a lock; an exception is thrown if the lock could not
+ * be obtained. This method may block for a few seconds if a lock cannot be
+ * obtained.
+ *
+ * @param shared true if the lock is shared, otherwise false
+ * @throws DirectoryLockException thrown if there is an exception obtaining
+ * the lock
+ */
+ protected void obtainLock(boolean shared) throws DirectoryLockException {
+ if (lock != null) {
+ release();
+ }
+ if (channel == null) {
+ throw new DirectoryLockException("Unable to create lock, no file channel exists");
+ }
+ int count = 0;
+ Exception lastException = null;
+ while (lock == null && count++ < maxWait) {
+ try {
+ lock = channel.lock(0, Long.MAX_VALUE, shared);
+ } catch (AsynchronousCloseException ex) {
+ lastException = ex;
+ } catch (ClosedChannelException ex) {
+ lastException = ex;
+ } catch (FileLockInterruptionException ex) {
+ lastException = ex;
+ } catch (OverlappingFileLockException ex) {
+ lastException = ex;
+ } catch (NonWritableChannelException ex) {
+ lastException = ex;
+ } catch (IOException ex) {
+ lastException = ex;
+ }
+ try {
+ Thread.sleep(500);
+ } catch (InterruptedException ex) {
+ Thread.currentThread().interrupt();
+ }
+ }
+ if (lock == null) {
+ if (lastException == null) {
+ throw new DirectoryLockException("Unable to obtain lock");
+ } else {
+ throw new DirectoryLockException("Unable to obtain lock", lastException);
+ }
+ }
+ }
+
+ /**
+ * Performs a few simple rudimentary checks on the specified directory.
+ * Specifically, does the file exist and is it a directory.
+ *
+ * @param directory the File object to inspect
+ * @throws InvalidDirectoryException thrown if the directory is null or is
+ * not a directory
+ */
+ private void checkDirectory(File directory) throws InvalidDirectoryException {
+ if (directory == null) {
+ throw new InvalidDirectoryException("Unable to obtain lock on a null File");
+ }
+ if (!directory.isDirectory()) {
+ final String msg = String.format("File, '%s', does not exist or is not a directory", directory.getAbsolutePath());
+ throw new InvalidDirectoryException(msg);
+ }
+ }
+
+ /**
+ * Releases any locks and closes the underlying channel.
+ *
+ * @throws IOException if an IO Exception occurs
+ */
+ @Override
+ public void close() throws IOException {
+ release();
+// TODO uncomment this once support for 1.6 is dropped.
+// if (lock != null) {
+// try {
+// lock.close();
+// } catch (IOException ex) {
+// Logger.getLogger(DirectorySpinLock.class.getName()).log(Level.FINEST, "Unable to close file lock due to IO Exception", ex);
+// }
+// }
+ if (channel != null) {
+ try {
+ channel.close();
+ } catch (IOException ex) {
+ Logger.getLogger(DirectorySpinLock.class.getName()).log(Level.FINEST, "Unable to close the channel for the file lock", ex);
+ }
+ }
+ if (lockFile != null) {
+ if (lockFile.exists()) {
+ /* yes, this delete could fail which is totally fine. The other
+ * thread holding the lock while delete it.
+ */
+ lockFile.delete();
+ }
+ }
+ }
+
+ /**
+ * Releases the lock. Any exceptions that are thrown by the underlying lock
+ * during the release are ignored.
+ */
+ public void release() {
+ if (lock != null) {
+ try {
+ lock.release();
+ } catch (ClosedChannelException ex) {
+ Logger.getLogger(DirectorySpinLock.class.getName()).log(Level.FINEST, "Uable to release file lock", ex);
+ } catch (IOException ex) {
+ Logger.getLogger(DirectorySpinLock.class.getName()).log(Level.FINEST, "Unable to release file lock due to IO Exception", ex);
+ }
+ }
+ }
+}
diff --git a/dependency-check-core/src/main/java/org/owasp/dependencycheck/concurrency/InvalidDirectoryException.java b/dependency-check-core/src/main/java/org/owasp/dependencycheck/concurrency/InvalidDirectoryException.java
new file mode 100644
index 000000000..08b29acff
--- /dev/null
+++ b/dependency-check-core/src/main/java/org/owasp/dependencycheck/concurrency/InvalidDirectoryException.java
@@ -0,0 +1,67 @@
+/*
+ * This file is part of dependency-check-core.
+ *
+ * Dependency-check-core is free software: you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation, either version 3 of the License, or (at your option) any
+ * later version.
+ *
+ * Dependency-check-core is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+ * details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * dependency-check-core. If not, see http://www.gnu.org/licenses/.
+ *
+ * Copyright (c) 2013 Jeremy Long. All Rights Reserved.
+ */
+package org.owasp.dependencycheck.concurrency;
+
+/**
+ * If thrown, indicates that there is a problem with a directory.
+ *
+ * @author Jeremy Long (jeremy.long@owasp.org)
+ */
+public class InvalidDirectoryException extends Exception {
+
+ /**
+ * Default serial version UID.
+ */
+ private static final long serialVersionUID = 1L;
+
+ /**
+ * Constructs a new Invalid Directory Exception.
+ */
+ public InvalidDirectoryException() {
+ super();
+ }
+
+ /**
+ * Constructs a new Invalid Directory Exception.
+ *
+ * @param msg the message describing the exception
+ */
+ public InvalidDirectoryException(String msg) {
+ super(msg);
+ }
+
+ /**
+ * Constructs a new Invalid Directory Exception.
+ *
+ * @param ex the cause of the exception
+ */
+ public InvalidDirectoryException(Throwable ex) {
+ super(ex);
+ }
+
+ /**
+ * Constructs a new Invalid Directory Exception.
+ *
+ * @param msg the message describing the exception
+ * @param ex the cause of the exception
+ */
+ public InvalidDirectoryException(String msg, Throwable ex) {
+ super(msg, ex);
+ }
+}
diff --git a/dependency-check-core/src/main/java/org/owasp/dependencycheck/concurrency/package-info.java b/dependency-check-core/src/main/java/org/owasp/dependencycheck/concurrency/package-info.java
new file mode 100644
index 000000000..ab7fba4fd
--- /dev/null
+++ b/dependency-check-core/src/main/java/org/owasp/dependencycheck/concurrency/package-info.java
@@ -0,0 +1,11 @@
+/**
+ *
+ *
+ * org.owasp.dependencycheck.concurrency
+ *
+ *
+ * Contains classes used to create shared and exclusive locks on directories.
+ *
+ *
+ */
+package org.owasp.dependencycheck.concurrency;
diff --git a/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/cpe/BaseIndex.java b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/cpe/BaseIndex.java
new file mode 100644
index 000000000..f895a99e8
--- /dev/null
+++ b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/cpe/BaseIndex.java
@@ -0,0 +1,117 @@
+/*
+ * This file is part of dependency-check-core.
+ *
+ * Dependency-check-core is free software: you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation, either version 3 of the License, or (at your option) any
+ * later version.
+ *
+ * Dependency-check-core is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+ * details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * dependency-check-core. If not, see http://www.gnu.org/licenses/.
+ *
+ * Copyright (c) 2013 Jeremy Long. All Rights Reserved.
+ */
+package org.owasp.dependencycheck.data.cpe;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.FSDirectory;
+import org.owasp.dependencycheck.utils.Settings;
+
+/**
+ * The Base Index class used to access the CPE Index.
+ *
+ * @author Jeremy Long (jeremy.long@owasp.org)
+ */
+public abstract class BaseIndex {
+
+ /**
+ * The Lucene directory containing the index.
+ */
+ private Directory directory;
+ /**
+ * Indicates whether or not the Lucene Index is open.
+ */
+ private boolean indexOpen = false;
+
+ /**
+ * Gets the directory.
+ *
+ * @return the directory
+ */
+ public Directory getDirectory() {
+ return directory;
+ }
+
+ /**
+ * Opens the CPE Index.
+ *
+ * @throws IOException is thrown if an IOException occurs opening the index.
+ */
+ public void open() throws IOException {
+ directory = this.openDirectory();
+ indexOpen = true;
+ }
+
+ /**
+ * Closes the CPE Index.
+ */
+ public void close() {
+ try {
+ directory.close();
+ } catch (IOException ex) {
+ final String msg = "Unable to update database due to an IO error.";
+ Logger.getLogger(BaseIndex.class.getName()).log(Level.SEVERE, msg);
+ Logger.getLogger(BaseIndex.class.getName()).log(Level.FINE, null, ex);
+ } finally {
+ directory = null;
+ }
+ indexOpen = false;
+
+ }
+
+ /**
+ * Returns the status of the data source - is the index open.
+ *
+ * @return true or false.
+ */
+ public boolean isOpen() {
+ return indexOpen;
+ }
+
+ /**
+ * Returns the Lucene directory object for the CPE Index.
+ *
+ * @return the Lucene Directory object for the CPE Index.
+ * @throws IOException is thrown if an IOException occurs.
+ */
+ protected Directory openDirectory() throws IOException {
+ final File path = getDataDirectory();
+ return FSDirectory.open(path);
+ }
+
+ /**
+ * Retrieves the directory that the JAR file exists in so that we can ensure
+ * we always use a common data directory.
+ *
+ * @return the data directory for this index.
+ * @throws IOException is thrown if an IOException occurs of course...
+ */
+ public static File getDataDirectory() throws IOException {
+ final File path = Settings.getFile(Settings.KEYS.CPE_DATA_DIRECTORY);
+ if (!path.exists()) {
+ if (!path.mkdirs()) {
+ throw new IOException("Unable to create CPE Data directory");
+ }
+ }
+ return path;
+ }
+}
diff --git a/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/cpe/CpeIndexReader.java b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/cpe/CpeIndexReader.java
new file mode 100644
index 000000000..68f9344f4
--- /dev/null
+++ b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/cpe/CpeIndexReader.java
@@ -0,0 +1,179 @@
+/*
+ * This file is part of dependency-check-core.
+ *
+ * Dependency-check-core is free software: you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation, either version 3 of the License, or (at your option) any
+ * later version.
+ *
+ * Dependency-check-core is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+ * details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * dependency-check-core. If not, see http://www.gnu.org/licenses/.
+ *
+ * Copyright (c) 2013 Jeremy Long. All Rights Reserved.
+ */
+package org.owasp.dependencycheck.data.cpe;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.core.KeywordAnalyzer;
+import org.apache.lucene.analysis.miscellaneous.PerFieldAnalyzerWrapper;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.index.CorruptIndexException;
+import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.queryparser.classic.ParseException;
+import org.apache.lucene.queryparser.classic.QueryParser;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.TopDocs;
+import org.apache.lucene.util.Version;
+import org.owasp.dependencycheck.data.lucene.FieldAnalyzer;
+import org.owasp.dependencycheck.data.lucene.SearchFieldAnalyzer;
+
+/**
+ *
+ * @author Jeremy Long (jeremy.long@owasp.org)
+ */
+public class CpeIndexReader extends BaseIndex {
+
+ /**
+ * The Lucene IndexReader.
+ */
+ private IndexReader indexReader;
+ /**
+ * The Lucene IndexSearcher.
+ */
+ private IndexSearcher indexSearcher;
+ /**
+ * The Lucene Analyzer used for Searching.
+ */
+ private Analyzer searchingAnalyzer;
+ /**
+ * The Lucene QueryParser used for Searching.
+ */
+ private QueryParser queryParser;
+ /**
+ * The search field analyzer for the product field.
+ */
+ private SearchFieldAnalyzer productSearchFieldAnalyzer;
+ /**
+ * The search field analyzer for the vendor field.
+ */
+ private SearchFieldAnalyzer vendorSearchFieldAnalyzer;
+
+ /**
+ * Opens the CPE Index.
+ *
+ * @throws IOException is thrown if an IOException occurs opening the index.
+ */
+ @Override
+ public void open() throws IOException {
+ //TODO add spinlock (shared)
+ super.open();
+ indexReader = DirectoryReader.open(getDirectory());
+ indexSearcher = new IndexSearcher(indexReader);
+ searchingAnalyzer = createSearchingAnalyzer();
+ queryParser = new QueryParser(Version.LUCENE_43, Fields.DOCUMENT_KEY, searchingAnalyzer);
+ }
+
+ /**
+ * Closes the CPE Index.
+ */
+ @Override
+ public void close() {
+ //TODO remove spinlock (shared)
+ if (searchingAnalyzer != null) {
+ searchingAnalyzer.close();
+ searchingAnalyzer = null;
+ }
+ if (indexReader != null) {
+ try {
+ indexReader.close();
+ } catch (IOException ex) {
+ Logger.getLogger(CpeIndexReader.class.getName()).log(Level.FINEST, null, ex);
+ }
+ indexReader = null;
+ }
+ queryParser = null;
+ indexSearcher = null;
+ super.close();
+ }
+
+ /**
+ * Searches the index using the given search string.
+ *
+ * @param searchString the query text
+ * @param maxQueryResults the maximum number of documents to return
+ * @return the TopDocs found by the search
+ * @throws ParseException thrown when the searchString is invalid
+ * @throws IOException is thrown if there is an issue with the underlying
+ * Index
+ */
+ public TopDocs search(String searchString, int maxQueryResults) throws ParseException, IOException {
+ final Query query = queryParser.parse(searchString);
+ return indexSearcher.search(query, maxQueryResults);
+ }
+
+ /**
+ * Searches the index using the given query.
+ *
+ * @param query the query used to search the index
+ * @param maxQueryResults the max number of results to return
+ * @return the TopDocs found be the query
+ * @throws CorruptIndexException thrown if the Index is corrupt
+ * @throws IOException thrown if there is an IOException
+ */
+ public TopDocs search(Query query, int maxQueryResults) throws CorruptIndexException, IOException {
+ resetSearchingAnalyzer();
+ return indexSearcher.search(query, maxQueryResults);
+ }
+
+ /**
+ * Retrieves a document from the Index.
+ *
+ * @param documentId the id of the document to retrieve
+ * @return the Document
+ * @throws IOException thrown if there is an IOException
+ */
+ public Document getDocument(int documentId) throws IOException {
+ return indexSearcher.doc(documentId);
+ }
+
+ /**
+ * Creates an Analyzer for searching the CPE Index.
+ *
+ * @return the CPE Analyzer.
+ */
+ @SuppressWarnings("unchecked")
+ private Analyzer createSearchingAnalyzer() {
+ final Map fieldAnalyzers = new HashMap();
+ fieldAnalyzers.put(Fields.DOCUMENT_KEY, new KeywordAnalyzer());
+ productSearchFieldAnalyzer = new SearchFieldAnalyzer(Version.LUCENE_43);
+ vendorSearchFieldAnalyzer = new SearchFieldAnalyzer(Version.LUCENE_43);
+ fieldAnalyzers.put(Fields.PRODUCT, productSearchFieldAnalyzer);
+ fieldAnalyzers.put(Fields.VENDOR, vendorSearchFieldAnalyzer);
+
+ return new PerFieldAnalyzerWrapper(new FieldAnalyzer(Version.LUCENE_43), fieldAnalyzers);
+ }
+
+ /**
+ * Resets the searching analyzers
+ */
+ private void resetSearchingAnalyzer() {
+ if (productSearchFieldAnalyzer != null) {
+ productSearchFieldAnalyzer.clear();
+ }
+ if (vendorSearchFieldAnalyzer != null) {
+ vendorSearchFieldAnalyzer.clear();
+ }
+ }
+}
diff --git a/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/cpe/CpeIndexWriter.java b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/cpe/CpeIndexWriter.java
new file mode 100644
index 000000000..7efe09f7c
--- /dev/null
+++ b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/cpe/CpeIndexWriter.java
@@ -0,0 +1,149 @@
+/*
+ * This file is part of dependency-check-core.
+ *
+ * Dependency-check-core is free software: you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation, either version 3 of the License, or (at your option) any
+ * later version.
+ *
+ * Dependency-check-core is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+ * details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * dependency-check-core. If not, see http://www.gnu.org/licenses/.
+ *
+ * Copyright (c) 2013 Jeremy Long. All Rights Reserved.
+ */
+package org.owasp.dependencycheck.data.cpe;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.core.KeywordAnalyzer;
+import org.apache.lucene.analysis.miscellaneous.PerFieldAnalyzerWrapper;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.document.StringField;
+import org.apache.lucene.document.TextField;
+import org.apache.lucene.index.CorruptIndexException;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.util.Version;
+import org.owasp.dependencycheck.data.lucene.FieldAnalyzer;
+
+/**
+ *
+ * @author Jeremy Long (jeremy.long@owasp.org)
+ */
+public class CpeIndexWriter extends BaseIndex {
+
+ /**
+ * The IndexWriter for the Lucene index.
+ */
+ private IndexWriter indexWriter;
+ /**
+ * The Lucene Analyzer used for Indexing.
+ */
+ private Analyzer indexingAnalyzer;
+
+ /**
+ * Opens the CPE Index.
+ *
+ * @throws IOException is thrown if an IOException occurs opening the index.
+ */
+ @Override
+ public void open() throws IOException {
+ //TODO add spinlock
+ super.open();
+ indexingAnalyzer = createIndexingAnalyzer();
+ final IndexWriterConfig conf = new IndexWriterConfig(Version.LUCENE_43, indexingAnalyzer);
+ indexWriter = new IndexWriter(getDirectory(), conf);
+ }
+
+ /**
+ * Closes the CPE Index.
+ */
+ @Override
+ public void close() {
+ //TODO remove spinlock
+ if (indexWriter != null) {
+ commit();
+ try {
+ indexWriter.close(true);
+ } catch (CorruptIndexException ex) {
+ final String msg = "Unable to update database, there is a corrupt index.";
+ Logger.getLogger(CpeIndexWriter.class.getName()).log(Level.SEVERE, msg);
+ Logger.getLogger(CpeIndexWriter.class.getName()).log(Level.FINE, null, ex);
+ } catch (IOException ex) {
+ final String msg = "Unable to update database due to an IO error.";
+ Logger.getLogger(CpeIndexWriter.class.getName()).log(Level.SEVERE, msg);
+ Logger.getLogger(CpeIndexWriter.class.getName()).log(Level.FINE, null, ex);
+ } finally {
+ indexWriter = null;
+ }
+ }
+ if (indexingAnalyzer != null) {
+ indexingAnalyzer.close();
+ indexingAnalyzer = null;
+ }
+ super.close();
+ }
+
+ /**
+ * Commits any pending changes.
+ */
+ public void commit() {
+ if (indexWriter != null) {
+ try {
+ indexWriter.forceMerge(1);
+ indexWriter.commit();
+ } catch (CorruptIndexException ex) {
+ final String msg = "Unable to update database, there is a corrupt index.";
+ Logger.getLogger(CpeIndexWriter.class.getName()).log(Level.SEVERE, msg);
+ Logger.getLogger(CpeIndexWriter.class.getName()).log(Level.FINE, null, ex);
+ } catch (IOException ex) {
+ final String msg = "Unable to update database due to an IO error.";
+ Logger.getLogger(CpeIndexWriter.class.getName()).log(Level.SEVERE, msg);
+ Logger.getLogger(CpeIndexWriter.class.getName()).log(Level.FINE, null, ex);
+ }
+ }
+ }
+
+ /**
+ * Creates the indexing analyzer for the CPE Index.
+ *
+ * @return the CPE Analyzer.
+ */
+ @SuppressWarnings("unchecked")
+ private Analyzer createIndexingAnalyzer() {
+ final Map fieldAnalyzers = new HashMap();
+ fieldAnalyzers.put(Fields.DOCUMENT_KEY, new KeywordAnalyzer());
+ return new PerFieldAnalyzerWrapper(new FieldAnalyzer(Version.LUCENE_43), fieldAnalyzers);
+ }
+
+ /**
+ * Saves a CPE IndexEntry into the Lucene index.
+ *
+ * @param entry a CPE entry.
+ * @throws CorruptIndexException is thrown if the index is corrupt.
+ * @throws IOException is thrown if an IOException occurs.
+ */
+ public void saveEntry(IndexEntry entry) throws CorruptIndexException, IOException {
+ final Document doc = new Document();
+ final Field documentKey = new StringField(Fields.DOCUMENT_KEY, entry.getDocumentId(), Field.Store.NO);
+ final Field vendor = new TextField(Fields.VENDOR, entry.getVendor(), Field.Store.YES);
+ final Field product = new TextField(Fields.PRODUCT, entry.getProduct(), Field.Store.YES);
+ doc.add(documentKey);
+ doc.add(vendor);
+ doc.add(product);
+
+ final Term term = new Term(Fields.DOCUMENT_KEY, entry.getDocumentId());
+ indexWriter.updateDocument(term, doc);
+ }
+}
diff --git a/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/cpe/Index.java b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/cpe/Index.java
deleted file mode 100644
index 9c1563ac3..000000000
--- a/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/cpe/Index.java
+++ /dev/null
@@ -1,170 +0,0 @@
-/*
- * This file is part of dependency-check-core.
- *
- * Dependency-check-core is free software: you can redistribute it and/or modify it
- * under the terms of the GNU General Public License as published by the Free
- * Software Foundation, either version 3 of the License, or (at your option) any
- * later version.
- *
- * Dependency-check-core is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
- * details.
- *
- * You should have received a copy of the GNU General Public License along with
- * dependency-check-core. If not, see http://www.gnu.org/licenses/.
- *
- * Copyright (c) 2012 Jeremy Long. All Rights Reserved.
- */
-package org.owasp.dependencycheck.data.cpe;
-
-import java.io.File;
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.Map;
-import org.apache.lucene.analysis.Analyzer;
-import org.apache.lucene.analysis.core.KeywordAnalyzer;
-import org.apache.lucene.analysis.miscellaneous.PerFieldAnalyzerWrapper;
-import org.apache.lucene.document.Document;
-import org.apache.lucene.document.Field;
-import org.apache.lucene.document.TextField;
-import org.apache.lucene.index.CorruptIndexException;
-import org.apache.lucene.index.Term;
-import org.apache.lucene.queryparser.classic.QueryParser;
-import org.apache.lucene.store.Directory;
-import org.apache.lucene.store.FSDirectory;
-import org.apache.lucene.util.Version;
-import org.owasp.dependencycheck.data.lucene.AbstractIndex;
-import org.owasp.dependencycheck.utils.Settings;
-import org.owasp.dependencycheck.data.lucene.FieldAnalyzer;
-import org.owasp.dependencycheck.data.lucene.SearchFieldAnalyzer;
-
-/**
- * The Index class is used to utilize and maintain the CPE Index.
- *
- * @author Jeremy Long (jeremy.long@owasp.org)
- */
-public class Index extends AbstractIndex {
-
- /**
- * Returns the directory that holds the CPE Index.
- *
- * @return the Directory containing the CPE Index.
- * @throws IOException is thrown if an IOException occurs.
- */
- @Override
- public Directory getDirectory() throws IOException {
- final File path = getDataDirectory();
- return FSDirectory.open(path);
- }
-
- /**
- * Retrieves the directory that the JAR file exists in so that we can ensure
- * we always use a common data directory.
- *
- * @return the data directory for this index.
- * @throws IOException is thrown if an IOException occurs of course...
- */
- public File getDataDirectory() throws IOException {
- final File path = Settings.getFile(Settings.KEYS.CPE_DATA_DIRECTORY);
- if (!path.exists()) {
- if (!path.mkdirs()) {
- throw new IOException("Unable to create CPE Data directory");
- }
- }
- return path;
- }
-
- /**
- * Creates an Analyzer for the CPE Index.
- *
- * @return the CPE Analyzer.
- */
- @SuppressWarnings("unchecked")
- @Override
- public Analyzer createIndexingAnalyzer() {
- final Map fieldAnalyzers = new HashMap();
- fieldAnalyzers.put(Fields.DOCUMENT_KEY, new KeywordAnalyzer());
- return new PerFieldAnalyzerWrapper(new FieldAnalyzer(Version.LUCENE_43), fieldAnalyzers);
- }
- /**
- * The search field analyzer for the product field.
- */
- private SearchFieldAnalyzer productSearchFieldAnalyzer;
- /**
- * The search field analyzer for the vendor field.
- */
- private SearchFieldAnalyzer vendorSearchFieldAnalyzer;
-
- /**
- * Creates an Analyzer for searching the CPE Index.
- *
- * @return the CPE Analyzer.
- */
- @SuppressWarnings("unchecked")
- @Override
- public Analyzer createSearchingAnalyzer() {
- final Map fieldAnalyzers = new HashMap();
-
- fieldAnalyzers.put(Fields.DOCUMENT_KEY, new KeywordAnalyzer());
- productSearchFieldAnalyzer = new SearchFieldAnalyzer(Version.LUCENE_43);
- vendorSearchFieldAnalyzer = new SearchFieldAnalyzer(Version.LUCENE_43);
- fieldAnalyzers.put(Fields.PRODUCT, productSearchFieldAnalyzer);
- fieldAnalyzers.put(Fields.VENDOR, vendorSearchFieldAnalyzer);
-
- return new PerFieldAnalyzerWrapper(new FieldAnalyzer(Version.LUCENE_43), fieldAnalyzers);
- }
-
- /**
- * Creates the Lucene QueryParser used when querying the index.
- *
- * @return a QueryParser.
- */
- @Override
- public QueryParser createQueryParser() {
- return new QueryParser(Version.LUCENE_43, Fields.DOCUMENT_KEY, getSearchingAnalyzer());
- }
-
- /**
- * Resets the searching analyzers
- */
- @Override
- protected void resetSearchingAnalyzer() {
- if (productSearchFieldAnalyzer != null) {
- productSearchFieldAnalyzer.clear();
- }
- if (vendorSearchFieldAnalyzer != null) {
- vendorSearchFieldAnalyzer.clear();
- }
- }
-
- /**
- * Saves a CPE IndexEntry into the Lucene index.
- *
- * @param entry a CPE entry.
- * @throws CorruptIndexException is thrown if the index is corrupt.
- * @throws IOException is thrown if an IOException occurs.
- */
- public void saveEntry(IndexEntry entry) throws CorruptIndexException, IOException {
- final Document doc = convertEntryToDoc(entry);
- final Term term = new Term(Fields.DOCUMENT_KEY, entry.getDocumentId());
- getIndexWriter().updateDocument(term, doc);
- }
-
- /**
- * Converts a CPE entry into a Lucene Document.
- *
- * @param entry a CPE IndexEntry.
- * @return a Lucene Document containing a CPE IndexEntry.
- */
- protected Document convertEntryToDoc(IndexEntry entry) {
- final Document doc = new Document();
-
- final Field vendor = new TextField(Fields.VENDOR, entry.getVendor(), Field.Store.YES);
- doc.add(vendor);
-
- final Field product = new TextField(Fields.PRODUCT, entry.getProduct(), Field.Store.YES);
- doc.add(product);
- return doc;
- }
-}
diff --git a/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/lucene/AbstractIndex.java b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/lucene/AbstractIndex.java
deleted file mode 100644
index 2fb2ad66f..000000000
--- a/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/lucene/AbstractIndex.java
+++ /dev/null
@@ -1,341 +0,0 @@
-/*
- * This file is part of dependency-check-core.
- *
- * Dependency-check-core is free software: you can redistribute it and/or modify it
- * under the terms of the GNU General Public License as published by the Free
- * Software Foundation, either version 3 of the License, or (at your option) any
- * later version.
- *
- * Dependency-check-core is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
- * details.
- *
- * You should have received a copy of the GNU General Public License along with
- * dependency-check-core. If not, see http://www.gnu.org/licenses/.
- *
- * Copyright (c) 2012 Jeremy Long. All Rights Reserved.
- */
-package org.owasp.dependencycheck.data.lucene;
-
-import java.io.IOException;
-import java.util.logging.Level;
-import java.util.logging.Logger;
-import org.apache.lucene.analysis.Analyzer;
-import org.apache.lucene.document.Document;
-import org.apache.lucene.index.CorruptIndexException;
-import org.apache.lucene.index.DirectoryReader;
-import org.apache.lucene.index.IndexReader;
-import org.apache.lucene.index.IndexWriter;
-import org.apache.lucene.index.IndexWriterConfig;
-import org.apache.lucene.queryparser.classic.ParseException;
-import org.apache.lucene.queryparser.classic.QueryParser;
-import org.apache.lucene.search.IndexSearcher;
-import org.apache.lucene.search.Query;
-import org.apache.lucene.search.TopDocs;
-import org.apache.lucene.store.Directory;
-import org.apache.lucene.store.LockObtainFailedException;
-import org.apache.lucene.util.Version;
-
-/**
- * The base Index for other index objects. Implements the open and close
- * methods.
- *
- * @author Jeremy Long (jeremy.long@owasp.org)
- */
-public abstract class AbstractIndex {
-
- /**
- * The Lucene directory containing the index.
- */
- private Directory directory;
- /**
- * The IndexWriter for the Lucene index.
- */
- private IndexWriter indexWriter;
- /**
- * The Lucene IndexReader.
- */
- private IndexReader indexReader;
- /**
- * The Lucene IndexSearcher.
- */
- private IndexSearcher indexSearcher;
- /**
- * The Lucene Analyzer used for Indexing.
- */
- private Analyzer indexingAnalyzer;
- /**
- * The Lucene Analyzer used for Searching.
- */
- private Analyzer searchingAnalyzer;
- /**
- * The Lucene QueryParser used for Searching.
- */
- private QueryParser queryParser;
- /**
- * Indicates whether or not the Lucene Index is open.
- */
- private boolean indexOpen = false;
-
- /**
- * Opens the CPE Index.
- *
- * @throws IOException is thrown if an IOException occurs opening the index.
- */
- public void open() throws IOException {
- directory = this.getDirectory();
- indexingAnalyzer = this.getIndexingAnalyzer();
- searchingAnalyzer = this.getSearchingAnalyzer();
- indexOpen = true;
- }
-
- /**
- * Commits any pending changes.
- */
- public void commit() {
- if (indexWriter != null) {
- try {
- indexWriter.commit();
- } catch (CorruptIndexException ex) {
- final String msg = "Unable to update database, there is a corrupt index.";
- Logger.getLogger(AbstractIndex.class.getName()).log(Level.SEVERE, msg);
- Logger.getLogger(AbstractIndex.class.getName()).log(Level.FINE, null, ex);
- } catch (IOException ex) {
- final String msg = "Unable to update database due to an IO error.";
- Logger.getLogger(AbstractIndex.class.getName()).log(Level.SEVERE, msg);
- Logger.getLogger(AbstractIndex.class.getName()).log(Level.FINE, null, ex);
- }
- }
- }
-
- /**
- * Closes the CPE Index.
- */
- public void close() {
- if (indexWriter != null) {
- commit();
- try {
- indexWriter.close(true);
- } catch (CorruptIndexException ex) {
- final String msg = "Unable to update database, there is a corrupt index.";
- Logger.getLogger(AbstractIndex.class.getName()).log(Level.SEVERE, msg);
- Logger.getLogger(AbstractIndex.class.getName()).log(Level.FINE, null, ex);
- } catch (IOException ex) {
- final String msg = "Unable to update database due to an IO error.";
- Logger.getLogger(AbstractIndex.class.getName()).log(Level.SEVERE, msg);
- Logger.getLogger(AbstractIndex.class.getName()).log(Level.FINE, null, ex);
- } finally {
- indexWriter = null;
- }
- }
- if (indexSearcher != null) {
- indexSearcher = null;
- }
-
- if (indexingAnalyzer != null) {
- indexingAnalyzer.close();
- indexingAnalyzer = null;
- }
-
- if (searchingAnalyzer != null) {
- searchingAnalyzer.close();
- searchingAnalyzer = null;
- }
-
- try {
- directory.close();
- } catch (IOException ex) {
- final String msg = "Unable to update database due to an IO error.";
- Logger.getLogger(AbstractIndex.class.getName()).log(Level.SEVERE, msg);
- Logger.getLogger(AbstractIndex.class.getName()).log(Level.FINE, null, ex);
- } finally {
- directory = null;
- }
- indexOpen = false;
- }
-
- /**
- * Returns the status of the data source - is the index open.
- *
- * @return true or false.
- */
- public boolean isOpen() {
- return indexOpen;
- }
-
- /**
- * Opens the Lucene Index Writer.
- *
- * @throws CorruptIndexException is thrown if the Lucene index is corrupt.
- * @throws IOException is thrown if an IOException occurs opening the index.
- */
- public void openIndexWriter() throws CorruptIndexException, IOException {
- if (!isOpen()) {
- open();
- }
- final IndexWriterConfig conf = new IndexWriterConfig(Version.LUCENE_43, indexingAnalyzer);
- indexWriter = new IndexWriter(directory, conf);
- }
-
- /**
- * Retrieves the IndexWriter for the Lucene Index.
- *
- * @return an IndexWriter.
- * @throws CorruptIndexException is thrown if the Lucene Index is corrupt.
- * @throws LockObtainFailedException is thrown if there is an exception
- * obtaining a lock on the Lucene index.
- * @throws IOException is thrown if an IOException occurs opening the index.
- */
- public IndexWriter getIndexWriter() throws CorruptIndexException, LockObtainFailedException, IOException {
- if (indexWriter == null) {
- openIndexWriter();
- }
- return indexWriter;
- }
-
- /**
- * Opens the Lucene Index for reading.
- *
- * @throws CorruptIndexException is thrown if the index is corrupt.
- * @throws IOException is thrown if there is an exception reading the index.
- */
- public void openIndexReader() throws CorruptIndexException, IOException {
- if (!isOpen()) {
- open();
- }
- //indexReader = IndexReader.open(directory, true);
- indexReader = DirectoryReader.open(directory);
- }
-
- /**
- * Returns an IndexSearcher for the Lucene Index.
- *
- * @return an IndexSearcher.
- * @throws CorruptIndexException is thrown if the index is corrupt.
- * @throws IOException is thrown if there is an exception reading the index.
- */
- protected IndexSearcher getIndexSearcher() throws CorruptIndexException, IOException {
- if (indexReader == null) {
- openIndexReader();
- }
- if (indexSearcher == null) {
- indexSearcher = new IndexSearcher(indexReader);
- }
- return indexSearcher;
- }
-
- /**
- * Returns an Analyzer to be used when indexing.
- *
- * @return an Analyzer.
- */
- public Analyzer getIndexingAnalyzer() {
- if (indexingAnalyzer == null) {
- indexingAnalyzer = createIndexingAnalyzer();
- }
- return indexingAnalyzer;
- }
-
- /**
- * Returns an analyzer used for searching the index
- *
- * @return a lucene analyzer
- */
- protected Analyzer getSearchingAnalyzer() {
- if (searchingAnalyzer == null) {
- searchingAnalyzer = createSearchingAnalyzer();
- }
- return searchingAnalyzer;
- }
-
- /**
- * Gets a query parser
- *
- * @return a query parser
- */
- protected QueryParser getQueryParser() {
- if (queryParser == null) {
- queryParser = createQueryParser();
- }
- return queryParser;
- }
-
- /**
- * Searches the index using the given search string.
- *
- * @param searchString the query text
- * @param maxQueryResults the maximum number of documents to return
- * @return the TopDocs found by the search
- * @throws ParseException thrown when the searchString is invalid
- * @throws IOException is thrown if there is an issue with the underlying
- * Index
- */
- public TopDocs search(String searchString, int maxQueryResults) throws ParseException, IOException {
- final QueryParser parser = getQueryParser();
- final Query query = parser.parse(searchString);
- resetSearchingAnalyzer();
- final IndexSearcher is = getIndexSearcher();
- return is.search(query, maxQueryResults);
- }
-
- /**
- * Searches the index using the given query.
- *
- * @param query the query used to search the index
- * @param maxQueryResults the max number of results to return
- * @return the TopDocs found be the query
- * @throws CorruptIndexException thrown if the Index is corrupt
- * @throws IOException thrown if there is an IOException
- */
- public TopDocs search(Query query, int maxQueryResults) throws CorruptIndexException, IOException {
- final IndexSearcher is = getIndexSearcher();
- return is.search(query, maxQueryResults);
- }
-
- /**
- * Retrieves a document from the Index.
- *
- * @param documentId the id of the document to retrieve
- * @return the Document
- * @throws IOException thrown if there is an IOException
- */
- public Document getDocument(int documentId) throws IOException {
- final IndexSearcher is = getIndexSearcher();
- return is.doc(documentId);
- }
-
- /**
- * Gets the directory that contains the Lucene Index.
- *
- * @return a Lucene Directory
- * @throws IOException is thrown when an IOException occurs
- */
- public abstract Directory getDirectory() throws IOException;
-
- /**
- * Creates the Lucene Analyzer used when indexing.
- *
- * @return a Lucene Analyzer
- */
- public abstract Analyzer createIndexingAnalyzer();
-
- /**
- * Creates the Lucene Analyzer used when querying the index.
- *
- * @return a Lucene Analyzer
- */
- public abstract Analyzer createSearchingAnalyzer();
-
- /**
- * Creates the Lucene QueryParser used when querying the index.
- *
- * @return a QueryParser
- */
- public abstract QueryParser createQueryParser();
-
- /**
- * Resets the searching analyzers
- */
- protected abstract void resetSearchingAnalyzer();
-}
diff --git a/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/nvdcve/CveDB.java b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/nvdcve/CveDB.java
index cdf85925a..df8299bdf 100644
--- a/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/nvdcve/CveDB.java
+++ b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/nvdcve/CveDB.java
@@ -58,7 +58,7 @@ public class CveDB {
/**
* The version of the current DB Schema.
*/
- public static final String DB_SCHEMA_VERSION = "2.5";
+ public static final String DB_SCHEMA_VERSION = "2.6";
/**
* Database connection
*/
@@ -162,27 +162,16 @@ public class CveDB {
value = "DMI_EMPTY_DB_PASSWORD",
justification = "Yes, I know... Blank password.")
public void open() throws IOException, SQLException, DatabaseException, ClassNotFoundException {
- /*
- * TODO - make it so we can exteralize the database (lucene index is a problem), could I store it as a blob
- * and just download it when needed?
- */
-// String dbDriver = Settings.getString(Settings.KEYS.DB_DRIVER);
-// String dbConnStr = Settings.getString(Settings.KEYS.DB_CONNECTION_STRING);
-// if (dbDriver != null && dbConnStr != null) {
-// Class.forName(dbDriver);
-// conn = DriverManager.getConnection(dbConnStr);
-// } else { //use the embeded version
final String fileName = CveDB.getDataDirectory().getCanonicalPath();
final File f = new File(fileName, "cve." + DB_SCHEMA_VERSION);
final File check = new File(f.getAbsolutePath() + ".h2.db");
final boolean createTables = !check.exists();
- final String connStr = "jdbc:h2:file:" + f.getAbsolutePath();
+ final String connStr = String.format("jdbc:h2:file:%s;AUTO_SERVER=TRUE", f.getAbsolutePath());
Class.forName("org.h2.Driver");
conn = DriverManager.getConnection(connStr, "sa", "");
if (createTables) {
createTables();
}
-// }
}
/**
diff --git a/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/nvdcve/xml/InvalidDataException.java b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/nvdcve/InvalidDataException.java
similarity index 96%
rename from dependency-check-core/src/main/java/org/owasp/dependencycheck/data/nvdcve/xml/InvalidDataException.java
rename to dependency-check-core/src/main/java/org/owasp/dependencycheck/data/nvdcve/InvalidDataException.java
index 498e9427c..133d17cec 100644
--- a/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/nvdcve/xml/InvalidDataException.java
+++ b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/nvdcve/InvalidDataException.java
@@ -16,7 +16,7 @@
*
* Copyright (c) 2012 Jeremy Long. All Rights Reserved.
*/
-package org.owasp.dependencycheck.data.nvdcve.xml;
+package org.owasp.dependencycheck.data.nvdcve;
/**
* An InvalidDataDataException is a generic exception used when trying to load
diff --git a/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/nvdcve/xml/NvdCve12Handler.java b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/nvdcve/NvdCve12Handler.java
similarity index 99%
rename from dependency-check-core/src/main/java/org/owasp/dependencycheck/data/nvdcve/xml/NvdCve12Handler.java
rename to dependency-check-core/src/main/java/org/owasp/dependencycheck/data/nvdcve/NvdCve12Handler.java
index 3c4a8856a..baca54635 100644
--- a/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/nvdcve/xml/NvdCve12Handler.java
+++ b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/nvdcve/NvdCve12Handler.java
@@ -16,7 +16,7 @@
*
* Copyright (c) 2012 Jeremy Long. All Rights Reserved.
*/
-package org.owasp.dependencycheck.data.nvdcve.xml;
+package org.owasp.dependencycheck.data.nvdcve;
import java.util.ArrayList;
import java.util.HashMap;
diff --git a/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/nvdcve/xml/NvdCve20Handler.java b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/nvdcve/NvdCve20Handler.java
similarity index 98%
rename from dependency-check-core/src/main/java/org/owasp/dependencycheck/data/nvdcve/xml/NvdCve20Handler.java
rename to dependency-check-core/src/main/java/org/owasp/dependencycheck/data/nvdcve/NvdCve20Handler.java
index b1ee760ba..da5686320 100644
--- a/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/nvdcve/xml/NvdCve20Handler.java
+++ b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/nvdcve/NvdCve20Handler.java
@@ -16,7 +16,7 @@
*
* Copyright (c) 2012 Jeremy Long. All Rights Reserved.
*/
-package org.owasp.dependencycheck.data.nvdcve.xml;
+package org.owasp.dependencycheck.data.nvdcve;
import java.io.IOException;
import java.util.List;
@@ -24,9 +24,7 @@ import java.util.Map;
import java.util.logging.Level;
import java.util.logging.Logger;
import org.apache.lucene.index.CorruptIndexException;
-import org.owasp.dependencycheck.data.cpe.Index;
-import org.owasp.dependencycheck.data.nvdcve.CveDB;
-import org.owasp.dependencycheck.data.nvdcve.DatabaseException;
+import org.owasp.dependencycheck.data.cpe.CpeIndexWriter;
import org.owasp.dependencycheck.dependency.Reference;
import org.owasp.dependencycheck.dependency.Vulnerability;
import org.owasp.dependencycheck.dependency.VulnerableSoftware;
@@ -274,14 +272,14 @@ public class NvdCve20Handler extends DefaultHandler {
/**
* the cpe index.
*/
- private Index cpeIndex;
+ private CpeIndexWriter cpeIndex;
/**
- * Sets the cpe index.
+ * Sets the cpe index writer.
*
* @param index the CPE Lucene Index
*/
- void setCpeIndex(Index index) {
+ public void setCpeIndex(CpeIndexWriter index) {
cpeIndex = index;
}
diff --git a/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/nvdcve/xml/DatabaseUpdater.java b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/nvdcve/xml/DatabaseUpdater.java
deleted file mode 100644
index a1d9a9823..000000000
--- a/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/nvdcve/xml/DatabaseUpdater.java
+++ /dev/null
@@ -1,671 +0,0 @@
-/*
- * This file is part of dependency-check-core.
- *
- * Dependency-check-core is free software: you can redistribute it and/or modify it
- * under the terms of the GNU General Public License as published by the Free
- * Software Foundation, either version 3 of the License, or (at your option) any
- * later version.
- *
- * Dependency-check-core is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
- * details.
- *
- * You should have received a copy of the GNU General Public License along with
- * dependency-check-core. If not, see http://www.gnu.org/licenses/.
- *
- * Copyright (c) 2012 Jeremy Long. All Rights Reserved.
- */
-package org.owasp.dependencycheck.data.nvdcve.xml;
-
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileNotFoundException;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.io.OutputStreamWriter;
-import javax.xml.parsers.ParserConfigurationException;
-import org.owasp.dependencycheck.data.CachedWebDataSource;
-import java.net.MalformedURLException;
-import java.net.URL;
-import java.sql.SQLException;
-import java.util.Calendar;
-import java.util.Date;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Properties;
-import java.util.logging.Level;
-import java.util.logging.Logger;
-import javax.xml.parsers.SAXParser;
-import javax.xml.parsers.SAXParserFactory;
-import org.owasp.dependencycheck.data.UpdateException;
-import org.owasp.dependencycheck.data.cpe.Index;
-import org.owasp.dependencycheck.data.nvdcve.CveDB;
-import org.owasp.dependencycheck.dependency.VulnerableSoftware;
-import org.owasp.dependencycheck.utils.DownloadFailedException;
-import org.owasp.dependencycheck.utils.Downloader;
-import org.owasp.dependencycheck.utils.FileUtils;
-import org.owasp.dependencycheck.utils.InvalidSettingException;
-import org.owasp.dependencycheck.utils.Settings;
-import org.xml.sax.SAXException;
-import org.owasp.dependencycheck.data.nvdcve.DatabaseException;
-
-/**
- *
- * @author Jeremy Long (jeremy.long@owasp.org)
- */
-public class DatabaseUpdater implements CachedWebDataSource {
-
- /**
- * The name of the properties file containing the timestamp of the last
- * update.
- */
- private static final String UPDATE_PROPERTIES_FILE = "lastupdated.prop";
- /**
- * The properties file key for the last updated field - used to store the
- * last updated time of the Modified NVD CVE xml file.
- */
- private static final String LAST_UPDATED_MODIFIED = "lastupdated.modified";
- /**
- * Stores the last updated time for each of the NVD CVE files. These
- * timestamps should be updated if we process the modified file within 7
- * days of the last update.
- */
- private static final String LAST_UPDATED_BASE = "lastupdated.";
- /**
- * Modified key word.
- */
- public static final String MODIFIED = "modified";
- /**
- * Reference to the Cve Database.
- */
- private CveDB cveDB = null;
- /**
- * Reference to the Cpe Index.
- */
- private Index cpeIndex = null;
-
- /**
- * Downloads the latest NVD CVE XML file from the web and imports it into
- * the current CVE Database.
- *
- * @throws UpdateException is thrown if there is an error updating the
- * database
- */
- public void update() throws UpdateException {
- try {
- final Map update = updateNeeded();
- int maxUpdates = 0;
- for (NvdCveUrl cve : update.values()) {
- if (cve.getNeedsUpdate()) {
- maxUpdates += 1;
- }
- }
- if (maxUpdates > 3) {
- Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.INFO,
- "NVD CVE requires several updates; this could take a couple of minutes.");
- }
- if (maxUpdates > 0) {
- openDataStores();
- }
- int count = 0;
-
- for (NvdCveUrl cve : update.values()) {
- if (cve.getNeedsUpdate()) {
- count += 1;
- Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.INFO,
- "Updating NVD CVE ({0} of {1})", new Object[]{count, maxUpdates});
- URL url = new URL(cve.getUrl());
- File outputPath = null;
- File outputPath12 = null;
- try {
- Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.INFO,
- "Downloading {0}", cve.getUrl());
-
- outputPath = File.createTempFile("cve" + cve.getId() + "_", ".xml");
- Downloader.fetchFile(url, outputPath, false);
-
- url = new URL(cve.getOldSchemaVersionUrl());
- outputPath12 = File.createTempFile("cve_1_2_" + cve.getId() + "_", ".xml");
- Downloader.fetchFile(url, outputPath12, false);
-
- Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.INFO,
- "Processing {0}", cve.getUrl());
-
- importXML(outputPath, outputPath12);
-
- cveDB.commit();
- cpeIndex.commit();
-
- writeLastUpdatedPropertyFile(cve);
-
- Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.INFO,
- "Completed update {0} of {1}", new Object[]{count, maxUpdates});
- } catch (FileNotFoundException ex) {
- throw new UpdateException(ex);
- } catch (ParserConfigurationException ex) {
- throw new UpdateException(ex);
- } catch (SAXException ex) {
- throw new UpdateException(ex);
- } catch (IOException ex) {
- throw new UpdateException(ex);
- } catch (SQLException ex) {
- throw new UpdateException(ex);
- } catch (DatabaseException ex) {
- throw new UpdateException(ex);
- } catch (ClassNotFoundException ex) {
- throw new UpdateException(ex);
- } finally {
- boolean deleted = false;
- try {
- if (outputPath != null && outputPath.exists()) {
- deleted = outputPath.delete();
- }
- } finally {
- if (outputPath != null && (outputPath.exists() || !deleted)) {
- outputPath.deleteOnExit();
- }
- }
- try {
- deleted = false;
- if (outputPath12 != null && outputPath12.exists()) {
- deleted = outputPath12.delete();
- }
- } finally {
- if (outputPath12 != null && (outputPath12.exists() || !deleted)) {
- outputPath12.deleteOnExit();
- }
- }
- }
- }
- }
- if (maxUpdates >= 1) {
- ensureModifiedIsInLastUpdatedProperties(update);
- cveDB.cleanupDatabase();
- }
- } catch (MalformedURLException ex) {
- throw new UpdateException(ex);
- } catch (DownloadFailedException ex) {
- throw new UpdateException(ex);
- } finally {
- closeDataStores();
- }
- }
-
- /**
- * Imports the NVD CVE XML File into the Lucene Index.
- *
- * @param file the file containing the NVD CVE XML
- * @param oldVersion contains the file containing the NVD CVE XML 1.2
- * @throws ParserConfigurationException is thrown if there is a parser
- * configuration exception
- * @throws SAXException is thrown if there is a SAXException
- * @throws IOException is thrown if there is a ioexception
- * @throws SQLException is thrown if there is a sql exception
- * @throws DatabaseException is thrown if there is a database exception
- * @throws ClassNotFoundException thrown if the h2 database driver cannot be
- * loaded
- */
- private void importXML(File file, File oldVersion)
- throws ParserConfigurationException, SAXException, IOException, SQLException, DatabaseException, ClassNotFoundException {
-
- final SAXParserFactory factory = SAXParserFactory.newInstance();
- final SAXParser saxParser = factory.newSAXParser();
-
- final NvdCve12Handler cve12Handler = new NvdCve12Handler();
- saxParser.parse(oldVersion, cve12Handler);
- final Map> prevVersionVulnMap = cve12Handler.getVulnerabilities();
-
- final NvdCve20Handler cve20Handler = new NvdCve20Handler();
- cve20Handler.setCveDB(cveDB);
- cve20Handler.setPrevVersionVulnMap(prevVersionVulnMap);
- cve20Handler.setCpeIndex(cpeIndex);
- saxParser.parse(file, cve20Handler);
- }
-
- /**
- * Closes the CVE and CPE data stores.
- */
- private void closeDataStores() {
- if (cveDB != null) {
- try {
- cveDB.close();
- } catch (Exception ignore) {
- Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.FINEST, "Error closing the cveDB", ignore);
- }
- }
- if (cpeIndex != null) {
- try {
- cpeIndex.close();
- } catch (Exception ignore) {
- Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.FINEST, "Error closing the cpeIndex", ignore);
- }
- }
- }
-
- /**
- * Opens the CVE and CPE data stores.
- *
- * @throws UpdateException thrown if a data store cannot be opened
- */
- private void openDataStores() throws UpdateException {
- //open the cve and cpe data stores
- try {
- cveDB = new CveDB();
- cveDB.open();
- cpeIndex = new Index();
- cpeIndex.openIndexWriter();
- } catch (IOException ex) {
- closeDataStores();
- Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.FINE, "IO Error opening databases", ex);
- throw new UpdateException("Error updating the CPE/CVE data, please see the log file for more details.");
- } catch (SQLException ex) {
- closeDataStores();
- Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.FINE, "SQL Exception opening databases", ex);
- throw new UpdateException("Error updating the CPE/CVE data, please see the log file for more details.");
- } catch (DatabaseException ex) {
- closeDataStores();
- Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.FINE, "Database Exception opening databases", ex);
- throw new UpdateException("Error updating the CPE/CVE data, please see the log file for more details.");
- } catch (ClassNotFoundException ex) {
- closeDataStores();
- Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.FINE, "Class not found exception opening databases", ex);
- throw new UpdateException("Error updating the CPE/CVE data, please see the log file for more details.");
- }
- }
-
- //
- /**
- * Writes a properties file containing the last updated date to the
- * VULNERABLE_CPE directory.
- *
- * @param updatedValue the updated nvdcve entry
- * @throws UpdateException is thrown if there is an update exception
- */
- private void writeLastUpdatedPropertyFile(NvdCveUrl updatedValue) throws UpdateException {
- if (updatedValue == null) {
- return;
- }
- String dir;
- try {
- dir = CveDB.getDataDirectory().getCanonicalPath();
- } catch (IOException ex) {
- Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.FINE, "Error updating the databases propterty file.", ex);
- throw new UpdateException("Unable to locate last updated properties file.", ex);
- }
- final File cveProp = new File(dir, UPDATE_PROPERTIES_FILE);
- final Properties prop = new Properties();
- if (cveProp.exists()) {
- FileInputStream in = null;
- try {
- in = new FileInputStream(cveProp);
- prop.load(in);
- } catch (Exception ignoreMe) {
- Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.FINEST, null, ignoreMe);
- } finally {
- if (in != null) {
- try {
- in.close();
- } catch (Exception ignoreMeToo) {
- Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.FINEST, null, ignoreMeToo);
- }
- }
- }
-
- }
- prop.put("version", CveDB.DB_SCHEMA_VERSION);
- prop.put(LAST_UPDATED_BASE + updatedValue.getId(), String.valueOf(updatedValue.getTimestamp()));
-
- OutputStream os = null;
- OutputStreamWriter out = null;
- try {
- os = new FileOutputStream(cveProp);
- out = new OutputStreamWriter(os, "UTF-8");
- prop.store(out, dir);
- } catch (FileNotFoundException ex) {
- Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.FINE, null, ex);
- throw new UpdateException("Unable to find last updated properties file.", ex);
- } catch (IOException ex) {
- Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.FINE, null, ex);
- throw new UpdateException("Unable to update last updated properties file.", ex);
- } finally {
- if (out != null) {
- try {
- out.close();
- } catch (IOException ex) {
- Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.FINEST, null, ex);
- }
- }
- if (os != null) {
- try {
- os.close();
- } catch (IOException ex) {
- Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.FINEST, null, ex);
- }
- }
- }
- }
-
- /**
- * Determines if the index needs to be updated. This is done by fetching the
- * nvd cve meta data and checking the last update date. If the data needs to
- * be refreshed this method will return the NvdCveUrl for the files that
- * need to be updated.
- *
- * @return the NvdCveUrl of the files that need to be updated.
- * @throws MalformedURLException is thrown if the URL for the NVD CVE Meta
- * data is incorrect.
- * @throws DownloadFailedException is thrown if there is an error.
- * downloading the nvd cve download data file.
- * @throws UpdateException Is thrown if there is an issue with the last
- * updated properties file.
- */
- public Map updateNeeded() throws MalformedURLException, DownloadFailedException, UpdateException {
-
- Map currentlyPublished;
- try {
- currentlyPublished = retrieveCurrentTimestampsFromWeb();
- } catch (InvalidDataException ex) {
- final String msg = "Unable to retrieve valid timestamp from nvd cve downloads page";
- Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.FINE, msg, ex);
- throw new DownloadFailedException(msg, ex);
-
- } catch (InvalidSettingException ex) {
- Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.FINE, "Invalid setting found when retrieving timestamps", ex);
- throw new DownloadFailedException("Invalid settings", ex);
- }
-
- if (currentlyPublished == null) {
- throw new DownloadFailedException("Unable to retrieve valid timestamp from nvd cve downloads page");
- }
- String dir;
- try {
- dir = CveDB.getDataDirectory().getCanonicalPath();
- } catch (IOException ex) {
- Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.FINE, "CveDB data directory doesn't exist?", ex);
- throw new UpdateException("Unable to locate last updated properties file.", ex);
- }
-
- final File f = new File(dir);
- if (f.exists()) {
- final File cveProp = new File(dir, UPDATE_PROPERTIES_FILE);
- if (cveProp.exists()) {
- final Properties prop = new Properties();
- InputStream is = null;
- try {
- is = new FileInputStream(cveProp);
- prop.load(is);
-
- boolean deleteAndRecreate = false;
- float version;
-
- if (prop.getProperty("version") == null) {
- deleteAndRecreate = true;
- } else {
- try {
- version = Float.parseFloat(prop.getProperty("version"));
- final float currentVersion = Float.parseFloat(CveDB.DB_SCHEMA_VERSION);
- if (currentVersion > version) {
- deleteAndRecreate = true;
- }
- } catch (NumberFormatException ex) {
- deleteAndRecreate = true;
- }
- }
- if (deleteAndRecreate) {
- Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.INFO, "The database version is old. Rebuilding the database.");
- is.close();
- //this is an old version of the lucene index - just delete it
- FileUtils.delete(f);
-
- //this importer also updates the CPE index and it is also using an old version
- final Index cpeId = new Index();
- final File cpeDir = cpeId.getDataDirectory();
- FileUtils.delete(cpeDir);
- return currentlyPublished;
- }
-
- final long lastUpdated = Long.parseLong(prop.getProperty(LAST_UPDATED_MODIFIED, "0"));
- final Date now = new Date();
- final int days = Settings.getInt(Settings.KEYS.CVE_MODIFIED_VALID_FOR_DAYS, 7);
- final int start = Settings.getInt(Settings.KEYS.CVE_START_YEAR, 2002);
- final int end = Calendar.getInstance().get(Calendar.YEAR);
- if (lastUpdated == currentlyPublished.get(MODIFIED).timestamp) {
- currentlyPublished.clear(); //we don't need to update anything.
- } else if (withinRange(lastUpdated, now.getTime(), days)) {
- currentlyPublished.get(MODIFIED).setNeedsUpdate(true);
- for (int i = start; i <= end; i++) {
- currentlyPublished.get(String.valueOf(i)).setNeedsUpdate(false);
- }
- } else { //we figure out which of the several XML files need to be downloaded.
- currentlyPublished.get(MODIFIED).setNeedsUpdate(false);
- for (int i = start; i <= end; i++) {
- final NvdCveUrl cve = currentlyPublished.get(String.valueOf(i));
- long currentTimestamp = 0;
- try {
- currentTimestamp = Long.parseLong(prop.getProperty(LAST_UPDATED_BASE + String.valueOf(i), "0"));
- } catch (NumberFormatException ex) {
- final String msg = String.format("Error parsing '%s' '%s' from nvdcve.lastupdated",
- LAST_UPDATED_BASE, String.valueOf(i));
- Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.FINE, msg, ex);
- }
- if (currentTimestamp == cve.getTimestamp()) {
- cve.setNeedsUpdate(false); //they default to true.
- }
- }
- }
- } catch (FileNotFoundException ex) {
- Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.FINEST, null, ex);
- } catch (IOException ex) {
- Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.FINEST, null, ex);
- } catch (NumberFormatException ex) {
- Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.FINEST, null, ex);
- } finally {
- if (is != null) {
- try {
- is.close();
- } catch (IOException ex) {
- Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.FINEST, null, ex);
- }
- }
- }
- }
- }
- return currentlyPublished;
- }
-
- /**
- * Determines if the epoch date is within the range specified of the
- * compareTo epoch time. This takes the (compareTo-date)/1000/60/60/24 to
- * get the number of days. If the calculated days is less then the range the
- * date is considered valid.
- *
- * @param date the date to be checked.
- * @param compareTo the date to compare to.
- * @param range the range in days to be considered valid.
- * @return whether or not the date is within the range.
- */
- private boolean withinRange(long date, long compareTo, int range) {
- final double differenceInDays = (compareTo - date) / 1000.0 / 60.0 / 60.0 / 24.0;
- return differenceInDays < range;
- }
-
- /**
- * Retrieves the timestamps from the NVD CVE meta data file.
- *
- * @return the timestamp from the currently published nvdcve downloads page
- * @throws MalformedURLException thrown if the URL for the NVD CCE Meta data
- * is incorrect.
- * @throws DownloadFailedException thrown if there is an error downloading
- * the nvd cve meta data file
- * @throws InvalidDataException thrown if there is an exception parsing the
- * timestamps
- * @throws InvalidSettingException thrown if the settings are invalid
- */
- protected Map retrieveCurrentTimestampsFromWeb()
- throws MalformedURLException, DownloadFailedException, InvalidDataException, InvalidSettingException {
-
- final Map map = new HashMap();
- String retrieveUrl = Settings.getString(Settings.KEYS.CVE_MODIFIED_20_URL);
-
- NvdCveUrl item = new NvdCveUrl();
- item.setNeedsUpdate(false); //the others default to true, to make life easier later this should default to false.
- item.setId(MODIFIED);
- item.setUrl(retrieveUrl);
- item.setOldSchemaVersionUrl(Settings.getString(Settings.KEYS.CVE_MODIFIED_12_URL));
-
- item.timestamp = Downloader.getLastModified(new URL(retrieveUrl));
- map.put(MODIFIED, item);
-
- final int start = Settings.getInt(Settings.KEYS.CVE_START_YEAR);
- final int end = Calendar.getInstance().get(Calendar.YEAR);
- final String baseUrl20 = Settings.getString(Settings.KEYS.CVE_SCHEMA_2_0);
- final String baseUrl12 = Settings.getString(Settings.KEYS.CVE_SCHEMA_1_2);
- for (int i = start; i <= end; i++) {
- retrieveUrl = String.format(baseUrl20, i);
- item = new NvdCveUrl();
- item.setId(Integer.toString(i));
- item.setUrl(retrieveUrl);
- item.setOldSchemaVersionUrl(String.format(baseUrl12, i));
- item.setTimestamp(Downloader.getLastModified(new URL(retrieveUrl)));
- map.put(item.id, item);
- }
- return map;
- }
-
- /**
- * Method to double check that the "modified" nvdcve file is listed and has
- * a timestamp in the last updated properties file.
- *
- * @param update a set of updated NvdCveUrl objects
- */
- private void ensureModifiedIsInLastUpdatedProperties(Map update) {
- try {
- writeLastUpdatedPropertyFile(update.get(MODIFIED));
- } catch (UpdateException ex) {
- Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.FINE, null, ex);
- }
- }
-
- /**
- * A pojo that contains the Url and timestamp of the current NvdCve XML
- * files.
- */
- protected static class NvdCveUrl {
-
- /**
- * an id.
- */
- private String id;
-
- /**
- * Get the value of id.
- *
- * @return the value of id
- */
- public String getId() {
- return id;
- }
-
- /**
- * Set the value of id.
- *
- * @param id new value of id
- */
- public void setId(String id) {
- this.id = id;
- }
- /**
- * a url.
- */
- private String url;
-
- /**
- * Get the value of url.
- *
- * @return the value of url
- */
- public String getUrl() {
- return url;
- }
-
- /**
- * Set the value of url.
- *
- * @param url new value of url
- */
- public void setUrl(String url) {
- this.url = url;
- }
- /**
- * The 1.2 schema URL.
- */
- private String oldSchemaVersionUrl;
-
- /**
- * Get the value of oldSchemaVersionUrl.
- *
- * @return the value of oldSchemaVersionUrl
- */
- public String getOldSchemaVersionUrl() {
- return oldSchemaVersionUrl;
- }
-
- /**
- * Set the value of oldSchemaVersionUrl.
- *
- * @param oldSchemaVersionUrl new value of oldSchemaVersionUrl
- */
- public void setOldSchemaVersionUrl(String oldSchemaVersionUrl) {
- this.oldSchemaVersionUrl = oldSchemaVersionUrl;
- }
- /**
- * a timestamp - epoch time.
- */
- private long timestamp;
-
- /**
- * Get the value of timestamp - epoch time.
- *
- * @return the value of timestamp - epoch time
- */
- public long getTimestamp() {
- return timestamp;
- }
-
- /**
- * Set the value of timestamp - epoch time.
- *
- * @param timestamp new value of timestamp - epoch time
- */
- public void setTimestamp(long timestamp) {
- this.timestamp = timestamp;
- }
- /**
- * indicates whether or not this item should be updated.
- */
- private boolean needsUpdate = true;
-
- /**
- * Get the value of needsUpdate.
- *
- * @return the value of needsUpdate
- */
- public boolean getNeedsUpdate() {
- return needsUpdate;
- }
-
- /**
- * Set the value of needsUpdate.
- *
- * @param needsUpdate new value of needsUpdate
- */
- public void setNeedsUpdate(boolean needsUpdate) {
- this.needsUpdate = needsUpdate;
- }
- }
- //
-}
diff --git a/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/DataStoreMetaInfo.java b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/DataStoreMetaInfo.java
new file mode 100644
index 000000000..b6aed84c1
--- /dev/null
+++ b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/DataStoreMetaInfo.java
@@ -0,0 +1,241 @@
+/*
+ * This file is part of dependency-check-core.
+ *
+ * Dependency-check-core is free software: you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation, either version 3 of the License, or (at your option) any
+ * later version.
+ *
+ * Dependency-check-core is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+ * details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * dependency-check-core. If not, see http://www.gnu.org/licenses/.
+ *
+ * Copyright (c) 2013 Jeremy Long. All Rights Reserved.
+ */
+package org.owasp.dependencycheck.data.update;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.io.OutputStreamWriter;
+import java.util.Properties;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+import org.owasp.dependencycheck.data.UpdateException;
+import org.owasp.dependencycheck.data.nvdcve.CveDB;
+import org.owasp.dependencycheck.utils.Settings;
+
+/**
+ *
+ * @author Jeremy Long (jeremy.long@owasp.org)
+ */
+public class DataStoreMetaInfo {
+
+ /**
+ * Batch key word, used as key to store information about batch mode.
+ */
+ public static final String BATCH = "batch";
+ /**
+ * Modified key word, used as a key to store information about the modified
+ * file (i.e. the containing the last 8 days of updates)..
+ */
+ public static final String MODIFIED = "modified";
+ /**
+ * The name of the properties file containing the timestamp of the last
+ * update.
+ */
+ private static final String UPDATE_PROPERTIES_FILE = "data.properties";
+ /**
+ * The properties file key for the last updated field - used to store the
+ * last updated time of the Modified NVD CVE xml file.
+ */
+ public static final String LAST_UPDATED = "lastupdated.modified";
+ /**
+ * Stores the last updated time for each of the NVD CVE files. These
+ * timestamps should be updated if we process the modified file within 7
+ * days of the last update.
+ */
+ public static final String LAST_UPDATED_BASE = "lastupdated.";
+ /**
+ * A collection of properties about the data.
+ */
+ private Properties properties = new Properties();
+ /**
+ * Indicates whether or not the updates are using a batch update mode or
+ * not.
+ */
+ private boolean batchUpdateMode;
+
+ /**
+ * Get the value of batchUpdateMode.
+ *
+ * @return the value of batchUpdateMode
+ */
+ protected boolean isBatchUpdateMode() {
+ return batchUpdateMode;
+ }
+
+ /**
+ * Set the value of batchUpdateMode.
+ *
+ * @param batchUpdateMode new value of batchUpdateMode
+ */
+ protected void setBatchUpdateMode(boolean batchUpdateMode) {
+ this.batchUpdateMode = batchUpdateMode;
+ }
+
+ /**
+ * Constructs a new data properties object.
+ */
+ public DataStoreMetaInfo() {
+ batchUpdateMode = !Settings.getString(Settings.KEYS.BATCH_UPDATE_URL, "").isEmpty();
+ loadProperties();
+ }
+
+ /**
+ * Loads the data's meta properties.
+ */
+ private void loadProperties() {
+ final File file = getPropertiesFile();
+ if (file.exists()) {
+ InputStream is = null;
+ try {
+ is = new FileInputStream(file);
+ } catch (FileNotFoundException ignore) {
+ //we will never get here as we check for existence above.
+ Logger.getLogger(DataStoreMetaInfo.class.getName()).log(Level.FINEST, null, ignore);
+ }
+ try {
+ properties.load(is);
+ } catch (IOException ex) {
+ final String msg = String.format("Unable to load properties file '%s'", file.getPath());
+ Logger.getLogger(DataStoreMetaInfo.class.getName()).log(Level.WARNING, msg);
+ Logger.getLogger(DataStoreMetaInfo.class.getName()).log(Level.FINE, null, ex);
+ } finally {
+ if (is != null) {
+ try {
+ is.close();
+ } catch (IOException ex) {
+ final String msg = String.format("Unable to close properties file '%s'", file.getPath());
+ Logger.getLogger(DataStoreMetaInfo.class.getName()).log(Level.WARNING, msg);
+ Logger.getLogger(DataStoreMetaInfo.class.getName()).log(Level.FINE, null, ex);
+ }
+ }
+ }
+ }
+ }
+
+ /**
+ * Returns whether or not any properties are set.
+ *
+ * @return whether or not any properties are set
+ */
+ public boolean isEmpty() {
+ return properties.isEmpty();
+ }
+
+ /**
+ * Writes a properties file containing the last updated date to the
+ * VULNERABLE_CPE directory.
+ *
+ * @param updatedValue the updated nvdcve entry
+ * @throws UpdateException is thrown if there is an update exception
+ */
+ public void save(NvdCveInfo updatedValue) throws UpdateException {
+ if (updatedValue == null) {
+ return;
+ }
+ final File cveProp = getPropertiesFile();
+ final Properties prop = new Properties();
+ if (cveProp.exists()) {
+ FileInputStream in = null;
+ try {
+ in = new FileInputStream(cveProp);
+ prop.load(in);
+ } catch (Exception ignoreMe) {
+ Logger.getLogger(DataStoreMetaInfo.class.getName()).log(Level.FINEST, null, ignoreMe);
+ } finally {
+ if (in != null) {
+ try {
+ in.close();
+ } catch (Exception ignoreMeToo) {
+ Logger.getLogger(DataStoreMetaInfo.class.getName()).log(Level.FINEST, null, ignoreMeToo);
+ }
+ }
+ }
+ }
+ prop.put("version", CveDB.DB_SCHEMA_VERSION);
+ prop.put(LAST_UPDATED_BASE + updatedValue.getId(), String.valueOf(updatedValue.getTimestamp()));
+
+ OutputStream os = null;
+ OutputStreamWriter out = null;
+ try {
+ os = new FileOutputStream(cveProp);
+ out = new OutputStreamWriter(os, "UTF-8");
+ prop.store(out, "Meta data about data and data sources used by dependency-check");
+ } catch (FileNotFoundException ex) {
+ Logger.getLogger(DataStoreMetaInfo.class.getName()).log(Level.FINE, null, ex);
+ throw new UpdateException("Unable to find last updated properties file.", ex);
+ } catch (IOException ex) {
+ Logger.getLogger(DataStoreMetaInfo.class.getName()).log(Level.FINE, null, ex);
+ throw new UpdateException("Unable to update last updated properties file.", ex);
+ } finally {
+ if (out != null) {
+ try {
+ out.close();
+ } catch (IOException ex) {
+ Logger.getLogger(DataStoreMetaInfo.class.getName()).log(Level.FINEST, null, ex);
+ }
+ }
+ if (os != null) {
+ try {
+ os.close();
+ } catch (IOException ex) {
+ Logger.getLogger(DataStoreMetaInfo.class.getName()).log(Level.FINEST, null, ex);
+ }
+ }
+ }
+ }
+
+ /**
+ * Returns the property value for the given key. If the key is not contained
+ * in the underlying properties null is returned.
+ *
+ * @param key the property key
+ * @return the value of the property
+ */
+ public String getProperty(String key) {
+ return properties.getProperty(key);
+ }
+
+ /**
+ * Returns the property value for the given key. If the key is not contained
+ * in the underlying properties the default value is returned.
+ *
+ * @param key the property key
+ * @param defaultValue the default value
+ * @return the value of the property
+ */
+ public String getProperty(String key, String defaultValue) {
+ return properties.getProperty(key, defaultValue);
+ }
+
+ /**
+ * Retrieves the properties file.
+ *
+ * @return the properties file
+ */
+ public static File getPropertiesFile() {
+ final File dataDirectory = Settings.getFile(Settings.KEYS.DATA_DIRECTORY);
+ final File file = new File(dataDirectory, UPDATE_PROPERTIES_FILE);
+ return file;
+ }
+}
diff --git a/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/DatabaseUpdater.java b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/DatabaseUpdater.java
new file mode 100644
index 000000000..f69ae3c63
--- /dev/null
+++ b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/DatabaseUpdater.java
@@ -0,0 +1,572 @@
+/*
+ * This file is part of dependency-check-core.
+ *
+ * Dependency-check-core is free software: you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation, either version 3 of the License, or (at your option) any
+ * later version.
+ *
+ * Dependency-check-core is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+ * details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * dependency-check-core. If not, see http://www.gnu.org/licenses/.
+ *
+ * Copyright (c) 2012 Jeremy Long. All Rights Reserved.
+ */
+package org.owasp.dependencycheck.data.update;
+
+import org.owasp.dependencycheck.data.nvdcve.NvdCve12Handler;
+import org.owasp.dependencycheck.data.nvdcve.NvdCve20Handler;
+import org.owasp.dependencycheck.data.nvdcve.InvalidDataException;
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import javax.xml.parsers.ParserConfigurationException;
+import org.xml.sax.SAXException;
+import org.owasp.dependencycheck.data.CachedWebDataSource;
+import java.net.MalformedURLException;
+import java.net.URISyntaxException;
+import java.net.URL;
+import java.sql.SQLException;
+import java.util.Calendar;
+import java.util.Date;
+import java.util.List;
+import java.util.Map;
+import java.util.TreeMap;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+import javax.xml.parsers.SAXParser;
+import javax.xml.parsers.SAXParserFactory;
+import org.owasp.dependencycheck.data.UpdateException;
+import org.owasp.dependencycheck.data.cpe.CpeIndexWriter;
+import org.owasp.dependencycheck.data.nvdcve.CveDB;
+import org.owasp.dependencycheck.dependency.VulnerableSoftware;
+import org.owasp.dependencycheck.utils.DownloadFailedException;
+import org.owasp.dependencycheck.utils.Downloader;
+import org.owasp.dependencycheck.utils.FileUtils;
+import org.owasp.dependencycheck.utils.Settings;
+import org.owasp.dependencycheck.data.nvdcve.DatabaseException;
+import org.owasp.dependencycheck.utils.InvalidSettingException;
+import static org.owasp.dependencycheck.data.update.DataStoreMetaInfo.BATCH;
+import static org.owasp.dependencycheck.data.update.DataStoreMetaInfo.MODIFIED;
+
+/**
+ * Class responsible for updating the CPE and NVDCVE data stores.
+ *
+ * @author Jeremy Long (jeremy.long@owasp.org)
+ */
+public class DatabaseUpdater implements CachedWebDataSource {
+
+ /**
+ * Utility to read and write meta-data about the data.
+ */
+ private DataStoreMetaInfo properties = null;
+ /**
+ * Reference to the Cve Database.
+ */
+ private CveDB cveDB = null;
+ /**
+ * Reference to the Cpe Index.
+ */
+ private CpeIndexWriter cpeIndex = null;
+ /**
+ * A flag indicating whether or not the batch update should be performed.
+ */
+ private boolean doBatchUpdate;
+
+ /**
+ * Get the value of doBatchUpdate
+ *
+ * @return the value of doBatchUpdate
+ */
+ protected boolean isDoBatchUpdate() {
+ return doBatchUpdate;
+ }
+
+ /**
+ * Set the value of doBatchUpdate
+ *
+ * @param doBatchUpdate new value of doBatchUpdate
+ */
+ protected void setDoBatchUpdate(boolean doBatchUpdate) {
+ this.doBatchUpdate = doBatchUpdate;
+ }
+
+ /**
+ * Downloads the latest NVD CVE XML file from the web and imports it into
+ * the current CVE Database.
+ *
+ * @throws UpdateException is thrown if there is an error updating the
+ * database
+ */
+ @Override
+ public void update() throws UpdateException {
+ doBatchUpdate = false;
+ properties = new DataStoreMetaInfo();
+ try {
+ final Map update = updateNeeded();
+ int maxUpdates = 0;
+ for (NvdCveInfo cve : update.values()) {
+ if (cve.getNeedsUpdate()) {
+ maxUpdates += 1;
+ }
+ }
+ if (maxUpdates > 3 && !properties.isBatchUpdateMode()) {
+ Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.INFO,
+ "NVD CVE requires several updates; this could take a couple of minutes.");
+ }
+ if (maxUpdates > 0 && !isDoBatchUpdate()) {
+ openDataStores();
+ }
+
+ if (properties.isBatchUpdateMode() && isDoBatchUpdate()) {
+ try {
+ performBatchUpdate();
+ openDataStores();
+ } catch (IOException ex) {
+ throw new UpdateException("Unable to perform batch update", ex);
+ }
+ }
+
+ int count = 0;
+ for (NvdCveInfo cve : update.values()) {
+ if (cve.getNeedsUpdate()) {
+ count += 1;
+ Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.INFO,
+ "Updating NVD CVE ({0} of {1})", new Object[]{count, maxUpdates});
+ URL url = new URL(cve.getUrl());
+ File outputPath = null;
+ File outputPath12 = null;
+ try {
+ Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.INFO,
+ "Downloading {0}", cve.getUrl());
+ outputPath = File.createTempFile("cve" + cve.getId() + "_", ".xml");
+ Downloader.fetchFile(url, outputPath);
+
+ url = new URL(cve.getOldSchemaVersionUrl());
+ outputPath12 = File.createTempFile("cve_1_2_" + cve.getId() + "_", ".xml");
+ Downloader.fetchFile(url, outputPath12);
+
+ Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.INFO,
+ "Processing {0}", cve.getUrl());
+
+ importXML(outputPath, outputPath12);
+
+ cveDB.commit();
+ cpeIndex.commit();
+
+ properties.save(cve);
+
+ Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.INFO,
+ "Completed update {0} of {1}", new Object[]{count, maxUpdates});
+ } catch (FileNotFoundException ex) {
+ throw new UpdateException(ex);
+ } catch (ParserConfigurationException ex) {
+ throw new UpdateException(ex);
+ } catch (SAXException ex) {
+ throw new UpdateException(ex);
+ } catch (IOException ex) {
+ throw new UpdateException(ex);
+ } catch (SQLException ex) {
+ throw new UpdateException(ex);
+ } catch (DatabaseException ex) {
+ throw new UpdateException(ex);
+ } catch (ClassNotFoundException ex) {
+ throw new UpdateException(ex);
+ } finally {
+ boolean deleted = false;
+ try {
+ if (outputPath != null && outputPath.exists()) {
+ deleted = outputPath.delete();
+ }
+ } finally {
+ if (outputPath != null && (outputPath.exists() || !deleted)) {
+ outputPath.deleteOnExit();
+ }
+ }
+ try {
+ deleted = false;
+ if (outputPath12 != null && outputPath12.exists()) {
+ deleted = outputPath12.delete();
+ }
+ } finally {
+ if (outputPath12 != null && (outputPath12.exists() || !deleted)) {
+ outputPath12.deleteOnExit();
+ }
+ }
+ }
+ }
+ }
+ if (maxUpdates >= 1) { //ensure the modified file date gets written
+ properties.save(update.get(MODIFIED));
+ cveDB.cleanupDatabase();
+ }
+ if (update.get(BATCH) != null) {
+ properties.save(update.get(BATCH));
+ }
+ } catch (MalformedURLException ex) {
+ throw new UpdateException(ex);
+ } catch (DownloadFailedException ex) {
+ throw new UpdateException(ex);
+ } finally {
+ closeDataStores();
+ }
+ }
+
+ /**
+ * Imports the NVD CVE XML File into the Lucene Index.
+ *
+ * @param file the file containing the NVD CVE XML
+ * @param oldVersion contains the file containing the NVD CVE XML 1.2
+ * @throws ParserConfigurationException is thrown if there is a parser
+ * configuration exception
+ * @throws SAXException is thrown if there is a SAXException
+ * @throws IOException is thrown if there is a IO Exception
+ * @throws SQLException is thrown if there is a SQL exception
+ * @throws DatabaseException is thrown if there is a database exception
+ * @throws ClassNotFoundException thrown if the h2 database driver cannot be
+ * loaded
+ */
+ private void importXML(File file, File oldVersion)
+ throws ParserConfigurationException, SAXException, IOException, SQLException, DatabaseException, ClassNotFoundException {
+
+ final SAXParserFactory factory = SAXParserFactory.newInstance();
+ final SAXParser saxParser = factory.newSAXParser();
+
+ final NvdCve12Handler cve12Handler = new NvdCve12Handler();
+ saxParser.parse(oldVersion, cve12Handler);
+ final Map> prevVersionVulnMap = cve12Handler.getVulnerabilities();
+
+ final NvdCve20Handler cve20Handler = new NvdCve20Handler();
+ cve20Handler.setCveDB(cveDB);
+ cve20Handler.setPrevVersionVulnMap(prevVersionVulnMap);
+ cve20Handler.setCpeIndex(cpeIndex);
+ saxParser.parse(file, cve20Handler);
+ }
+
+ /**
+ * Deletes the existing data directories.
+ *
+ * @throws IOException thrown if the directory cannot be deleted
+ */
+ protected void deleteExistingData() throws IOException {
+ File data = Settings.getFile(Settings.KEYS.CVE_DATA_DIRECTORY);
+ if (data.exists()) {
+ FileUtils.delete(data);
+ }
+ data = Settings.getFile(Settings.KEYS.CPE_DATA_DIRECTORY);
+ if (data.exists()) {
+ FileUtils.delete(data);
+ }
+ data = DataStoreMetaInfo.getPropertiesFile();
+ if (data.exists()) {
+ FileUtils.delete(data);
+ }
+ }
+
+ /**
+ * Performs the batch update based on the configured batch update URL.
+ *
+ * @throws UpdateException thrown if there is an exception during the update
+ * process
+ */
+ private void performBatchUpdate() throws UpdateException {
+ if (properties.isBatchUpdateMode() && doBatchUpdate) {
+ final String batchSrc = Settings.getString(Settings.KEYS.BATCH_UPDATE_URL);
+ File tmp = null;
+ try {
+ deleteExistingData();
+ final File dataDirectory = CveDB.getDataDirectory().getParentFile();
+ final URL batchUrl = new URL(batchSrc);
+ if ("file".equals(batchUrl.getProtocol())) {
+ try {
+ tmp = new File(batchUrl.toURI());
+ } catch (URISyntaxException ex) {
+ final String msg = String.format("Invalid batch update URI: %s", batchSrc);
+ throw new UpdateException(msg, ex);
+ }
+ } else if ("http".equals(batchUrl.getProtocol())
+ || "https".equals(batchUrl.getProtocol())) {
+ tmp = File.createTempFile("batch_", ".zip");
+ Downloader.fetchFile(batchUrl, tmp);
+ }
+ //TODO add FTP?
+ FileUtils.extractFiles(tmp, dataDirectory);
+
+ } catch (IOException ex) {
+ final String msg = String.format("IO Exception Occured performing batch update using: %s", batchSrc);
+ throw new UpdateException(msg, ex);
+ } finally {
+ if (tmp != null && !tmp.delete()) {
+ tmp.deleteOnExit();
+ }
+ }
+ }
+ }
+
+ /**
+ * Closes the CVE and CPE data stores.
+ */
+ private void closeDataStores() {
+ if (cveDB != null) {
+ try {
+ cveDB.close();
+ } catch (Exception ignore) {
+ Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.FINEST, "Error closing the cveDB", ignore);
+ }
+ }
+ if (cpeIndex != null) {
+ try {
+ cpeIndex.close();
+ } catch (Exception ignore) {
+ Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.FINEST, "Error closing the cpeIndex", ignore);
+ }
+ }
+ }
+
+ /**
+ * Opens the CVE and CPE data stores.
+ *
+ * @throws UpdateException thrown if a data store cannot be opened
+ */
+ private void openDataStores() throws UpdateException {
+ //open the cve and cpe data stores
+ try {
+ cveDB = new CveDB();
+ cveDB.open();
+ cpeIndex = new CpeIndexWriter();
+ cpeIndex.open();
+ } catch (IOException ex) {
+ closeDataStores();
+ Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.FINE, "IO Error opening databases", ex);
+ throw new UpdateException("Error updating the CPE/CVE data, please see the log file for more details.");
+ } catch (SQLException ex) {
+ closeDataStores();
+ Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.FINE, "SQL Exception opening databases", ex);
+ throw new UpdateException("Error updating the CPE/CVE data, please see the log file for more details.");
+ } catch (DatabaseException ex) {
+ closeDataStores();
+ Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.FINE, "Database Exception opening databases", ex);
+ throw new UpdateException("Error updating the CPE/CVE data, please see the log file for more details.");
+ } catch (ClassNotFoundException ex) {
+ closeDataStores();
+ Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.FINE, "Class not found exception opening databases", ex);
+ throw new UpdateException("Error updating the CPE/CVE data, please see the log file for more details.");
+ }
+ }
+
+ /**
+ * Determines if the index needs to be updated. This is done by fetching the
+ * NVD CVE meta data and checking the last update date. If the data needs to
+ * be refreshed this method will return the NvdCveUrl for the files that
+ * need to be updated.
+ *
+ * @return the NvdCveUrl of the files that need to be updated.
+ * @throws MalformedURLException is thrown if the URL for the NVD CVE Meta
+ * data is incorrect.
+ * @throws DownloadFailedException is thrown if there is an error.
+ * downloading the NVD CVE download data file.
+ * @throws UpdateException Is thrown if there is an issue with the last
+ * updated properties file.
+ */
+ private Map updateNeeded() throws MalformedURLException, DownloadFailedException, UpdateException {
+
+ Map currentlyPublished;
+ try {
+ currentlyPublished = retrieveCurrentTimestampsFromWeb();
+ } catch (InvalidDataException ex) {
+ final String msg = "Unable to retrieve valid timestamp from nvd cve downloads page";
+ Logger.getLogger(DataStoreMetaInfo.class.getName()).log(Level.FINE, msg, ex);
+ throw new DownloadFailedException(msg, ex);
+ } catch (InvalidSettingException ex) {
+ Logger.getLogger(DataStoreMetaInfo.class.getName()).log(Level.FINE, "Invalid setting found when retrieving timestamps", ex);
+ throw new DownloadFailedException("Invalid settings", ex);
+ }
+
+ if (currentlyPublished == null) {
+ throw new DownloadFailedException("Unable to retrieve the timestamps of the currently published NVD CVE data");
+ }
+
+// final File cpeDataDirectory;
+// try {
+// cpeDataDirectory = CveDB.getDataDirectory();
+// } catch (IOException ex) {
+// String msg;
+// try {
+// msg = String.format("Unable to create the CVE Data Directory '%s'",
+// Settings.getFile(Settings.KEYS.CVE_DATA_DIRECTORY).getCanonicalPath());
+// } catch (IOException ex1) {
+// msg = String.format("Unable to create the CVE Data Directory, this is likely a configuration issue: '%s%s%s'",
+// Settings.getString(Settings.KEYS.DATA_DIRECTORY, ""),
+// File.separator,
+// Settings.getString(Settings.KEYS.CVE_DATA_DIRECTORY, ""));
+// }
+// throw new UpdateException(msg, ex);
+// }
+
+ if (!properties.isEmpty()) {
+ try {
+ boolean deleteAndRecreate = false;
+ float version;
+
+ if (properties.getProperty("version") == null) {
+ deleteAndRecreate = true;
+ } else {
+ try {
+ version = Float.parseFloat(properties.getProperty("version"));
+ final float currentVersion = Float.parseFloat(CveDB.DB_SCHEMA_VERSION);
+ if (currentVersion > version) {
+ deleteAndRecreate = true;
+ }
+ } catch (NumberFormatException ex) {
+ deleteAndRecreate = true;
+ }
+ }
+
+ final NvdCveInfo batchInfo = currentlyPublished.get(BATCH);
+ if (properties.isBatchUpdateMode() && batchInfo != null) {
+ final long lastUpdated = Long.parseLong(properties.getProperty(DataStoreMetaInfo.BATCH, "0"));
+ if (lastUpdated != batchInfo.getTimestamp()) {
+ deleteAndRecreate = true;
+ }
+ }
+
+ if (deleteAndRecreate) {
+ setDoBatchUpdate(properties.isBatchUpdateMode());
+ try {
+ deleteExistingData();
+ } catch (IOException ex) {
+ final String msg = "Unable to delete existing data";
+ Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.WARNING, msg);
+ Logger.getLogger(DatabaseUpdater.class.getName()).log(Level.FINE, null, ex);
+ }
+ return currentlyPublished;
+ }
+
+ final long lastUpdated = Long.parseLong(properties.getProperty(DataStoreMetaInfo.LAST_UPDATED, "0"));
+ final Date now = new Date();
+ final int days = Settings.getInt(Settings.KEYS.CVE_MODIFIED_VALID_FOR_DAYS, 7);
+ final int start = Settings.getInt(Settings.KEYS.CVE_START_YEAR, 2002);
+ final int end = Calendar.getInstance().get(Calendar.YEAR);
+ if (lastUpdated == currentlyPublished.get(MODIFIED).getTimestamp()) {
+ currentlyPublished.clear(); //we don't need to update anything.
+ setDoBatchUpdate(properties.isBatchUpdateMode());
+ } else if (withinRange(lastUpdated, now.getTime(), days)) {
+ currentlyPublished.get(MODIFIED).setNeedsUpdate(true);
+ if (properties.isBatchUpdateMode()) {
+ setDoBatchUpdate(false);
+ } else {
+ for (int i = start; i <= end; i++) {
+ currentlyPublished.get(String.valueOf(i)).setNeedsUpdate(false);
+ }
+ }
+ } else if (properties.isBatchUpdateMode()) {
+ currentlyPublished.get(MODIFIED).setNeedsUpdate(true);
+ setDoBatchUpdate(true);
+ } else { //we figure out which of the several XML files need to be downloaded.
+ currentlyPublished.get(MODIFIED).setNeedsUpdate(false);
+ for (int i = start; i <= end; i++) {
+ final NvdCveInfo cve = currentlyPublished.get(String.valueOf(i));
+ long currentTimestamp = 0;
+ try {
+ currentTimestamp = Long.parseLong(properties.getProperty(DataStoreMetaInfo.LAST_UPDATED_BASE + String.valueOf(i), "0"));
+ } catch (NumberFormatException ex) {
+ final String msg = String.format("Error parsing '%s' '%s' from nvdcve.lastupdated",
+ DataStoreMetaInfo.LAST_UPDATED_BASE, String.valueOf(i));
+ Logger.getLogger(DataStoreMetaInfo.class.getName()).log(Level.FINE, msg, ex);
+ }
+ if (currentTimestamp == cve.getTimestamp()) {
+ cve.setNeedsUpdate(false); //they default to true.
+ }
+ }
+ }
+ } catch (NumberFormatException ex) {
+ final String msg = "An invalid schema version or timestamp exists in the data.properties file.";
+ Logger.getLogger(DataStoreMetaInfo.class.getName()).log(Level.WARNING, msg);
+ Logger.getLogger(DataStoreMetaInfo.class.getName()).log(Level.FINE, null, ex);
+ setDoBatchUpdate(properties.isBatchUpdateMode());
+ }
+ } else {
+ setDoBatchUpdate(properties.isBatchUpdateMode());
+ }
+ return currentlyPublished;
+ }
+
+ /**
+ * Determines if the epoch date is within the range specified of the
+ * compareTo epoch time. This takes the (compareTo-date)/1000/60/60/24 to
+ * get the number of days. If the calculated days is less then the range the
+ * date is considered valid.
+ *
+ * @param date the date to be checked.
+ * @param compareTo the date to compare to.
+ * @param range the range in days to be considered valid.
+ * @return whether or not the date is within the range.
+ */
+ private boolean withinRange(long date, long compareTo, int range) {
+ final double differenceInDays = (compareTo - date) / 1000.0 / 60.0 / 60.0 / 24.0;
+ return differenceInDays < range;
+ }
+
+ /**
+ * Retrieves the timestamps from the NVD CVE meta data file.
+ *
+ * @return the timestamp from the currently published nvdcve downloads page
+ * @throws MalformedURLException thrown if the URL for the NVD CCE Meta data
+ * is incorrect.
+ * @throws DownloadFailedException thrown if there is an error downloading
+ * the nvd cve meta data file
+ * @throws InvalidDataException thrown if there is an exception parsing the
+ * timestamps
+ * @throws InvalidSettingException thrown if the settings are invalid
+ */
+ private Map retrieveCurrentTimestampsFromWeb()
+ throws MalformedURLException, DownloadFailedException, InvalidDataException, InvalidSettingException {
+
+ final Map map = new TreeMap();
+ String retrieveUrl = Settings.getString(Settings.KEYS.CVE_MODIFIED_20_URL);
+ if (retrieveUrl == null && properties.isBatchUpdateMode()) {
+ final NvdCveInfo item = new NvdCveInfo();
+ retrieveUrl = Settings.getString(Settings.KEYS.BATCH_UPDATE_URL);
+ if (retrieveUrl == null) {
+ final String msg = "Invalid configuration - neither the modified or batch update URLs are specified in the configuration.";
+ Logger.getLogger(DataStoreMetaInfo.class.getName()).log(Level.SEVERE, msg);
+ throw new InvalidSettingException(msg);
+ }
+ item.setTimestamp(Downloader.getLastModified(new URL(retrieveUrl)));
+ item.setId(BATCH);
+ item.setNeedsUpdate(false);
+ map.put(BATCH, item);
+ } else {
+ NvdCveInfo item = new NvdCveInfo();
+ item.setNeedsUpdate(false); //the others default to true, to make life easier later this should default to false.
+ item.setId(MODIFIED);
+ item.setUrl(retrieveUrl);
+ item.setOldSchemaVersionUrl(Settings.getString(Settings.KEYS.CVE_MODIFIED_12_URL));
+
+ item.setTimestamp(Downloader.getLastModified(new URL(retrieveUrl)));
+ map.put(MODIFIED, item);
+
+ //only add these urls if we are not in batch mode
+ if (!properties.isBatchUpdateMode()) {
+ final int start = Settings.getInt(Settings.KEYS.CVE_START_YEAR);
+ final int end = Calendar.getInstance().get(Calendar.YEAR);
+ final String baseUrl20 = Settings.getString(Settings.KEYS.CVE_SCHEMA_2_0);
+ final String baseUrl12 = Settings.getString(Settings.KEYS.CVE_SCHEMA_1_2);
+ for (int i = start; i <= end; i++) {
+ retrieveUrl = String.format(baseUrl20, i);
+ item = new NvdCveInfo();
+ item.setId(Integer.toString(i));
+ item.setUrl(retrieveUrl);
+ item.setOldSchemaVersionUrl(String.format(baseUrl12, i));
+ item.setTimestamp(Downloader.getLastModified(new URL(retrieveUrl)));
+ map.put(item.getId(), item);
+ }
+ }
+ }
+ return map;
+ }
+}
diff --git a/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/NvdCveInfo.java b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/NvdCveInfo.java
new file mode 100644
index 000000000..7a27a9124
--- /dev/null
+++ b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/NvdCveInfo.java
@@ -0,0 +1,138 @@
+/*
+ * This file is part of dependency-check-core.
+ *
+ * Dependency-check-core is free software: you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation, either version 3 of the License, or (at your option) any
+ * later version.
+ *
+ * Dependency-check-core is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+ * details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * dependency-check-core. If not, see http://www.gnu.org/licenses/.
+ *
+ * Copyright (c) 2013 Jeremy Long. All Rights Reserved.
+ */
+package org.owasp.dependencycheck.data.update;
+
+/**
+ * A pojo that contains the Url and timestamp of the current NvdCve XML files.
+ *
+ * @author Jeremy Long (jeremy.long@owasp.org)
+ */
+public class NvdCveInfo {
+
+ /**
+ * an id.
+ */
+ private String id;
+
+ /**
+ * Get the value of id.
+ *
+ * @return the value of id
+ */
+ public String getId() {
+ return id;
+ }
+
+ /**
+ * Set the value of id.
+ *
+ * @param id new value of id
+ */
+ public void setId(String id) {
+ this.id = id;
+ }
+ /**
+ * a url.
+ */
+ private String url;
+
+ /**
+ * Get the value of url.
+ *
+ * @return the value of url
+ */
+ public String getUrl() {
+ return url;
+ }
+
+ /**
+ * Set the value of url.
+ *
+ * @param url new value of url
+ */
+ public void setUrl(String url) {
+ this.url = url;
+ }
+ /**
+ * The 1.2 schema URL.
+ */
+ private String oldSchemaVersionUrl;
+
+ /**
+ * Get the value of oldSchemaVersionUrl.
+ *
+ * @return the value of oldSchemaVersionUrl
+ */
+ public String getOldSchemaVersionUrl() {
+ return oldSchemaVersionUrl;
+ }
+
+ /**
+ * Set the value of oldSchemaVersionUrl.
+ *
+ * @param oldSchemaVersionUrl new value of oldSchemaVersionUrl
+ */
+ public void setOldSchemaVersionUrl(String oldSchemaVersionUrl) {
+ this.oldSchemaVersionUrl = oldSchemaVersionUrl;
+ }
+ /**
+ * a timestamp - epoch time.
+ */
+ private long timestamp;
+
+ /**
+ * Get the value of timestamp - epoch time.
+ *
+ * @return the value of timestamp - epoch time
+ */
+ public long getTimestamp() {
+ return timestamp;
+ }
+
+ /**
+ * Set the value of timestamp - epoch time.
+ *
+ * @param timestamp new value of timestamp - epoch time
+ */
+ public void setTimestamp(long timestamp) {
+ this.timestamp = timestamp;
+ }
+ /**
+ * indicates whether or not this item should be updated.
+ */
+ private boolean needsUpdate = true;
+
+ /**
+ * Get the value of needsUpdate.
+ *
+ * @return the value of needsUpdate
+ */
+ public boolean getNeedsUpdate() {
+ return needsUpdate;
+ }
+
+ /**
+ * Set the value of needsUpdate.
+ *
+ * @param needsUpdate new value of needsUpdate
+ */
+ public void setNeedsUpdate(boolean needsUpdate) {
+ this.needsUpdate = needsUpdate;
+ }
+}
diff --git a/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/nvdcve/xml/package-info.java b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/package-info.java
similarity index 91%
rename from dependency-check-core/src/main/java/org/owasp/dependencycheck/data/nvdcve/xml/package-info.java
rename to dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/package-info.java
index c04c24a6a..a6251592d 100644
--- a/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/nvdcve/xml/package-info.java
+++ b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/update/package-info.java
@@ -15,4 +15,4 @@
*