From cb3cf79beb09be3038b190575bd4f0437acf312f Mon Sep 17 00:00:00 2001 From: Mark Rekveld Date: Fri, 7 Jul 2017 17:11:49 +0200 Subject: [PATCH] Engine execution modes --- .../org/owasp/dependencycheck/Engine.java | 134 ++++++++++++------ .../analyzer/AnalyzerService.java | 29 +++- .../data/nvdcve/DatabaseException.java | 2 +- 3 files changed, 114 insertions(+), 51 deletions(-) diff --git a/dependency-check-core/src/main/java/org/owasp/dependencycheck/Engine.java b/dependency-check-core/src/main/java/org/owasp/dependencycheck/Engine.java index 1d6cb2ab6..b81da05e8 100644 --- a/dependency-check-core/src/main/java/org/owasp/dependencycheck/Engine.java +++ b/dependency-check-core/src/main/java/org/owasp/dependencycheck/Engine.java @@ -24,6 +24,7 @@ import org.owasp.dependencycheck.analyzer.FileTypeAnalyzer; import org.owasp.dependencycheck.data.nvdcve.ConnectionFactory; import org.owasp.dependencycheck.data.nvdcve.CveDB; import org.owasp.dependencycheck.data.nvdcve.DatabaseException; +import org.owasp.dependencycheck.data.nvdcve.DatabaseProperties; import org.owasp.dependencycheck.data.update.CachedWebDataSource; import org.owasp.dependencycheck.data.update.UpdateService; import org.owasp.dependencycheck.data.update.exception.UpdateException; @@ -31,6 +32,8 @@ import org.owasp.dependencycheck.dependency.Dependency; import org.owasp.dependencycheck.exception.ExceptionCollection; import org.owasp.dependencycheck.exception.InitializationException; import org.owasp.dependencycheck.exception.NoDataException; +import org.owasp.dependencycheck.exception.ReportException; +import org.owasp.dependencycheck.reporting.ReportGenerator; import org.owasp.dependencycheck.utils.InvalidSettingException; import org.owasp.dependencycheck.utils.Settings; import org.slf4j.Logger; @@ -39,24 +42,10 @@ import org.slf4j.LoggerFactory; import java.io.File; import java.io.FileFilter; import java.io.IOException; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.EnumMap; -import java.util.HashSet; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.CancellationException; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.Future; -import java.util.concurrent.TimeUnit; -import org.owasp.dependencycheck.data.nvdcve.DatabaseProperties; -import org.owasp.dependencycheck.exception.ReportException; -import org.owasp.dependencycheck.reporting.ReportGenerator; +import java.util.*; +import java.util.concurrent.*; + +import static org.owasp.dependencycheck.analyzer.AnalysisPhase.*; /** * Scans files, directories, etc. for Dependencies. Analyzers are loaded and @@ -68,6 +57,35 @@ import org.owasp.dependencycheck.reporting.ReportGenerator; */ public class Engine implements FileFilter { + public enum Mode { + EVIDENCE_COLLECTION( + false, + INITIAL, + PRE_INFORMATION_COLLECTION, + INFORMATION_COLLECTION, + POST_INFORMATION_COLLECTION + ), + EVIDENCE_PROCESSING( + true, + PRE_IDENTIFIER_ANALYSIS, + IDENTIFIER_ANALYSIS, + POST_IDENTIFIER_ANALYSIS, + PRE_FINDING_ANALYSIS, + FINDING_ANALYSIS, + POST_FINDING_ANALYSIS, + FINAL + ), + STANDALONE(true, AnalysisPhase.values()); + + public final boolean requiresDatabase; + public final AnalysisPhase[] phases; + + Mode(boolean requiresDatabase, AnalysisPhase... phases) { + this.requiresDatabase = requiresDatabase; + this.phases = phases; + } + } + /** * The list of dependencies. */ @@ -82,11 +100,13 @@ public class Engine implements FileFilter { */ private final Set fileTypeAnalyzers = new HashSet<>(); + private final Mode mode; + /** * The ClassLoader to use when dynamically loading Analyzer and Update * services. */ - private ClassLoader serviceClassLoader = Thread.currentThread().getContextClassLoader(); + private ClassLoader serviceClassLoader; /** * A reference to the database. */ @@ -98,23 +118,32 @@ public class Engine implements FileFilter { /** * Creates a new Engine. - * - * @throws DatabaseException thrown if there is an error connecting to the - * database */ - public Engine() throws DatabaseException { - initializeEngine(); + public Engine() { + this(Mode.STANDALONE); + } + + public Engine(Mode mode) { + this(Thread.currentThread().getContextClassLoader(), mode); } /** * Creates a new Engine. * * @param serviceClassLoader a reference the class loader being used - * @throws DatabaseException thrown if there is an error connecting to the - * database */ - public Engine(ClassLoader serviceClassLoader) throws DatabaseException { + public Engine(ClassLoader serviceClassLoader) { + this(serviceClassLoader, Mode.STANDALONE); + } + + /** + * Creates a new Engine. + * + * @param serviceClassLoader a reference the class loader being used + */ + public Engine(ClassLoader serviceClassLoader, Mode mode) { this.serviceClassLoader = serviceClassLoader; + this.mode = mode; initializeEngine(); } @@ -125,8 +154,10 @@ public class Engine implements FileFilter { * @throws DatabaseException thrown if there is an error connecting to the * database */ - protected final void initializeEngine() throws DatabaseException { - ConnectionFactory.initialize(); + protected final void initializeEngine() { + if (mode.requiresDatabase) { + ConnectionFactory.initialize(); + } loadAnalyzers(); } @@ -134,11 +165,13 @@ public class Engine implements FileFilter { * Properly cleans up resources allocated during analysis. */ public void cleanup() { - if (database != null) { - database.close(); - database = null; + if (mode.requiresDatabase) { + if (database != null) { + database.close(); + database = null; + } + ConnectionFactory.cleanup(); } - ConnectionFactory.cleanup(); } /** @@ -149,12 +182,12 @@ public class Engine implements FileFilter { if (!analyzers.isEmpty()) { return; } - for (AnalysisPhase phase : AnalysisPhase.values()) { + for (AnalysisPhase phase : mode.phases) { analyzers.put(phase, new ArrayList()); } final AnalyzerService service = new AnalyzerService(serviceClassLoader); - final List iterator = service.getAnalyzers(); + final List iterator = service.getAnalyzers(mode.phases); for (Analyzer a : iterator) { analyzers.get(a.getAnalysisPhase()).add(a); if (a instanceof FileTypeAnalyzer) { @@ -503,7 +536,7 @@ public class Engine implements FileFilter { final long analysisStart = System.currentTimeMillis(); // analysis phases - for (AnalysisPhase phase : AnalysisPhase.values()) { + for (AnalysisPhase phase : mode.phases) { final List analyzerList = analyzers.get(phase); for (final Analyzer analyzer : analyzerList) { @@ -549,6 +582,9 @@ public class Engine implements FileFilter { * @throws ExceptionCollection thrown if fatal exceptions occur */ private void initializeAndUpdateDatabase(final List exceptions) throws ExceptionCollection { + if (!mode.requiresDatabase) { + return; + } boolean autoUpdate = true; try { autoUpdate = Settings.getBoolean(Settings.KEYS.AUTO_UPDATE); @@ -705,15 +741,19 @@ public class Engine implements FileFilter { * @throws UpdateException thrown if the operation fails */ public void doUpdates() throws UpdateException { - LOGGER.info("Checking for updates"); - final long updateStart = System.currentTimeMillis(); - final UpdateService service = new UpdateService(serviceClassLoader); - final Iterator iterator = service.getDataSources(); - while (iterator.hasNext()) { - final CachedWebDataSource source = iterator.next(); - source.update(); + if (mode.requiresDatabase) { + LOGGER.info("Checking for updates"); + final long updateStart = System.currentTimeMillis(); + final UpdateService service = new UpdateService(serviceClassLoader); + final Iterator iterator = service.getDataSources(); + while (iterator.hasNext()) { + final CachedWebDataSource source = iterator.next(); + source.update(); + } + LOGGER.info("Check for updates complete ({} ms)", System.currentTimeMillis() - updateStart); + } else { + LOGGER.info("Skipping update check in evidence collection mode."); } - LOGGER.info("Check for updates complete ({} ms)", System.currentTimeMillis() - updateStart); } /** @@ -778,7 +818,7 @@ public class Engine implements FileFilter { * @throws NoDataException thrown if no data exists in the CPE Index */ private void ensureDataExists() throws NoDataException { - if (database == null || !database.dataExists()) { + if (mode.requiresDatabase && (database == null || !database.dataExists())) { throw new NoDataException("No documents exist"); } } @@ -813,7 +853,9 @@ public class Engine implements FileFilter { */ public synchronized void writeReports(String applicationName, String groupId, String artifactId, String version, File outputDir, String format) throws ReportException { - + if (mode == Mode.EVIDENCE_COLLECTION) { + throw new UnsupportedOperationException("Cannot generate report in evidence collection mode."); + } final DatabaseProperties prop = database.getDatabaseProperties(); final ReportGenerator r = new ReportGenerator(applicationName, groupId, artifactId, version, dependencies, getAnalyzers(), prop); try { diff --git a/dependency-check-core/src/main/java/org/owasp/dependencycheck/analyzer/AnalyzerService.java b/dependency-check-core/src/main/java/org/owasp/dependencycheck/analyzer/AnalyzerService.java index 7c2c18d2f..e4274217a 100644 --- a/dependency-check-core/src/main/java/org/owasp/dependencycheck/analyzer/AnalyzerService.java +++ b/dependency-check-core/src/main/java/org/owasp/dependencycheck/analyzer/AnalyzerService.java @@ -17,14 +17,14 @@ */ package org.owasp.dependencycheck.analyzer; -import java.util.ArrayList; -import java.util.Iterator; -import java.util.List; -import java.util.ServiceLoader; import org.owasp.dependencycheck.utils.InvalidSettingException; import org.owasp.dependencycheck.utils.Settings; import org.slf4j.LoggerFactory; +import java.util.*; + +import static java.util.Arrays.asList; + /** * The Analyzer Service Loader. This class loads all services that implement * org.owasp.dependencycheck.analyzer.Analyzer. @@ -57,6 +57,24 @@ public class AnalyzerService { * @return a list of Analyzers. */ public List getAnalyzers() { + return getAnalyzers(AnalysisPhase.values()); + } + + /** + * Returns a list of all instances of the Analyzer interface that are bound to one of the given phases. + * + * @return a list of Analyzers. + */ + public List getAnalyzers(AnalysisPhase... phases) { + return getAnalyzers(asList(phases)); + } + + /** + * Returns a list of all instances of the Analyzer interface that are bound to one of the given phases. + * + * @return a list of Analyzers. + */ + private List getAnalyzers(List phases) { final List analyzers = new ArrayList<>(); final Iterator iterator = service.iterator(); boolean experimentalEnabled = false; @@ -67,6 +85,9 @@ public class AnalyzerService { } while (iterator.hasNext()) { final Analyzer a = iterator.next(); + if (!phases.contains(a.getAnalysisPhase())) { + continue; + } if (!experimentalEnabled && a.getClass().isAnnotationPresent(Experimental.class)) { continue; } diff --git a/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/nvdcve/DatabaseException.java b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/nvdcve/DatabaseException.java index 40ac796f4..474664302 100644 --- a/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/nvdcve/DatabaseException.java +++ b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/nvdcve/DatabaseException.java @@ -22,7 +22,7 @@ package org.owasp.dependencycheck.data.nvdcve; * * @author Jeremy Long */ -public class DatabaseException extends Exception { +public class DatabaseException extends RuntimeException { /** * the serial version uid.