diff --git a/dependency-check-core/src/main/java/org/owasp/dependencycheck/Engine.java b/dependency-check-core/src/main/java/org/owasp/dependencycheck/Engine.java index 1d6cb2ab6..b4469dfc2 100644 --- a/dependency-check-core/src/main/java/org/owasp/dependencycheck/Engine.java +++ b/dependency-check-core/src/main/java/org/owasp/dependencycheck/Engine.java @@ -24,6 +24,7 @@ import org.owasp.dependencycheck.analyzer.FileTypeAnalyzer; import org.owasp.dependencycheck.data.nvdcve.ConnectionFactory; import org.owasp.dependencycheck.data.nvdcve.CveDB; import org.owasp.dependencycheck.data.nvdcve.DatabaseException; +import org.owasp.dependencycheck.data.nvdcve.DatabaseProperties; import org.owasp.dependencycheck.data.update.CachedWebDataSource; import org.owasp.dependencycheck.data.update.UpdateService; import org.owasp.dependencycheck.data.update.exception.UpdateException; @@ -31,6 +32,8 @@ import org.owasp.dependencycheck.dependency.Dependency; import org.owasp.dependencycheck.exception.ExceptionCollection; import org.owasp.dependencycheck.exception.InitializationException; import org.owasp.dependencycheck.exception.NoDataException; +import org.owasp.dependencycheck.exception.ReportException; +import org.owasp.dependencycheck.reporting.ReportGenerator; import org.owasp.dependencycheck.utils.InvalidSettingException; import org.owasp.dependencycheck.utils.Settings; import org.slf4j.Logger; @@ -39,24 +42,10 @@ import org.slf4j.LoggerFactory; import java.io.File; import java.io.FileFilter; import java.io.IOException; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.EnumMap; -import java.util.HashSet; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.CancellationException; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.Future; -import java.util.concurrent.TimeUnit; -import org.owasp.dependencycheck.data.nvdcve.DatabaseProperties; -import org.owasp.dependencycheck.exception.ReportException; -import org.owasp.dependencycheck.reporting.ReportGenerator; +import java.util.*; +import java.util.concurrent.*; + +import static org.owasp.dependencycheck.analyzer.AnalysisPhase.*; /** * Scans files, directories, etc. for Dependencies. Analyzers are loaded and @@ -66,7 +55,51 @@ import org.owasp.dependencycheck.reporting.ReportGenerator; * * @author Jeremy Long */ -public class Engine implements FileFilter { +public class Engine implements FileFilter, AutoCloseable { + + /** + * {@link Engine} execution modes. + */ + public enum Mode { + /** + * In evidence collection mode the {@link Engine} only collects evidence from the scan targets, + * and doesn't require a database. + */ + EVIDENCE_COLLECTION( + false, + INITIAL, + PRE_INFORMATION_COLLECTION, + INFORMATION_COLLECTION, + POST_INFORMATION_COLLECTION + ), + /** + * In evidence processing mode the {@link Engine} processes the evidence collected using the + * {@link #EVIDENCE_COLLECTION} mode. Dependencies should be injected into the {@link Engine} + * using {@link Engine#setDependencies(List)}. + */ + EVIDENCE_PROCESSING( + true, + PRE_IDENTIFIER_ANALYSIS, + IDENTIFIER_ANALYSIS, + POST_IDENTIFIER_ANALYSIS, + PRE_FINDING_ANALYSIS, + FINDING_ANALYSIS, + POST_FINDING_ANALYSIS, + FINAL + ), + /** + * In standalone mode the {@link Engine} will collect and process evidence in a single execution. + */ + STANDALONE(true, AnalysisPhase.values()); + + public final boolean requiresDatabase; + public final AnalysisPhase[] phases; + + Mode(boolean requiresDatabase, AnalysisPhase... phases) { + this.requiresDatabase = requiresDatabase; + this.phases = phases; + } + } /** * The list of dependencies. @@ -82,11 +115,16 @@ public class Engine implements FileFilter { */ private final Set fileTypeAnalyzers = new HashSet<>(); + /** + * The engine execution mode indicating it will either collect evidence or process evidence or both. + */ + private final Mode mode; + /** * The ClassLoader to use when dynamically loading Analyzer and Update * services. */ - private ClassLoader serviceClassLoader = Thread.currentThread().getContextClassLoader(); + private ClassLoader serviceClassLoader; /** * A reference to the database. */ @@ -97,24 +135,37 @@ public class Engine implements FileFilter { private static final Logger LOGGER = LoggerFactory.getLogger(Engine.class); /** - * Creates a new Engine. - * - * @throws DatabaseException thrown if there is an error connecting to the - * database + * Creates a new {@link Mode#STANDALONE} Engine. */ - public Engine() throws DatabaseException { - initializeEngine(); + public Engine() { + this(Mode.STANDALONE); + } + + /** + * Creates a new Engine. + */ + public Engine(Mode mode) { + this(Thread.currentThread().getContextClassLoader(), mode); + } + + /** + * Creates a new {@link Mode#STANDALONE} Engine. + * + * @param serviceClassLoader a reference the class loader being used + */ + public Engine(ClassLoader serviceClassLoader) { + this(serviceClassLoader, Mode.STANDALONE); } /** * Creates a new Engine. * * @param serviceClassLoader a reference the class loader being used - * @throws DatabaseException thrown if there is an error connecting to the - * database + * @param mode the mode of the engine */ - public Engine(ClassLoader serviceClassLoader) throws DatabaseException { + public Engine(ClassLoader serviceClassLoader, Mode mode) { this.serviceClassLoader = serviceClassLoader; + this.mode = mode; initializeEngine(); } @@ -125,8 +176,10 @@ public class Engine implements FileFilter { * @throws DatabaseException thrown if there is an error connecting to the * database */ - protected final void initializeEngine() throws DatabaseException { - ConnectionFactory.initialize(); + protected final void initializeEngine() { + if (mode.requiresDatabase) { + ConnectionFactory.initialize(); + } loadAnalyzers(); } @@ -134,11 +187,18 @@ public class Engine implements FileFilter { * Properly cleans up resources allocated during analysis. */ public void cleanup() { - if (database != null) { - database.close(); - database = null; + if (mode.requiresDatabase) { + if (database != null) { + database.close(); + database = null; + } + ConnectionFactory.cleanup(); } - ConnectionFactory.cleanup(); + } + + @Override + public void close() { + cleanup(); } /** @@ -149,12 +209,12 @@ public class Engine implements FileFilter { if (!analyzers.isEmpty()) { return; } - for (AnalysisPhase phase : AnalysisPhase.values()) { + for (AnalysisPhase phase : mode.phases) { analyzers.put(phase, new ArrayList()); } final AnalyzerService service = new AnalyzerService(serviceClassLoader); - final List iterator = service.getAnalyzers(); + final List iterator = service.getAnalyzers(mode.phases); for (Analyzer a : iterator) { analyzers.get(a.getAnalysisPhase()).add(a); if (a instanceof FileTypeAnalyzer) { @@ -503,7 +563,7 @@ public class Engine implements FileFilter { final long analysisStart = System.currentTimeMillis(); // analysis phases - for (AnalysisPhase phase : AnalysisPhase.values()) { + for (AnalysisPhase phase : mode.phases) { final List analyzerList = analyzers.get(phase); for (final Analyzer analyzer : analyzerList) { @@ -526,7 +586,7 @@ public class Engine implements FileFilter { } } } - for (AnalysisPhase phase : AnalysisPhase.values()) { + for (AnalysisPhase phase : mode.phases) { final List analyzerList = analyzers.get(phase); for (Analyzer a : analyzerList) { @@ -549,6 +609,9 @@ public class Engine implements FileFilter { * @throws ExceptionCollection thrown if fatal exceptions occur */ private void initializeAndUpdateDatabase(final List exceptions) throws ExceptionCollection { + if (!mode.requiresDatabase) { + return; + } boolean autoUpdate = true; try { autoUpdate = Settings.getBoolean(Settings.KEYS.AUTO_UPDATE); @@ -705,15 +768,19 @@ public class Engine implements FileFilter { * @throws UpdateException thrown if the operation fails */ public void doUpdates() throws UpdateException { - LOGGER.info("Checking for updates"); - final long updateStart = System.currentTimeMillis(); - final UpdateService service = new UpdateService(serviceClassLoader); - final Iterator iterator = service.getDataSources(); - while (iterator.hasNext()) { - final CachedWebDataSource source = iterator.next(); - source.update(); + if (mode.requiresDatabase) { + LOGGER.info("Checking for updates"); + final long updateStart = System.currentTimeMillis(); + final UpdateService service = new UpdateService(serviceClassLoader); + final Iterator iterator = service.getDataSources(); + while (iterator.hasNext()) { + final CachedWebDataSource source = iterator.next(); + source.update(); + } + LOGGER.info("Check for updates complete ({} ms)", System.currentTimeMillis() - updateStart); + } else { + LOGGER.info("Skipping update check in evidence collection mode."); } - LOGGER.info("Check for updates complete ({} ms)", System.currentTimeMillis() - updateStart); } /** @@ -724,7 +791,7 @@ public class Engine implements FileFilter { */ public List getAnalyzers() { final List ret = new ArrayList<>(); - for (AnalysisPhase phase : AnalysisPhase.values()) { + for (AnalysisPhase phase : mode.phases) { final List analyzerList = analyzers.get(phase); ret.addAll(analyzerList); } @@ -778,7 +845,7 @@ public class Engine implements FileFilter { * @throws NoDataException thrown if no data exists in the CPE Index */ private void ensureDataExists() throws NoDataException { - if (database == null || !database.dataExists()) { + if (mode.requiresDatabase && (database == null || !database.dataExists())) { throw new NoDataException("No documents exist"); } } @@ -813,7 +880,9 @@ public class Engine implements FileFilter { */ public synchronized void writeReports(String applicationName, String groupId, String artifactId, String version, File outputDir, String format) throws ReportException { - + if (mode == Mode.EVIDENCE_COLLECTION) { + throw new UnsupportedOperationException("Cannot generate report in evidence collection mode."); + } final DatabaseProperties prop = database.getDatabaseProperties(); final ReportGenerator r = new ReportGenerator(applicationName, groupId, artifactId, version, dependencies, getAnalyzers(), prop); try { diff --git a/dependency-check-core/src/main/java/org/owasp/dependencycheck/analyzer/AnalyzerService.java b/dependency-check-core/src/main/java/org/owasp/dependencycheck/analyzer/AnalyzerService.java index 7c2c18d2f..e4274217a 100644 --- a/dependency-check-core/src/main/java/org/owasp/dependencycheck/analyzer/AnalyzerService.java +++ b/dependency-check-core/src/main/java/org/owasp/dependencycheck/analyzer/AnalyzerService.java @@ -17,14 +17,14 @@ */ package org.owasp.dependencycheck.analyzer; -import java.util.ArrayList; -import java.util.Iterator; -import java.util.List; -import java.util.ServiceLoader; import org.owasp.dependencycheck.utils.InvalidSettingException; import org.owasp.dependencycheck.utils.Settings; import org.slf4j.LoggerFactory; +import java.util.*; + +import static java.util.Arrays.asList; + /** * The Analyzer Service Loader. This class loads all services that implement * org.owasp.dependencycheck.analyzer.Analyzer. @@ -57,6 +57,24 @@ public class AnalyzerService { * @return a list of Analyzers. */ public List getAnalyzers() { + return getAnalyzers(AnalysisPhase.values()); + } + + /** + * Returns a list of all instances of the Analyzer interface that are bound to one of the given phases. + * + * @return a list of Analyzers. + */ + public List getAnalyzers(AnalysisPhase... phases) { + return getAnalyzers(asList(phases)); + } + + /** + * Returns a list of all instances of the Analyzer interface that are bound to one of the given phases. + * + * @return a list of Analyzers. + */ + private List getAnalyzers(List phases) { final List analyzers = new ArrayList<>(); final Iterator iterator = service.iterator(); boolean experimentalEnabled = false; @@ -67,6 +85,9 @@ public class AnalyzerService { } while (iterator.hasNext()) { final Analyzer a = iterator.next(); + if (!phases.contains(a.getAnalysisPhase())) { + continue; + } if (!experimentalEnabled && a.getClass().isAnnotationPresent(Experimental.class)) { continue; } diff --git a/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/nvdcve/DatabaseException.java b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/nvdcve/DatabaseException.java index 40ac796f4..474664302 100644 --- a/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/nvdcve/DatabaseException.java +++ b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/nvdcve/DatabaseException.java @@ -22,7 +22,7 @@ package org.owasp.dependencycheck.data.nvdcve; * * @author Jeremy Long */ -public class DatabaseException extends Exception { +public class DatabaseException extends RuntimeException { /** * the serial version uid. diff --git a/dependency-check-core/src/test/java/org/owasp/dependencycheck/EngineModeIT.java b/dependency-check-core/src/test/java/org/owasp/dependencycheck/EngineModeIT.java new file mode 100644 index 000000000..d0fc74227 --- /dev/null +++ b/dependency-check-core/src/test/java/org/owasp/dependencycheck/EngineModeIT.java @@ -0,0 +1,102 @@ +package org.owasp.dependencycheck; + +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.junit.rules.TestName; +import org.owasp.dependencycheck.analyzer.AnalysisPhase; +import org.owasp.dependencycheck.dependency.Dependency; +import org.owasp.dependencycheck.utils.Settings; + +import java.io.File; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; + +import static org.hamcrest.CoreMatchers.notNullValue; +import static org.hamcrest.CoreMatchers.nullValue; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.core.Is.is; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +/** + * @author Mark Rekveld + */ +public class EngineModeIT extends BaseTest { + + @Rule + public TemporaryFolder tempDir = new TemporaryFolder(); + @Rule + public TestName testName = new TestName(); + + @Before + public void setUp() throws Exception { + Settings.setString(Settings.KEYS.DATA_DIRECTORY, tempDir.newFolder().getAbsolutePath()); + } + + @Test + public void testEvidenceCollectionAndEvidenceProcessingModes() throws Exception { + List dependencies; + try (Engine engine = new Engine(Engine.Mode.EVIDENCE_COLLECTION)) { + assertDatabase(false); + for (AnalysisPhase phase : Engine.Mode.EVIDENCE_COLLECTION.phases) { + assertThat(engine.getAnalyzers(phase), is(notNullValue())); + } + for (AnalysisPhase phase : Engine.Mode.EVIDENCE_PROCESSING.phases) { + assertThat(engine.getAnalyzers(phase), is(nullValue())); + } + File file = BaseTest.getResourceAsFile(this, "struts2-core-2.1.2.jar"); + engine.scan(file); + engine.analyzeDependencies(); + dependencies = engine.getDependencies(); + assertThat(dependencies.size(), is(1)); + Dependency dependency = dependencies.get(0); + assertTrue(dependency.getVendorEvidence().toString().toLowerCase().contains("apache")); + assertTrue(dependency.getVendorEvidence().getWeighting().contains("apache")); + assertTrue(dependency.getVulnerabilities().isEmpty()); + } + + try (Engine engine = new Engine(Engine.Mode.EVIDENCE_PROCESSING)) { + assertDatabase(true); + for (AnalysisPhase phase : Engine.Mode.EVIDENCE_PROCESSING.phases) { + assertThat(engine.getAnalyzers(phase), is(notNullValue())); + } + for (AnalysisPhase phase : Engine.Mode.EVIDENCE_COLLECTION.phases) { + assertThat(engine.getAnalyzers(phase), is(nullValue())); + } + engine.setDependencies(dependencies); + engine.analyzeDependencies(); + Dependency dependency = dependencies.get(0); + assertFalse(dependency.getVulnerabilities().isEmpty()); + } + } + + @Test + public void testStandaloneMode() throws Exception { + try (Engine engine = new Engine(Engine.Mode.STANDALONE)) { + assertDatabase(true); + for (AnalysisPhase phase : Engine.Mode.STANDALONE.phases) { + assertThat(engine.getAnalyzers(phase), is(notNullValue())); + } + File file = BaseTest.getResourceAsFile(this, "struts2-core-2.1.2.jar"); + engine.scan(file); + engine.analyzeDependencies(); + List dependencies = engine.getDependencies(); + assertThat(dependencies.size(), is(1)); + Dependency dependency = dependencies.get(0); + assertTrue(dependency.getVendorEvidence().toString().toLowerCase().contains("apache")); + assertTrue(dependency.getVendorEvidence().getWeighting().contains("apache")); + assertFalse(dependency.getVulnerabilities().isEmpty()); + } + } + + private void assertDatabase(boolean exists) throws Exception { + Path directory = Settings.getDataDirectory().toPath(); + assertThat(Files.exists(directory), is(true)); + assertThat(Files.isDirectory(directory), is(true)); + Path database = directory.resolve(Settings.getString(Settings.KEYS.DB_FILE_NAME)); + assertThat(Files.exists(database), is(exists)); + } +} diff --git a/dependency-check-core/src/test/java/org/owasp/dependencycheck/analyzer/AnalyzerServiceTest.java b/dependency-check-core/src/test/java/org/owasp/dependencycheck/analyzer/AnalyzerServiceTest.java index b88bc4f3d..34f83be44 100644 --- a/dependency-check-core/src/test/java/org/owasp/dependencycheck/analyzer/AnalyzerServiceTest.java +++ b/dependency-check-core/src/test/java/org/owasp/dependencycheck/analyzer/AnalyzerServiceTest.java @@ -17,13 +17,18 @@ */ package org.owasp.dependencycheck.analyzer; -import java.util.List; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; import org.junit.Test; import org.owasp.dependencycheck.BaseDBTestCase; import org.owasp.dependencycheck.utils.Settings; +import java.util.List; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; +import static org.owasp.dependencycheck.analyzer.AnalysisPhase.FINAL; +import static org.owasp.dependencycheck.analyzer.AnalysisPhase.INITIAL; + /** * * @author Jeremy Long @@ -46,7 +51,22 @@ public class AnalyzerServiceTest extends BaseDBTestCase { } assertTrue("JarAnalyzer loaded", found); } - + + /** + * Test of getAnalyzers method, of class AnalyzerService. + */ + @Test + public void testGetAnalyzers_SpecificPhases() throws Exception { + AnalyzerService instance = new AnalyzerService(Thread.currentThread().getContextClassLoader()); + List result = instance.getAnalyzers(INITIAL, FINAL); + + for (Analyzer a : result) { + if (a.getAnalysisPhase() != INITIAL && a.getAnalysisPhase() != FINAL) { + fail("Only expecting analyzers for phases " + INITIAL + " and " + FINAL); + } + } + } + /** * Test of getAnalyzers method, of class AnalyzerService. */