updates to reduce load times in multi-module projects per issue #168

Former-commit-id: adfaaaddffffa9b078d6b78a1ac031e6d8343f21

dependency-check-core/src/main/java/org/owasp/dependencycheck/Engine.java

@@ -18,7 +18,6 @@
 package org.owasp.dependencycheck;
 
 import java.io.File;
-import java.io.Serializable;
 import java.util.ArrayList;
 import java.util.EnumMap;
 import java.util.HashSet;
@@ -51,7 +50,7 @@ import org.owasp.dependencycheck.utils.Settings;
  *
  * @author Jeremy Long <jeremy.long@owasp.org>
  */
-public class Engine implements Serializable {
+public class Engine {
 
     /**
      * The list of dependencies.
@@ -60,19 +59,32 @@ public class Engine implements Serializable {
     /**
      * A Map of analyzers grouped by Analysis phase.
      */
-    private final transient EnumMap<AnalysisPhase, List<Analyzer>> analyzers;
+    private EnumMap<AnalysisPhase, List<Analyzer>> analyzers;
     /**
      * A Map of analyzers grouped by Analysis phase.
      */
-    private final transient Set<FileTypeAnalyzer> fileTypeAnalyzers;
+    private Set<FileTypeAnalyzer> fileTypeAnalyzers;
     /**
      * The ClassLoader to use when dynamically loading Analyzer and Update services.
      */
-    private transient ClassLoader serviceClassLoader;
+    private ClassLoader serviceClassLoader;
     /**
      * The Logger for use throughout the class.
      */
-    private static final transient Logger LOGGER = Logger.getLogger(Engine.class.getName());
+    private static Logger LOGGER = Logger.getLogger(Engine.class.getName());
+    /**
+     * A flag indicating whether or not an update has been performed.
+     */
+    private boolean hasBeenUpdated = false;
+
+    /**
+     * Returns <code>true</code> if an update has been performed.
+     *
+     * @return <code>true</code> if an update has been performed; otherwise <code>false</code>.
+     */
+    public boolean getHasBeenUpdated() {
+        return this.hasBeenUpdated;
+    }
 
     /**
      * Creates a new Engine.
@@ -80,7 +92,26 @@ public class Engine implements Serializable {
      * @throws DatabaseException thrown if there is an error connecting to the database
      */
     public Engine() throws DatabaseException {
-        this(Thread.currentThread().getContextClassLoader());
+        initializeEngine();
+    }
+
+    /**
+     * Creates a new Engine.
+     *
+     * @param serviceClassLoader a reference the class loader being used
+     * @throws DatabaseException thrown if there is an error connecting to the database
+     */
+    public Engine(ClassLoader serviceClassLoader) throws DatabaseException {
+        initializeEngine(serviceClassLoader);
+    }
+
+    /**
+     * Initializes the engine.
+     *
+     * @throws DatabaseException thrown if there is an error connecting to the database
+     */
+    protected final void initializeEngine() throws DatabaseException {
+        initializeEngine(Thread.currentThread().getContextClassLoader());
     }
 
     /**
@@ -89,7 +120,7 @@ public class Engine implements Serializable {
      * @param serviceClassLoader the ClassLoader to use when dynamically loading Analyzer and Update services
      * @throws DatabaseException thrown if there is an error connecting to the database
      */
-    public Engine(ClassLoader serviceClassLoader) throws DatabaseException {
+    protected final void initializeEngine(ClassLoader serviceClassLoader) throws DatabaseException {
         this.dependencies = new ArrayList<Dependency>();
         this.analyzers = new EnumMap<AnalysisPhase, List<Analyzer>>(AnalysisPhase.class);
         this.fileTypeAnalyzers = new HashSet<FileTypeAnalyzer>();
@@ -157,9 +188,6 @@ public class Engine implements Serializable {
 
     public void setDependencies(List<Dependency> dependencies) {
         this.dependencies = dependencies;
-        //for (Dependency dependency: dependencies) {
-        //    dependencies.add(dependency);
-        //}
     }
 
     /**
@@ -365,7 +393,7 @@ public class Engine implements Serializable {
             final List<Analyzer> analyzerList = analyzers.get(phase);
 
             for (Analyzer a : analyzerList) {
-                initializeAnalyzer(a);
+                a = initializeAnalyzer(a);
 
                 /* need to create a copy of the collection because some of the
                  * analyzers may modify it. This prevents ConcurrentModificationExceptions.
@@ -420,8 +448,9 @@ public class Engine implements Serializable {
      * Initializes the given analyzer.
      *
      * @param analyzer the analyzer to initialize
+     * @return the initialized analyzer
      */
-    private void initializeAnalyzer(Analyzer analyzer) {
+    protected Analyzer initializeAnalyzer(Analyzer analyzer) {
         try {
             final String msg = String.format("Initializing %s", analyzer.getName());
             LOGGER.log(Level.FINE, msg);
@@ -436,6 +465,7 @@ public class Engine implements Serializable {
                 LOGGER.log(Level.FINEST, null, ex1);
             }
         }
+        return analyzer;
     }
 
     /**
@@ -443,7 +473,7 @@ public class Engine implements Serializable {
      *
      * @param analyzer the analyzer to close
      */
-    private void closeAnalyzer(Analyzer analyzer) {
+    protected void closeAnalyzer(Analyzer analyzer) {
         final String msg = String.format("Closing Analyzer '%s'", analyzer.getName());
         LOGGER.log(Level.FINE, msg);
         try {
@@ -457,6 +487,7 @@ public class Engine implements Serializable {
      * Cycles through the cached web data sources and calls update on all of them.
      */
     private void doUpdates() {
+        LOGGER.info("Checking for updates");
         final UpdateService service = new UpdateService(serviceClassLoader);
         final Iterator<CachedWebDataSource> iterator = service.getDataSources();
         while (iterator.hasNext()) {
@@ -469,6 +500,8 @@ public class Engine implements Serializable {
                 LOGGER.log(Level.FINE, String.format("Unable to update details for %s", source.getClass().getName()), ex);
             }
         }
+        this.hasBeenUpdated = true;
+        LOGGER.info("Updates complete");
     }
 
     /**
@@ -511,16 +544,12 @@ public class Engine implements Serializable {
      * @throws DatabaseException thrown if there is an exception opening the database
      */
     private void ensureDataExists() throws NoDataException, DatabaseException {
-        //final CpeMemoryIndex cpe = CpeMemoryIndex.getInstance();
         final CveDB cve = new CveDB();
         try {
             cve.open();
             if (!cve.dataExists()) {
                 throw new NoDataException("No documents exist");
             }
-//            cpe.open(cve);
-//        } catch (IndexException ex) {
-//            throw new NoDataException(ex.getMessage(), ex);
         } catch (DatabaseException ex) {
             throw new NoDataException(ex.getMessage(), ex);
         } finally {

dependency-check-maven/src/main/java/org/owasp/dependencycheck/maven/DependencyCheckMojo.java

@@ -43,7 +43,6 @@ import org.apache.maven.project.MavenProject;
 import org.apache.maven.reporting.MavenReport;
 import org.apache.maven.reporting.MavenReportException;
 import org.apache.maven.settings.Proxy;
-import org.owasp.dependencycheck.Engine;
 import org.owasp.dependencycheck.analyzer.DependencyBundlingAnalyzer;
 import org.owasp.dependencycheck.analyzer.exception.AnalysisException;
 import org.owasp.dependencycheck.data.nexus.MavenArtifact;
@@ -326,7 +325,7 @@ public class DependencyCheckMojo extends ReportAggregationMojo {
      * @throws DatabaseException thrown if there is an exception connecting to the database
      */
     private Engine executeDependencyCheck(MavenProject project) throws DatabaseException {
-        final Engine localEngine = initializeEngine();
+        final Engine localEngine = initializeEngine(project);
 
         final Set<Artifact> artifacts = project.getArtifacts();
         for (Artifact a : artifacts) {
@@ -359,9 +358,9 @@ public class DependencyCheckMojo extends ReportAggregationMojo {
      * @return a newly instantiated <code>Engine</code>
      * @throws DatabaseException thrown if there is a database exception
      */
-    private Engine initializeEngine() throws DatabaseException {
+    private Engine initializeEngine(MavenProject project) throws DatabaseException {
         populateSettings();
-        final Engine localEngine = new Engine();
+        final Engine localEngine = new Engine(project);
         return localEngine;
     }
 
@@ -594,7 +593,7 @@ public class DependencyCheckMojo extends ReportAggregationMojo {
         final List<Dependency> deps = readDataFile();
         if (deps != null) {
             try {
-                engine = initializeEngine();
+                engine = initializeEngine(getProject());
                 engine.getDependencies().addAll(deps);
             } catch (DatabaseException ex) {
                 final String msg = String.format("An unrecoverable exception with the dependency-check initialization occured while scanning %s",
@@ -618,7 +617,7 @@ public class DependencyCheckMojo extends ReportAggregationMojo {
         List<Dependency> deps = readDataFile(project);
         if (deps != null) {
             try {
-                engine = initializeEngine();
+                engine = initializeEngine(project);
                 engine.getDependencies().addAll(deps);
             } catch (DatabaseException ex) {
                 final String msg = String.format("An unrecoverable exception with the dependency-check initialization occured while scanning %s",

dependency-check-maven/src/main/java/org/owasp/dependencycheck/maven/Engine.java

@@ -0,0 +1,121 @@
+/*
+ * This file is part of dependency-check-maven.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * Copyright (c) 2014 Jeremy Long. All Rights Reserved.
+ */
+package org.owasp.dependencycheck.maven;
+
+import java.util.logging.Logger;
+import org.apache.maven.project.MavenProject;
+import org.owasp.dependencycheck.analyzer.Analyzer;
+import org.owasp.dependencycheck.analyzer.CPEAnalyzer;
+import org.owasp.dependencycheck.data.nvdcve.DatabaseException;
+
+/**
+ * A modified version of the core engine specifically designed to persist some data between multiple executions of a
+ * multi-module Maven project.
+ *
+ * @author Jeremy Long <jeremy.long@owasp.org>
+ */
+public class Engine extends org.owasp.dependencycheck.Engine {
+
+    /**
+     * The logger.
+     */
+    private static final transient Logger LOGGER = Logger.getLogger(Engine.class.getName());
+    /**
+     * A key used to persist an object in the MavenProject.
+     */
+    private static final String CPE_ANALYZER_KEY = "dependency-check-CPEAnalyzer";
+    /**
+     * The current MavenProject.
+     */
+    private MavenProject currentProject;
+
+    private Engine() throws DatabaseException {
+    }
+
+    public Engine(MavenProject project) throws DatabaseException {
+        this.currentProject = project;
+        MavenProject parent = getRootParent();
+        if ((parent != null) && (parent.getContextValue("dependency-check-data-was-updated") != null)) {
+            System.setProperty("autoupdate", Boolean.FALSE.toString());
+        }
+        initializeEngine();
+        if (getHasBeenUpdated()) {
+            getRootParent().setContextValue("dependency-check-data-was-updated", Boolean.valueOf(true));
+        }
+    }
+
+    protected Analyzer initializeAnalyzer(Analyzer analyzer) {
+        if ((analyzer instanceof CPEAnalyzer)) {
+            CPEAnalyzer cpe = getPreviouslyLoadedAnalyzer();
+            if (cpe != null) {
+                return cpe;
+            }
+            cpe = (CPEAnalyzer) super.initializeAnalyzer(analyzer);
+            storeCPEAnalyzer(cpe);
+        }
+        return super.initializeAnalyzer(analyzer);
+    }
+
+    protected void closeAnalyzer(Analyzer analyzer) {
+        if ((analyzer instanceof CPEAnalyzer)) {
+            if (getPreviouslyLoadedAnalyzer() == null) {
+                super.closeAnalyzer(analyzer);
+            }
+        } else {
+            super.closeAnalyzer(analyzer);
+        }
+    }
+
+    public void cleanup() {
+        super.cleanup();
+    }
+
+    public void cleanupFinal() {
+        CPEAnalyzer cpe = getPreviouslyLoadedAnalyzer();
+        if (cpe != null) {
+            cpe.close();
+        }
+    }
+
+    private CPEAnalyzer getPreviouslyLoadedAnalyzer() {
+        CPEAnalyzer cpe = null;
+        MavenProject project = getRootParent();
+        if (project != null) {
+            cpe = (CPEAnalyzer) project.getContextValue(CPE_ANALYZER_KEY);
+        }
+        return cpe;
+    }
+
+    private void storeCPEAnalyzer(CPEAnalyzer cpe) {
+        MavenProject p = getRootParent();
+        if (p != null) {
+            p.setContextValue(CPE_ANALYZER_KEY, cpe);
+        }
+    }
+
+    private MavenProject getRootParent() {
+        if (this.currentProject == null) {
+            return null;
+        }
+        MavenProject p = this.currentProject;
+        while (p.getParent() != null) {
+            p = p.getParent();
+        }
+        return p;
+    }
+}

dependency-check-maven/src/main/java/org/owasp/dependencycheck/maven/ReportingUtil.java

@@ -28,7 +28,6 @@ import java.util.Set;
 import java.util.logging.Level;
 import java.util.logging.Logger;
 import org.apache.maven.doxia.sink.Sink;
-import org.owasp.dependencycheck.Engine;
 import org.owasp.dependencycheck.data.nvdcve.CveDB;
 import org.owasp.dependencycheck.data.nvdcve.DatabaseException;
 import org.owasp.dependencycheck.data.nvdcve.DatabaseProperties;