reworked aggregation mojo to resolve issues #325, #386, and #531

dependency-check-maven/pom.xml

@@ -208,6 +208,10 @@ Copyright (c) 2013 Jeremy Long. All Rights Reserved.
             <groupId>org.sonatype.plexus</groupId>
             <artifactId>plexus-sec-dispatcher</artifactId>
         </dependency>
+        <dependency>
+            <groupId>org.apache.maven.shared</groupId>
+            <artifactId>maven-dependency-tree</artifactId>
+        </dependency>
         <dependency>
             <groupId>org.jmockit</groupId>
             <artifactId>jmockit</artifactId>

dependency-check-maven/src/main/java/org/owasp/dependencycheck/maven/AggregateMojo.java

@@ -19,10 +19,8 @@ package org.owasp.dependencycheck.maven;
 
 import java.io.File;
 import java.io.IOException;
-import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashSet;
-import java.util.List;
 import java.util.Locale;
 import java.util.Set;
 import org.apache.maven.plugin.MojoExecutionException;
@@ -32,10 +30,7 @@ import org.apache.maven.plugins.annotations.Mojo;
 import org.apache.maven.plugins.annotations.Parameter;
 import org.apache.maven.plugins.annotations.ResolutionScope;
 import org.apache.maven.project.MavenProject;
-import org.owasp.dependencycheck.analyzer.DependencyBundlingAnalyzer;
-import org.owasp.dependencycheck.analyzer.exception.AnalysisException;
 import org.owasp.dependencycheck.data.nvdcve.DatabaseException;
-import org.owasp.dependencycheck.dependency.Dependency;
 import org.owasp.dependencycheck.exception.ExceptionCollection;
 import org.owasp.dependencycheck.exception.ReportException;
 import org.owasp.dependencycheck.utils.Settings;
@@ -49,7 +44,7 @@ import org.owasp.dependencycheck.utils.Settings;
 @Mojo(
         name = "aggregate",
         defaultPhase = LifecyclePhase.VERIFY,
-        /*aggregator = true,*/
+        aggregator = true,
         threadSafe = false,
         requiresDependencyResolution = ResolutionScope.COMPILE_PLUS_RUNTIME,
         requiresOnline = true
@@ -72,103 +67,87 @@ public class AggregateMojo extends BaseDependencyCheckMojo {
      */
     @Override
     public void runCheck() throws MojoExecutionException, MojoFailureException {
-        final MavenEngine engine = generateDataFile();
+        final MavenEngine engine = loadEngine();
         if (engine == null) {
             return;
         }
 
-        if (getProject() == getLastProject()) {
-            //ensure that the .ser file was created for each.
-            for (MavenProject current : getReactorProjects()) {
-                final File dataFile = getDataFile(current);
-                if (dataFile == null && !skipProject(current)) { //dc was never run on this project. write the ser to the target.
-                    getLog().error(String.format("Module '%s' did not execute dependency-check; an attempt will be made to perform "
-                            + "the check but dependencies may be missed resulting in false negatives.", current.getName()));
-                    generateDataFile(engine, current);
-                }
-            }
-            for (MavenProject current : getReactorProjects()) {
-                List<Dependency> dependencies = readDataFile(current);
-                if (dependencies == null) {
-                    dependencies = new ArrayList<Dependency>();
-                }
-                final Set<MavenProject> childProjects = getDescendants(current);
-                for (MavenProject reportOn : childProjects) {
-                    final List<Dependency> childDeps = readDataFile(reportOn);
-                    if (childDeps != null && !childDeps.isEmpty()) {
-                        if (getLog().isDebugEnabled()) {
-                            getLog().debug(String.format("Adding %d dependencies from %s", childDeps.size(), reportOn.getName()));
-                        }
-                        dependencies.addAll(childDeps);
-                    } else if (getLog().isDebugEnabled()) {
-                        getLog().debug(String.format("No dependencies read for %s", reportOn.getName()));
-                    }
-                }
-                engine.getDependencies().clear();
-                engine.getDependencies().addAll(dependencies);
-                final DependencyBundlingAnalyzer bundler = new DependencyBundlingAnalyzer();
-                try {
-                    if (getLog().isDebugEnabled()) {
-                        getLog().debug(String.format("Dependency count pre-bundler: %s", engine.getDependencies().size()));
-                    }
-                    bundler.analyze(null, engine);
-                    if (getLog().isDebugEnabled()) {
-                        getLog().debug(String.format("Dependency count post-bundler: %s", engine.getDependencies().size()));
-                    }
-                } catch (AnalysisException ex) {
-                    getLog().warn("An error occurred grouping the dependencies; duplicate entries may exist in the report", ex);
-                    getLog().debug("Bundling Exception", ex);
-                }
+        ExceptionCollection exCol = scanArtifacts(getProject(), engine);
 
-                File outputDir = getCorrectOutputDirectory(current);
-                if (outputDir == null) {
-                    //in some regards we shouldn't be writting this, but we are anyway.
-                    //we shouldn't write this because nothing is configured to generate this report.
-                    outputDir = new File(current.getBuild().getDirectory());
+        for (MavenProject childProject : getDescendants(this.getProject())) {
+            ExceptionCollection ex = scanArtifacts(childProject, engine);
+            if (ex != null) {
+                if (exCol == null) {
+                    exCol = ex;
                 }
-                try {
-                    writeReports(engine, current, outputDir);
-                } catch (ReportException ex) {
-                    ExceptionCollection exCol = (ExceptionCollection) engine.getExecutionRoot().getContextValue(AGGREGATE_EXCEPTIONS);
-                    if (exCol == null) {
-                        exCol = new ExceptionCollection("Error writing aggregate report", ex);
-                    } else {
-                        exCol.addException(ex);
-                    }
-                    if (this.isFailOnError()) {
-                        throw new MojoExecutionException("One or more exceptions occured during dependency-check analysis", exCol);
-                    } else {
-                        getLog().debug("One or more exceptions occured during dependency-check analysis", exCol);
-                    }
+                exCol.getExceptions().addAll(ex.getExceptions());
+                if (ex.isFatal()) {
+                    exCol.setFatal(true);
                 }
             }
         }
+
+        try {
+            engine.analyzeDependencies();
+        } catch (ExceptionCollection ex) {
+            if (exCol == null) {
+                exCol = ex;
+            } else if (ex.isFatal()) {
+                exCol.setFatal(true);
+                exCol.getExceptions().addAll(ex.getExceptions());
+            }
+            if (exCol.isFatal()) {
+                final String msg = String.format("Fatal exception(s) analyzing %s", getProject().getName());
+                if (this.isFailOnError()) {
+                    throw new MojoExecutionException(msg, exCol);
+                }
+                getLog().error(msg);
+                if (getLog().isDebugEnabled()) {
+                    getLog().debug(exCol);
+                }
+                return;
+            } else {
+                final String msg = String.format("Exception(s) analyzing %s", getProject().getName());
+                if (getLog().isDebugEnabled()) {
+                    getLog().debug(msg, exCol);
+                }
+            }
+        }
+        File outputDir = getCorrectOutputDirectory(this.getProject());
+        if (outputDir == null) {
+            //in some regards we shouldn't be writting this, but we are anyway.
+            //we shouldn't write this because nothing is configured to generate this report.
+            outputDir = new File(this.getProject().getBuild().getDirectory());
+        }
+        try {
+            writeReports(engine, this.getProject(), outputDir);
+        } catch (ReportException ex) {
+            if (exCol == null) {
+                exCol = new ExceptionCollection("Error writing aggregate report", ex);
+            } else {
+                exCol.addException(ex);
+            }
+            if (this.isFailOnError()) {
+                throw new MojoExecutionException("One or more exceptions occured during dependency-check analysis", exCol);
+            } else {
+                getLog().debug("One or more exceptions occured during dependency-check analysis", exCol);
+            }
+        }
+        showSummary(this.getProject(), engine.getDependencies());
+        checkForFailure(engine.getDependencies());
         engine.cleanup();
         Settings.cleanup();
     }
 
-    /**
-     * Gets the last project in the reactor - taking into account skipped
-     * projects.
-     *
-     * @return the last project in the reactor
-     */
-    private MavenProject getLastProject() {
-        for (int x = getReactorProjects().size() - 1; x >= 0; x--) {
-            final MavenProject p = getReactorProjects().get(x);
-            if (!skipProject(p)) {
-                return p;
-            }
-        }
-        return null;
-    }
-
     /**
      * Tests if the project is being skipped in the Maven site report.
      *
      * @param project a project in the reactor
      * @return true if the project is skipped; otherwise false
+     * @deprecated this function is no longer used, keeping this code around for
+     * a little bit longer in case this needs to be used
      */
+    @Deprecated
     private boolean skipProject(MavenProject project) {
         final String skip = (String) project.getProperties().get("maven.site.skip");
         return "true".equalsIgnoreCase(skip) && isGeneratingSite();
@@ -264,16 +243,15 @@ public class AggregateMojo extends BaseDependencyCheckMojo {
     }
 
     /**
-     * Initializes the engine, runs a scan, and writes the serialized
-     * dependencies to disk.
+     * Initializes the engine.
      *
      * @return the MavenEngine used to execute dependency-check
      * @throws MojoExecutionException thrown if there is an exception running
-     * the mojo
+     * the Mojo
      * @throws MojoFailureException thrown if dependency-check is configured to
      * fail the build if severe CVEs are identified.
      */
-    protected MavenEngine generateDataFile() throws MojoExecutionException, MojoFailureException {
+    protected MavenEngine loadEngine() throws MojoExecutionException, MojoFailureException {
         MavenEngine engine = null;
         try {
             engine = initializeEngine();
@@ -286,59 +264,7 @@ public class AggregateMojo extends BaseDependencyCheckMojo {
                 throw new MojoExecutionException(msg, ex);
             }
             getLog().error(msg, ex);
-            return null;
         }
-        return generateDataFile(engine, getProject());
-    }
-
-    /**
-     * Runs dependency-check's MavenEngine and writes the serialized
-     * dependencies to disk.
-     *
-     * @param engine the MavenEngine to use when scanning.
-     * @param project the project to scan and generate the data file for
-     * @return the MavenEngine used to execute dependency-check
-     * @throws MojoExecutionException thrown if there is an exception running
-     * the mojo
-     * @throws MojoFailureException thrown if dependency-check is configured to
-     * fail the build if severe CVEs are identified.
-     */
-    protected MavenEngine generateDataFile(MavenEngine engine, MavenProject project) throws MojoExecutionException, MojoFailureException {
-        if (getLog().isDebugEnabled()) {
-            getLog().debug(String.format("Begin Scanning: %s", project.getName()));
-        }
-        engine.getDependencies().clear();
-        engine.resetFileTypeAnalyzers();
-        scanArtifacts(project, engine);
-        try {
-            engine.analyzeDependencies();
-        } catch (ExceptionCollection ex) {
-            ExceptionCollection col = (ExceptionCollection) engine.getExecutionRoot().getContextValue(AGGREGATE_EXCEPTIONS);
-            if (col == null) {
-                col = ex;
-            } else if (ex.isFatal()) {
-                col.setFatal(true);
-                col.getExceptions().addAll(ex.getExceptions());
-            }
-            if (col.isFatal()) {
-                final String msg = String.format("Fatal exception(s) analyzing %s", project.getName());
-                if (this.isFailOnError()) {
-                    throw new MojoExecutionException(msg, ex);
-                }
-                getLog().error(msg, col);
-                return null;
-            } else {
-                final String msg = String.format("Exception(s) analyzing %s", project.getName());
-                if (getLog().isDebugEnabled()) {
-                    getLog().debug(msg, ex);
-                }
-                engine.getExecutionRoot().setContextValue(AGGREGATE_EXCEPTIONS, col);
-            }
-        }
-        final File target = new File(project.getBuild().getDirectory());
-        writeDataFile(project, target, engine.getDependencies());
-        showSummary(project, engine.getDependencies());
-        checkForFailure(engine.getDependencies());
         return engine;
     }
 

dependency-check-maven/src/main/java/org/owasp/dependencycheck/maven/BaseDependencyCheckMojo.java

@@ -25,9 +25,11 @@ import java.io.FileOutputStream;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.ObjectOutputStream;
+import java.util.ArrayList;
 import java.util.List;
 import java.util.Locale;
-import org.apache.maven.artifact.Artifact;
+//import org.apache.maven.artifact.Artifact;
+import org.eclipse.aether.artifact.Artifact;
 import org.apache.maven.doxia.sink.Sink;
 import org.apache.maven.plugin.AbstractMojo;
 import org.apache.maven.plugin.MojoExecutionException;
@@ -39,6 +41,16 @@ import org.apache.maven.reporting.MavenReport;
 import org.apache.maven.reporting.MavenReportException;
 import org.apache.maven.settings.Proxy;
 import org.apache.maven.settings.Server;
+import org.apache.maven.shared.dependency.graph.DependencyGraphBuilder;
+import org.apache.maven.shared.dependency.graph.DependencyGraphBuilderException;
+import org.apache.maven.shared.dependency.graph.DependencyNode;
+import org.eclipse.aether.RepositorySystem;
+import org.eclipse.aether.RepositorySystemSession;
+import org.eclipse.aether.artifact.DefaultArtifact;
+import org.eclipse.aether.repository.RemoteRepository;
+import org.eclipse.aether.resolution.ArtifactRequest;
+import org.eclipse.aether.resolution.ArtifactResolutionException;
+import org.eclipse.aether.resolution.ArtifactResult;
 import org.owasp.dependencycheck.data.nexus.MavenArtifact;
 import org.owasp.dependencycheck.data.nvdcve.CveDB;
 import org.owasp.dependencycheck.data.nvdcve.DatabaseException;
@@ -47,6 +59,7 @@ import org.owasp.dependencycheck.dependency.Confidence;
 import org.owasp.dependencycheck.dependency.Dependency;
 import org.owasp.dependencycheck.dependency.Identifier;
 import org.owasp.dependencycheck.dependency.Vulnerability;
+import org.owasp.dependencycheck.exception.ExceptionCollection;
 import org.owasp.dependencycheck.exception.ReportException;
 import org.owasp.dependencycheck.reporting.ReportGenerator;
 import org.owasp.dependencycheck.utils.ExpectedOjectInputStream;
@@ -102,6 +115,30 @@ public abstract class BaseDependencyCheckMojo extends AbstractMojo implements Ma
      */
     @Parameter(readonly = true, required = true, property = "reactorProjects")
     private List<MavenProject> reactorProjects;
+    /**
+     * The entry point to Aether, i.e. the component doing all the work.
+     */
+    @Component
+    private RepositorySystem repoSystem;
+
+    /**
+     * The current repository/network configuration of Maven.
+     */
+    @Parameter(defaultValue = "${repositorySystemSession}", readonly = true)
+    private RepositorySystemSession repoSession;
+
+    /**
+     * The project's remote repositories to use for the resolution of plug-ins
+     * and their dependencies.
+     */
+    @Parameter(defaultValue = "${project.remotePluginRepositories}", readonly = true)
+    private List<RemoteRepository> remoteRepos;
+
+    /**
+     * Component within Maven to build the dependency graph.
+     */
+    @Component
+    private DependencyGraphBuilder dependencyGraphBuilder;
 
     /**
      * The output directory. This generally maps to "target".
@@ -553,32 +590,121 @@ public abstract class BaseDependencyCheckMojo extends AbstractMojo implements Ma
      *
      * @param project the project to scan the dependencies of
      * @param engine the engine to use to scan the dependencies
+     * @return a collection of exceptions that may have occurred while resolving
+     * and scanning the dependencies
      */
-    protected void scanArtifacts(MavenProject project, MavenEngine engine) {
-        for (Artifact a : project.getArtifacts()) {
+    protected ExceptionCollection scanArtifacts(MavenProject project, MavenEngine engine) {
+        // <editor-fold defaultstate="collapsed" desc="old implementation">
+        /*
+            for (Artifact a : project.getArtifacts()) {
             if (excludeFromScan(a)) {
-                continue;
+            continue;
             }
             final List<Dependency> deps = engine.scan(a.getFile().getAbsoluteFile());
             if (deps != null) {
-                if (deps.size() == 1) {
-                    final Dependency d = deps.get(0);
-                    if (d != null) {
-                        final MavenArtifact ma = new MavenArtifact(a.getGroupId(), a.getArtifactId(), a.getVersion());
-                        d.addAsEvidence("pom", ma, Confidence.HIGHEST);
-                        d.addProjectReference(project.getName());
-                        if (getLog().isDebugEnabled()) {
-                            getLog().debug(String.format("Adding project reference %s on dependency %s", project.getName(),
-                                    d.getDisplayFileName()));
+            if (deps.size() == 1) {
+            final Dependency d = deps.get(0);
+            if (d != null) {
+            final MavenArtifact ma = new MavenArtifact(a.getGroupId(), a.getArtifactId(), a.getVersion());
+            d.addAsEvidence("pom", ma, Confidence.HIGHEST);
+            d.addProjectReference(project.getName());
+            if (getLog().isDebugEnabled()) {
+            getLog().debug(String.format("Adding project reference %s on dependency %s", project.getName(),
+            d.getDisplayFileName()));
+            }
+            }
+            } else if (getLog().isDebugEnabled()) {
+            final String msg = String.format("More then 1 dependency was identified in first pass scan of '%s:%s:%s'",
+            a.getGroupId(), a.getArtifactId(), a.getVersion());
+            getLog().debug(msg);
+            }
+            }
+            }
+         */
+        // </editor-fold>
+        try {
+            DependencyNode dn = dependencyGraphBuilder.buildDependencyGraph(project, null, reactorProjects);
+            return collectDependencies(engine, project, dn.getChildren());
+        } catch (DependencyGraphBuilderException ex) {
+            final String msg = String.format("Unable to build dependency graph on project %s", project.getName());
+            getLog().debug(msg, ex);
+            return new ExceptionCollection(msg, ex);
+        }
+    }
+
+    /**
+     * Resolves the projects artifacts using Aether and scans the resulting
+     * dependencies.
+     *
+     * @param engine the core dependency-check engine
+     * @param project the project being scanned
+     * @param nodes the list of dependency nodes, generally obtained via the
+     * DependencyGraphBuilder
+     * @return a collection of exceptions that may have occurred while resolving
+     * and scanning the dependencies
+     */
+    private ExceptionCollection collectDependencies(MavenEngine engine, MavenProject project, List<DependencyNode> nodes) {
+        ExceptionCollection exCol = null;
+        for (DependencyNode dependencyNode : nodes) {
+            exCol = collectDependencies(engine, project, dependencyNode.getChildren());
+            if (excludeFromScan(dependencyNode.getArtifact().getScope())) {
+                continue;
+            }
+            ArtifactRequest request = new ArtifactRequest();
+            request.setArtifact(new DefaultArtifact(dependencyNode.getArtifact().getId()));
+            request.setRepositories(remoteRepos);
+            try {
+                ArtifactResult result = repoSystem.resolveArtifact(repoSession, request);
+                if (result.isResolved() && result.getArtifact() != null && result.getArtifact().getFile() != null) {
+                    final List<Dependency> deps = engine.scan(result.getArtifact().getFile().getAbsoluteFile());
+                    if (deps != null) {
+                        if (deps.size() == 1) {
+                            final Dependency d = deps.get(0);
+                            if (d != null) {
+                                Artifact a = result.getArtifact();
+                                final MavenArtifact ma = new MavenArtifact(a.getGroupId(), a.getArtifactId(), a.getVersion());
+                                d.addAsEvidence("pom", ma, Confidence.HIGHEST);
+                                d.addProjectReference(project.getName() + ":" + dependencyNode.getArtifact().getScope());
+                                if (getLog().isDebugEnabled()) {
+                                    getLog().debug(String.format("Adding project reference %s on dependency %s",
+                                            project.getName(), d.getDisplayFileName()));
+                                }
+                            }
+                        } else if (getLog().isDebugEnabled()) {
+                            final String msg = String.format("More then 1 dependency was identified in first pass scan of '%s' in project %s",
+                                    dependencyNode.getArtifact().getId(), project.getName());
+                            getLog().debug(msg);
+                        }
+                    } else {
+                        final String msg = String.format("Error resolving '%s' in project %s",
+                                dependencyNode.getArtifact().getId(), project.getName());
+                        if (exCol == null) {
+                            exCol = new ExceptionCollection();
+                        }
+                        getLog().error(msg);
+                        for (Exception ex : result.getExceptions()) {
+                            exCol.addException(ex);
                         }
                     }
-                } else if (getLog().isDebugEnabled()) {
-                    final String msg = String.format("More then 1 dependency was identified in first pass scan of '%s:%s:%s'",
-                            a.getGroupId(), a.getArtifactId(), a.getVersion());
+                } else {
+                    final String msg = String.format("Unable to resolve '%s' in project %s",
+                            dependencyNode.getArtifact().getId(), project.getName());
                     getLog().debug(msg);
+                    if (exCol == null) {
+                        exCol = new ExceptionCollection();
+                    }
+                    for (Exception ex : result.getExceptions()) {
+                        exCol.addException(ex);
+                    }
                 }
+            } catch (ArtifactResolutionException ex) {
+                if (exCol == null) {
+                    exCol = new ExceptionCollection();
+                }
+                exCol.addException(ex);
             }
         }
+        return exCol;
     }
 
     /**
@@ -669,8 +795,7 @@ public abstract class BaseDependencyCheckMojo extends AbstractMojo implements Ma
      */
     protected MavenEngine initializeEngine() throws DatabaseException {
         populateSettings();
-        return new MavenEngine(this.project,
-                this.reactorProjects);
+        return new MavenEngine(this.project, this.reactorProjects);
     }
 
     /**
@@ -835,18 +960,18 @@ public abstract class BaseDependencyCheckMojo extends AbstractMojo implements Ma
      * Tests is the artifact should be included in the scan (i.e. is the
      * dependency in a scope that is being scanned).
      *
-     * @param a the Artifact to test
+     * @param scope the scope of the artifact to test
      * @return <code>true</code> if the artifact is in an excluded scope;
      * otherwise <code>false</code>
      */
-    protected boolean excludeFromScan(Artifact a) {
-        if (skipTestScope && Artifact.SCOPE_TEST.equals(a.getScope())) {
+    protected boolean excludeFromScan(String scope) {
+        if (skipTestScope && org.apache.maven.artifact.Artifact.SCOPE_TEST.equals(scope)) {
             return true;
         }
-        if (skipProvidedScope && Artifact.SCOPE_PROVIDED.equals(a.getScope())) {
+        if (skipProvidedScope && org.apache.maven.artifact.Artifact.SCOPE_PROVIDED.equals(scope)) {
             return true;
         }
-        if (skipRuntimeScope && !Artifact.SCOPE_RUNTIME.equals(a.getScope())) {
+        if (skipRuntimeScope && !org.apache.maven.artifact.Artifact.SCOPE_RUNTIME.equals(scope)) {
             return true;
         }
         return false;
@@ -1133,4 +1258,5 @@ public abstract class BaseDependencyCheckMojo extends AbstractMojo implements Ma
         return ret;
     }
     //</editor-fold>
+
 }

dependency-check-maven/src/main/java/org/owasp/dependencycheck/maven/CheckMojo.java

@@ -55,7 +55,7 @@ public class CheckMojo extends BaseDependencyCheckMojo {
     public boolean canGenerateReport() {
         boolean isCapable = false;
         for (Artifact a : getProject().getArtifacts()) {
-            if (!excludeFromScan(a)) {
+            if (!excludeFromScan(a.getScope())) {
                 isCapable = true;
                 break;
             }
@@ -88,11 +88,10 @@ public class CheckMojo extends BaseDependencyCheckMojo {
             getLog().error(msg);
         }
         if (engine != null) {
-            scanArtifacts(getProject(), engine);
+            ExceptionCollection exCol = scanArtifacts(getProject(), engine);
             if (engine.getDependencies().isEmpty()) {
                 getLog().info("No dependencies were identified that could be analyzed by dependency-check");
             } else {
-                ExceptionCollection exCol = null;
                 try {
                     engine.analyzeDependencies();
                 } catch (ExceptionCollection ex) {

pom.xml

@@ -671,6 +671,11 @@ Copyright (c) 2012 - Jeremy Long
                 <artifactId>plexus-sec-dispatcher</artifactId>
                 <version>1.4</version>
             </dependency>
+            <dependency>
+                <groupId>org.apache.maven.shared</groupId>
+                <artifactId>maven-dependency-tree</artifactId>
+                <version>2.2</version>
+            </dependency>
             <dependency>
                 <groupId>org.glassfish</groupId>
                 <artifactId>javax.json</artifactId>