Squashed commits for the new Solr/Central Search capability

Former-commit-id: 28ca3ca0ff5de4e097082f6f73003c0a67455efd

dependency-check-core/src/main/java/org/owasp/dependencycheck/analyzer/CentralAnalyzer.java

@@ -0,0 +1,167 @@
+package org.owasp.dependencycheck.analyzer;
+
+import org.owasp.dependencycheck.Engine;
+import org.owasp.dependencycheck.analyzer.exception.AnalysisException;
+import org.owasp.dependencycheck.data.nexus.MavenArtifact;
+import org.owasp.dependencycheck.data.central.CentralSearch;
+import org.owasp.dependencycheck.dependency.Confidence;
+import org.owasp.dependencycheck.dependency.Dependency;
+import org.owasp.dependencycheck.utils.InvalidSettingException;
+import org.owasp.dependencycheck.utils.Settings;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.net.URL;
+import java.util.List;
+import java.util.Set;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+/**
+ * Created by colezlaw on 10/9/14.
+ */
+public class CentralAnalyzer extends AbstractFileTypeAnalyzer {
+    /**
+     * The logger.
+     */
+    private static final Logger LOGGER = Logger.getLogger(CentralAnalyzer.class.getName());
+
+    /**
+     * The name of the analyzer.
+     */
+   private static final String ANALYZER_NAME = "Solr Analyzer";
+
+    /**
+     * The phase in which this analyzer runs.
+     */
+    private static final AnalysisPhase ANALYSIS_PHASE = AnalysisPhase.INFORMATION_COLLECTION;
+
+    /**
+     * The types of files on which this will work.
+     */
+    private static final Set<String> SUPPORTED_EXTENSIONS = newHashSet("jar");
+
+    /**
+     * The analyzer should be disabled if there are errors, so this is a flag
+     * to determine if such an error has occurred.
+     */
+    protected boolean errorFlag = false;
+
+    /**
+     * The searcher itself.
+     */
+    private CentralSearch searcher;
+
+    /**
+     * Determine whether to enable this analyzer or not.
+     *
+     * @return whether the analyzer should be enabled
+     */
+    @Override
+    public boolean isEnabled() {
+        boolean retval = false;
+
+        try {
+            if (Settings.getBoolean(Settings.KEYS.ANALYZER_SOLR_ENABLED)) {
+                if (!Settings.getBoolean(Settings.KEYS.ANALYZER_NEXUS_ENABLED)
+                        || NexusAnalyzer.DEFAULT_URL.equals(Settings.getString(Settings.KEYS.ANALYZER_NEXUS_URL))) {
+                    LOGGER.info("Enabling the Solr analyzer");
+                    retval = true;
+                } else {
+                    LOGGER.info("Nexus analyzer is enabled, disabling Solr");
+                }
+            } else {
+                LOGGER.info("Solr analyzer disabled");
+            }
+        } catch (InvalidSettingException ise) {
+            LOGGER.warning("Invalid setting. Disabling the Solr analyzer");
+        }
+
+        return retval;
+    }
+
+    /**
+     * Initializes the analyzer once before any analysis is performed.
+     *
+     * @throws Exception if there's an error during initalization
+     */
+    @Override
+    public void initializeFileTypeAnalyzer() throws Exception {
+        LOGGER.fine("Initializing Solr analyzer");
+        LOGGER.fine(String.format("Solr analyzer enabled: %s", isEnabled()));
+        if (isEnabled()) {
+            final String searchUrl = Settings.getString(Settings.KEYS.ANALYZER_SOLR_URL);
+            LOGGER.fine(String.format("Solr Analyzer URL: %s", searchUrl));
+            searcher = new CentralSearch(new URL(searchUrl));
+        }
+    }
+
+    /**
+     * Returns the analyzer's name.
+     *
+     * @return the name of the analyzer
+     */
+    @Override
+    public String getName() {
+        return ANALYZER_NAME;
+    }
+
+    /** Returns the key used in the properties file to to reference the analyzer's enabled property.
+     *
+     * @return the analyzer's enabled property setting key.
+     */
+    @Override
+    protected String getAnalyzerEnabledSettingKey() {
+        return Settings.KEYS.ANALYZER_SOLR_ENABLED;
+    }
+
+    /**
+     * Returns the analysis phase under which the analyzer runs.
+     *
+     * @return the phase under which the analyzer runs
+     */
+    @Override
+    public AnalysisPhase getAnalysisPhase() {
+        return ANALYSIS_PHASE;
+    }
+
+    /**
+     * Returns the extensions for which this Analyzer runs.
+     *
+     * @return the extensions for which this Analyzer runs
+     */
+    @Override
+    public Set<String> getSupportedExtensions() {
+        return SUPPORTED_EXTENSIONS;
+    }
+
+    /**
+     * Performs the analysis.
+     *
+     * @param dependency the dependency to analyze
+     * @param engine the engine
+     * @throws AnalysisException when there's an exception during analysis
+     */
+    @Override
+    public void analyzeFileType(Dependency dependency, Engine engine) throws AnalysisException {
+        if (errorFlag || !isEnabled()) {
+            return;
+        }
+
+        try {
+            final List<MavenArtifact> mas = searcher.searchSha1(dependency.getSha1sum());
+            final Confidence confidence = mas.size() > 1 ? Confidence.HIGH : Confidence.HIGHEST;
+            for (MavenArtifact ma : mas) {
+                LOGGER.fine(String.format("Central analyzer found artifact (%s) for dependency (%s)", ma.toString(), dependency.getFileName()));
+                dependency.addAsEvidence("central", ma, confidence);
+            }
+        } catch (IllegalArgumentException iae) {
+            LOGGER.info(String.format("invalid sha1-hash on %s", dependency.getFileName()));
+        } catch (FileNotFoundException fnfe) {
+            LOGGER.fine(String.format("Artifact not found in repository: '%s", dependency.getFileName()));
+        } catch (IOException ioe) {
+            LOGGER.log(Level.FINE, "Could not connect to Solr search", ioe);
+            errorFlag = true;
+        }
+    }
+}

dependency-check-core/src/main/java/org/owasp/dependencycheck/analyzer/NexusAnalyzer.java

@@ -31,8 +31,11 @@ import org.owasp.dependencycheck.data.nexus.MavenArtifact;
 import org.owasp.dependencycheck.data.nexus.NexusSearch;
 import org.owasp.dependencycheck.dependency.Confidence;
 import org.owasp.dependencycheck.dependency.Dependency;
+import org.owasp.dependencycheck.utils.InvalidSettingException;
 import org.owasp.dependencycheck.utils.Settings;
 
+import javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter;
+
 /**
  * Analyzer which will attempt to locate a dependency on a Nexus service by SHA-1 digest of the dependency.
  *
@@ -48,6 +51,10 @@ import org.owasp.dependencycheck.utils.Settings;
  * @author colezlaw
  */
 public class NexusAnalyzer extends AbstractFileTypeAnalyzer {
+    /**
+     * The default URL - this will be used by the SolrAnalyzer to determine whether to enable this.
+     */
+    public static final String DEFAULT_URL = "https://repository.sonatype.org/service/local/";
 
     /**
      * The logger.
@@ -74,6 +81,33 @@ public class NexusAnalyzer extends AbstractFileTypeAnalyzer {
      */
     private NexusSearch searcher;
 
+    /**
+     * Determine whether to enable this analyzer or not.
+     *
+     * @return whether the analyzer should be enabled
+     */
+    @Override
+    public boolean isEnabled() {
+        /* Enable this analyzer ONLY if the Nexus URL has been set to something
+           other than the default one (if it's the default one, we'll use the
+           central one) and it's enabled by the user.
+         */
+        boolean retval = false;
+        try {
+            if ((! DEFAULT_URL.equals(Settings.getString(Settings.KEYS.ANALYZER_NEXUS_URL)))
+                && Settings.getBoolean(Settings.KEYS.ANALYZER_NEXUS_ENABLED)) {
+                LOGGER.info("Enabling Nexus analyzer");
+                retval = true;
+            } else {
+                LOGGER.info("Nexus analyzer disabled");
+            }
+        } catch (InvalidSettingException ise) {
+            LOGGER.warning("Invalid setting. Disabling Nexus analyzer");
+        }
+
+        return retval;
+    }
+
     /**
      * Initializes the analyzer once before any analysis is performed.
      *
@@ -150,6 +184,9 @@ public class NexusAnalyzer extends AbstractFileTypeAnalyzer {
      */
     @Override
     public void analyzeFileType(Dependency dependency, Engine engine) throws AnalysisException {
+        if (! isEnabled()) {
+            return;
+        }
         try {
             final MavenArtifact ma = searcher.searchSha1(dependency.getSha1sum());
             dependency.addAsEvidence("nexus", ma, Confidence.HIGH);

dependency-check-core/src/main/java/org/owasp/dependencycheck/data/central/CentralSearch.java

@@ -0,0 +1,146 @@
+package org.owasp.dependencycheck.data.central;
+
+import org.owasp.dependencycheck.data.nexus.MavenArtifact;
+import org.owasp.dependencycheck.utils.InvalidSettingException;
+import org.owasp.dependencycheck.utils.Settings;
+import org.owasp.dependencycheck.utils.URLConnectionFactory;
+import org.w3c.dom.Document;
+import org.w3c.dom.NodeList;
+
+import javax.xml.parsers.DocumentBuilder;
+import javax.xml.parsers.DocumentBuilderFactory;
+import javax.xml.xpath.XPath;
+import javax.xml.xpath.XPathConstants;
+import javax.xml.xpath.XPathFactory;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.net.HttpURLConnection;
+import java.net.URL;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.logging.Logger;
+
+/**
+ * Class of methods to search Maven Central via Solr.
+ *
+ * @author colezlaw
+ */
+public class CentralSearch {
+    /**
+     * The URL for the Solr service
+     */
+    private final URL rootURL;
+
+    /**
+     * Whether to use the Proxy when making requests
+     */
+    private boolean useProxy;
+
+    /**
+     * Used for logging.
+     */
+    private static final Logger LOGGER = Logger.getLogger(CentralSearch.class.getName());
+
+    /**
+     * Determines whether we'll continue using the analyzer. If there's some sort
+     * of HTTP failure, we'll disable the analyzer.
+     */
+    private boolean isEnabled = true;
+
+    /**
+     * Creates a NexusSearch for the given repository URL.
+     *
+     * @param rootURL the URL of the repository on which searches should execute.
+     *                Only parameters are added to this (so it should end in /select)
+     */
+    public CentralSearch(URL rootURL) {
+        this.rootURL = rootURL;
+        try {
+            if (null != Settings.getString(Settings.KEYS.PROXY_SERVER)
+                    && Settings.getBoolean(Settings.KEYS.ANALYZER_SOLR_PROXY)) {
+                useProxy = true;
+                LOGGER.fine("Using proxy");
+            } else {
+                useProxy = false;
+                LOGGER.fine("Not using proxy");
+            }
+        } catch (InvalidSettingException ise) {
+            useProxy = false;
+        }
+    }
+
+    /**
+     * Searches the configured Solr URL for the given sha1 hash. If the artifact is found, a
+     * <code>MavenArtifact</code> is populated with the GAV.
+     *
+     * @param sha1 the SHA-1 hash string for which to search
+     * @return the populated Maven GAV.
+     * @throws IOException if it's unable to connect to the specified repository or if
+     *         the specified artifact is not found.
+     */
+    public List<MavenArtifact> searchSha1(String sha1) throws IOException {
+        if (null == sha1 || !sha1.matches("^[0-9A-Fa-f]{40}$")) {
+            throw new IllegalArgumentException("Invalid SHA1 format");
+        }
+
+        final URL url = new URL(rootURL + String.format("?q=1:\"%s\"&wt=xml", sha1));
+
+        LOGGER.info(String.format("Searching Solr url %s", url.toString()));
+
+        // Determine if we need to use a proxy. The rules:
+        // 1) If the proxy is set, AND the setting is set to true, use the proxy
+        // 2) Otherwise, don't use the proxy (either the proxy isn't configured,
+        // or proxy is specifically set to false)
+        final HttpURLConnection conn = URLConnectionFactory.createHttpURLConnection(url, useProxy);
+
+        conn.setDoOutput(true);
+
+        // JSON would be more elegant, but there's not currently a dependency
+        // on JSON, so don't want to add one just for this
+        conn.addRequestProperty("Accept", "application/xml");
+        conn.connect();
+
+        if (conn.getResponseCode() == 200) {
+            boolean missing = false;
+            try {
+                final DocumentBuilder builder = DocumentBuilderFactory
+                        .newInstance().newDocumentBuilder();
+                final Document doc = builder.parse(conn.getInputStream());
+                final XPath xpath = XPathFactory.newInstance().newXPath();
+                final String numFound = xpath.evaluate("/response/result/@numFound", doc);
+                if ("0".equals(numFound)) {
+                    missing = true;
+                } else {
+                    ArrayList<MavenArtifact> result = new ArrayList<MavenArtifact>();
+                    NodeList docs = (NodeList)xpath.evaluate("/response/result/doc", doc, XPathConstants.NODESET);
+                    for (int i = 0; i < docs.getLength(); i++) {
+                        final String g = xpath.evaluate("./str[@name='g']", docs.item(i));
+                        LOGGER.finest(String.format("GroupId: %s", g));
+                        final String a = xpath.evaluate("./str[@name='a']", docs.item(i));
+                        LOGGER.finest(String.format("ArtifactId: %s", a));
+                        final String v = xpath.evaluate("./str[@name='v']", docs.item(i));
+                        LOGGER.finest(String.format("Version: %s", v));
+                        result.add(new MavenArtifact(g, a, v, url.toString()));
+                    }
+
+                    return result;
+                }
+            } catch (Throwable e) {
+                // Anything else is jacked up XML stuff that we really can't recover
+                // from well
+                throw new IOException(e.getMessage(), e);
+            }
+
+            if (missing) {
+                throw new FileNotFoundException("Artifact not found in Solr");
+            }
+        } else {
+            final String msg = String.format("Could not connect to Solr received response code: %d %s",
+                    conn.getResponseCode(), conn.getResponseMessage());
+            LOGGER.fine(msg);
+            throw new IOException(msg);
+        }
+
+        return null;
+    }
+}

dependency-check-core/src/main/java/org/owasp/dependencycheck/dependency/Dependency.java

@@ -341,10 +341,12 @@ public class Dependency implements Serializable, Comparable<Dependency> {
                     found = true;
                     i.setConfidence(Confidence.HIGHEST);
                     i.setUrl(mavenArtifact.getArtifactUrl());
+                    LOGGER.fine(String.format("Already found identifier %s. Confidence set to highest", i.getValue()));
                     break;
                 }
             }
             if (!found) {
+                LOGGER.fine(String.format("Adding new maven identifier %s", mavenArtifact.toString()));
                 this.addIdentifier("maven", mavenArtifact.toString(), mavenArtifact.getArtifactUrl(), Confidence.HIGHEST);
             }
         }

dependency-check-core/src/main/resources/META-INF/services/org.owasp.dependencycheck.analyzer.Analyzer

@@ -8,6 +8,7 @@ org.owasp.dependencycheck.analyzer.CpeSuppressionAnalyzer
 org.owasp.dependencycheck.analyzer.DependencyBundlingAnalyzer
 org.owasp.dependencycheck.analyzer.NvdCveAnalyzer
 org.owasp.dependencycheck.analyzer.VulnerabilitySuppressionAnalyzer
+org.owasp.dependencycheck.analyzer.CentralAnalyzer
 org.owasp.dependencycheck.analyzer.NexusAnalyzer
 org.owasp.dependencycheck.analyzer.NuspecAnalyzer
 org.owasp.dependencycheck.analyzer.AssemblyAnalyzer

dependency-check-core/src/main/resources/dependencycheck.properties

@@ -58,3 +58,10 @@ analyzer.nexus.url=https://repository.sonatype.org/service/local/
 # If set to true, the proxy will still ONLY be used if the proxy properties (proxy.url, proxy.port)
 # are configured
 analyzer.nexus.proxy=true
+
+# the URL for searching search.maven.org for SHA-1 and whether it's enabled
+analyzer.solr.enabled=true
+analyzer.solr.url=http://search.maven.org/solrsearch/select
+# If set to true, the proxy will still ONLY be used if the proxy properties (proxy.url, proxy.port)
+# are configured
+analyzer.solr.proxy=true

dependency-check-core/src/test/java/org/owasp/dependencycheck/data/central/CentralSearchTest.java

@@ -0,0 +1,63 @@
+package org.owasp.dependencycheck.data.central;
+
+import org.junit.Before;
+import org.junit.Test;
+import org.owasp.dependencycheck.BaseTest;
+import org.owasp.dependencycheck.data.nexus.MavenArtifact;
+import org.owasp.dependencycheck.utils.Settings;
+
+import java.io.FileNotFoundException;
+import java.net.URL;
+import java.util.List;
+import java.util.logging.Logger;
+
+import static org.junit.Assert.*;
+
+/**
+ * Created by colezlaw on 10/13/14.
+ */
+public class CentralSearchTest extends BaseTest {
+    private static final Logger LOGGER = Logger.getLogger(CentralSearchTest.class.getName());
+    private CentralSearch searcher;
+
+    @Before
+    public void setUp() throws Exception {
+        String solrUrl = Settings.getString(Settings.KEYS.ANALYZER_SOLR_URL);
+        LOGGER.fine(solrUrl);
+        searcher = new CentralSearch(new URL(solrUrl));
+    }
+
+    @Test(expected = IllegalArgumentException.class)
+    public void testNullSha1() throws Exception { searcher.searchSha1(null); }
+
+    @Test(expected = IllegalArgumentException.class)
+    public void testMalformedSha1() throws Exception {
+        searcher.searchSha1("invalid");
+    }
+
+    // This test does generate network traffic and communicates with a host
+    // you may not be able to reach. Remove the @Ignore annotation if you want to
+    // test it anyway
+    @Test
+    public void testValidSha1() throws Exception {
+        List<MavenArtifact> ma = searcher.searchSha1("9977a8d04e75609cf01badc4eb6a9c7198c4c5ea");
+        assertEquals("Incorrect group", "org.apache.maven.plugins", ma.get(0).getGroupId());
+        assertEquals("Incorrect artifact", "maven-compiler-plugin", ma.get(0).getArtifactId());
+        assertEquals("Incorrect version", "3.1", ma.get(0).getVersion());
+    }
+
+    // This test does generate network traffic and communicates with a host
+    // you may not be able to reach. Remove the @Ignore annotation if you want to
+    // test it anyway
+    @Test(expected = FileNotFoundException.class)
+    public void testMissingSha1() throws Exception {
+        searcher.searchSha1("AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA");
+    }
+
+    // This test should give us multiple results back from Solr
+    @Test
+    public void testMultipleReturns() throws Exception {
+        List<MavenArtifact> ma = searcher.searchSha1("94A9CE681A42D0352B3AD22659F67835E560D107");
+        assertTrue(ma.size() > 1);
+    }
+}

dependency-check-core/src/test/resources/dependencycheck.properties

@@ -59,3 +59,9 @@ analyzer.nexus.url=https://repository.sonatype.org/service/local/
 # If set to true, the proxy will still ONLY be used if the proxy properties (proxy.url, proxy.port)
 # are configured
 analyzer.nexus.proxy=true
+
+# the URL for searching search.maven.org for SHA-1 and whether it's enabled
+analyzer.solr.enabled=true
+analyzer.solr.url=http://search.maven.org/solrsearch/select
+# If set to true, the proxy will still ONLY be used if the proxy properties (proxy.url, proxy.port)
+# are configured

dependency-check-utils/src/main/java/org/owasp/dependencycheck/utils/Settings.java

@@ -188,6 +188,18 @@ public final class Settings {
          * The properties key for using the proxy to reach Nexus.
          */
         public static final String ANALYZER_NEXUS_PROXY = "analyzer.nexus.proxy";
+        /**
+         * The properties key for whether the Solr analyzer is enabled.
+         */
+        public static final String ANALYZER_SOLR_ENABLED = "analyzer.solr.enabled";
+        /**
+         * The properties key for the Solr search URL.
+         */
+        public static final String ANALYZER_SOLR_URL = "analyzer.solr.url";
+        /**
+         * The properties key for using the proxy to reach Solr.
+         */
+        public static final String ANALYZER_SOLR_PROXY = "analyzer.solr.proxy";
         /**
          * The path to mono, if available.
          */