diff --git a/dependency-check-core/src/main/java/org/owasp/dependencycheck/analyzer/CentralAnalyzer.java b/dependency-check-core/src/main/java/org/owasp/dependencycheck/analyzer/CentralAnalyzer.java new file mode 100644 index 000000000..76744b675 --- /dev/null +++ b/dependency-check-core/src/main/java/org/owasp/dependencycheck/analyzer/CentralAnalyzer.java @@ -0,0 +1,167 @@ +package org.owasp.dependencycheck.analyzer; + +import org.owasp.dependencycheck.Engine; +import org.owasp.dependencycheck.analyzer.exception.AnalysisException; +import org.owasp.dependencycheck.data.nexus.MavenArtifact; +import org.owasp.dependencycheck.data.central.CentralSearch; +import org.owasp.dependencycheck.dependency.Confidence; +import org.owasp.dependencycheck.dependency.Dependency; +import org.owasp.dependencycheck.utils.InvalidSettingException; +import org.owasp.dependencycheck.utils.Settings; + +import java.io.FileNotFoundException; +import java.io.IOException; +import java.net.URL; +import java.util.List; +import java.util.Set; +import java.util.logging.Level; +import java.util.logging.Logger; + +/** + * Created by colezlaw on 10/9/14. + */ +public class CentralAnalyzer extends AbstractFileTypeAnalyzer { + /** + * The logger. + */ + private static final Logger LOGGER = Logger.getLogger(CentralAnalyzer.class.getName()); + + /** + * The name of the analyzer. + */ + private static final String ANALYZER_NAME = "Solr Analyzer"; + + /** + * The phase in which this analyzer runs. + */ + private static final AnalysisPhase ANALYSIS_PHASE = AnalysisPhase.INFORMATION_COLLECTION; + + /** + * The types of files on which this will work. + */ + private static final Set SUPPORTED_EXTENSIONS = newHashSet("jar"); + + /** + * The analyzer should be disabled if there are errors, so this is a flag + * to determine if such an error has occurred. + */ + protected boolean errorFlag = false; + + /** + * The searcher itself. + */ + private CentralSearch searcher; + + /** + * Determine whether to enable this analyzer or not. + * + * @return whether the analyzer should be enabled + */ + @Override + public boolean isEnabled() { + boolean retval = false; + + try { + if (Settings.getBoolean(Settings.KEYS.ANALYZER_SOLR_ENABLED)) { + if (!Settings.getBoolean(Settings.KEYS.ANALYZER_NEXUS_ENABLED) + || NexusAnalyzer.DEFAULT_URL.equals(Settings.getString(Settings.KEYS.ANALYZER_NEXUS_URL))) { + LOGGER.info("Enabling the Solr analyzer"); + retval = true; + } else { + LOGGER.info("Nexus analyzer is enabled, disabling Solr"); + } + } else { + LOGGER.info("Solr analyzer disabled"); + } + } catch (InvalidSettingException ise) { + LOGGER.warning("Invalid setting. Disabling the Solr analyzer"); + } + + return retval; + } + + /** + * Initializes the analyzer once before any analysis is performed. + * + * @throws Exception if there's an error during initalization + */ + @Override + public void initializeFileTypeAnalyzer() throws Exception { + LOGGER.fine("Initializing Solr analyzer"); + LOGGER.fine(String.format("Solr analyzer enabled: %s", isEnabled())); + if (isEnabled()) { + final String searchUrl = Settings.getString(Settings.KEYS.ANALYZER_SOLR_URL); + LOGGER.fine(String.format("Solr Analyzer URL: %s", searchUrl)); + searcher = new CentralSearch(new URL(searchUrl)); + } + } + + /** + * Returns the analyzer's name. + * + * @return the name of the analyzer + */ + @Override + public String getName() { + return ANALYZER_NAME; + } + + /** Returns the key used in the properties file to to reference the analyzer's enabled property. + * + * @return the analyzer's enabled property setting key. + */ + @Override + protected String getAnalyzerEnabledSettingKey() { + return Settings.KEYS.ANALYZER_SOLR_ENABLED; + } + + /** + * Returns the analysis phase under which the analyzer runs. + * + * @return the phase under which the analyzer runs + */ + @Override + public AnalysisPhase getAnalysisPhase() { + return ANALYSIS_PHASE; + } + + /** + * Returns the extensions for which this Analyzer runs. + * + * @return the extensions for which this Analyzer runs + */ + @Override + public Set getSupportedExtensions() { + return SUPPORTED_EXTENSIONS; + } + + /** + * Performs the analysis. + * + * @param dependency the dependency to analyze + * @param engine the engine + * @throws AnalysisException when there's an exception during analysis + */ + @Override + public void analyzeFileType(Dependency dependency, Engine engine) throws AnalysisException { + if (errorFlag || !isEnabled()) { + return; + } + + try { + final List mas = searcher.searchSha1(dependency.getSha1sum()); + final Confidence confidence = mas.size() > 1 ? Confidence.HIGH : Confidence.HIGHEST; + for (MavenArtifact ma : mas) { + LOGGER.fine(String.format("Central analyzer found artifact (%s) for dependency (%s)", ma.toString(), dependency.getFileName())); + dependency.addAsEvidence("central", ma, confidence); + } + } catch (IllegalArgumentException iae) { + LOGGER.info(String.format("invalid sha1-hash on %s", dependency.getFileName())); + } catch (FileNotFoundException fnfe) { + LOGGER.fine(String.format("Artifact not found in repository: '%s", dependency.getFileName())); + } catch (IOException ioe) { + LOGGER.log(Level.FINE, "Could not connect to Solr search", ioe); + errorFlag = true; + } + } +} diff --git a/dependency-check-core/src/main/java/org/owasp/dependencycheck/analyzer/NexusAnalyzer.java b/dependency-check-core/src/main/java/org/owasp/dependencycheck/analyzer/NexusAnalyzer.java index 1a21ba2b0..2d9d608e5 100644 --- a/dependency-check-core/src/main/java/org/owasp/dependencycheck/analyzer/NexusAnalyzer.java +++ b/dependency-check-core/src/main/java/org/owasp/dependencycheck/analyzer/NexusAnalyzer.java @@ -31,8 +31,11 @@ import org.owasp.dependencycheck.data.nexus.MavenArtifact; import org.owasp.dependencycheck.data.nexus.NexusSearch; import org.owasp.dependencycheck.dependency.Confidence; import org.owasp.dependencycheck.dependency.Dependency; +import org.owasp.dependencycheck.utils.InvalidSettingException; import org.owasp.dependencycheck.utils.Settings; +import javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter; + /** * Analyzer which will attempt to locate a dependency on a Nexus service by SHA-1 digest of the dependency. * @@ -48,6 +51,10 @@ import org.owasp.dependencycheck.utils.Settings; * @author colezlaw */ public class NexusAnalyzer extends AbstractFileTypeAnalyzer { + /** + * The default URL - this will be used by the SolrAnalyzer to determine whether to enable this. + */ + public static final String DEFAULT_URL = "https://repository.sonatype.org/service/local/"; /** * The logger. @@ -74,6 +81,33 @@ public class NexusAnalyzer extends AbstractFileTypeAnalyzer { */ private NexusSearch searcher; + /** + * Determine whether to enable this analyzer or not. + * + * @return whether the analyzer should be enabled + */ + @Override + public boolean isEnabled() { + /* Enable this analyzer ONLY if the Nexus URL has been set to something + other than the default one (if it's the default one, we'll use the + central one) and it's enabled by the user. + */ + boolean retval = false; + try { + if ((! DEFAULT_URL.equals(Settings.getString(Settings.KEYS.ANALYZER_NEXUS_URL))) + && Settings.getBoolean(Settings.KEYS.ANALYZER_NEXUS_ENABLED)) { + LOGGER.info("Enabling Nexus analyzer"); + retval = true; + } else { + LOGGER.info("Nexus analyzer disabled"); + } + } catch (InvalidSettingException ise) { + LOGGER.warning("Invalid setting. Disabling Nexus analyzer"); + } + + return retval; + } + /** * Initializes the analyzer once before any analysis is performed. * @@ -150,6 +184,9 @@ public class NexusAnalyzer extends AbstractFileTypeAnalyzer { */ @Override public void analyzeFileType(Dependency dependency, Engine engine) throws AnalysisException { + if (! isEnabled()) { + return; + } try { final MavenArtifact ma = searcher.searchSha1(dependency.getSha1sum()); dependency.addAsEvidence("nexus", ma, Confidence.HIGH); diff --git a/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/central/CentralSearch.java b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/central/CentralSearch.java new file mode 100644 index 000000000..4c55e47a1 --- /dev/null +++ b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/central/CentralSearch.java @@ -0,0 +1,146 @@ +package org.owasp.dependencycheck.data.central; + +import org.owasp.dependencycheck.data.nexus.MavenArtifact; +import org.owasp.dependencycheck.utils.InvalidSettingException; +import org.owasp.dependencycheck.utils.Settings; +import org.owasp.dependencycheck.utils.URLConnectionFactory; +import org.w3c.dom.Document; +import org.w3c.dom.NodeList; + +import javax.xml.parsers.DocumentBuilder; +import javax.xml.parsers.DocumentBuilderFactory; +import javax.xml.xpath.XPath; +import javax.xml.xpath.XPathConstants; +import javax.xml.xpath.XPathFactory; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.net.HttpURLConnection; +import java.net.URL; +import java.util.ArrayList; +import java.util.List; +import java.util.logging.Logger; + +/** + * Class of methods to search Maven Central via Solr. + * + * @author colezlaw + */ +public class CentralSearch { + /** + * The URL for the Solr service + */ + private final URL rootURL; + + /** + * Whether to use the Proxy when making requests + */ + private boolean useProxy; + + /** + * Used for logging. + */ + private static final Logger LOGGER = Logger.getLogger(CentralSearch.class.getName()); + + /** + * Determines whether we'll continue using the analyzer. If there's some sort + * of HTTP failure, we'll disable the analyzer. + */ + private boolean isEnabled = true; + + /** + * Creates a NexusSearch for the given repository URL. + * + * @param rootURL the URL of the repository on which searches should execute. + * Only parameters are added to this (so it should end in /select) + */ + public CentralSearch(URL rootURL) { + this.rootURL = rootURL; + try { + if (null != Settings.getString(Settings.KEYS.PROXY_SERVER) + && Settings.getBoolean(Settings.KEYS.ANALYZER_SOLR_PROXY)) { + useProxy = true; + LOGGER.fine("Using proxy"); + } else { + useProxy = false; + LOGGER.fine("Not using proxy"); + } + } catch (InvalidSettingException ise) { + useProxy = false; + } + } + + /** + * Searches the configured Solr URL for the given sha1 hash. If the artifact is found, a + * MavenArtifact is populated with the GAV. + * + * @param sha1 the SHA-1 hash string for which to search + * @return the populated Maven GAV. + * @throws IOException if it's unable to connect to the specified repository or if + * the specified artifact is not found. + */ + public List searchSha1(String sha1) throws IOException { + if (null == sha1 || !sha1.matches("^[0-9A-Fa-f]{40}$")) { + throw new IllegalArgumentException("Invalid SHA1 format"); + } + + final URL url = new URL(rootURL + String.format("?q=1:\"%s\"&wt=xml", sha1)); + + LOGGER.info(String.format("Searching Solr url %s", url.toString())); + + // Determine if we need to use a proxy. The rules: + // 1) If the proxy is set, AND the setting is set to true, use the proxy + // 2) Otherwise, don't use the proxy (either the proxy isn't configured, + // or proxy is specifically set to false) + final HttpURLConnection conn = URLConnectionFactory.createHttpURLConnection(url, useProxy); + + conn.setDoOutput(true); + + // JSON would be more elegant, but there's not currently a dependency + // on JSON, so don't want to add one just for this + conn.addRequestProperty("Accept", "application/xml"); + conn.connect(); + + if (conn.getResponseCode() == 200) { + boolean missing = false; + try { + final DocumentBuilder builder = DocumentBuilderFactory + .newInstance().newDocumentBuilder(); + final Document doc = builder.parse(conn.getInputStream()); + final XPath xpath = XPathFactory.newInstance().newXPath(); + final String numFound = xpath.evaluate("/response/result/@numFound", doc); + if ("0".equals(numFound)) { + missing = true; + } else { + ArrayList result = new ArrayList(); + NodeList docs = (NodeList)xpath.evaluate("/response/result/doc", doc, XPathConstants.NODESET); + for (int i = 0; i < docs.getLength(); i++) { + final String g = xpath.evaluate("./str[@name='g']", docs.item(i)); + LOGGER.finest(String.format("GroupId: %s", g)); + final String a = xpath.evaluate("./str[@name='a']", docs.item(i)); + LOGGER.finest(String.format("ArtifactId: %s", a)); + final String v = xpath.evaluate("./str[@name='v']", docs.item(i)); + LOGGER.finest(String.format("Version: %s", v)); + result.add(new MavenArtifact(g, a, v, url.toString())); + } + + return result; + } + } catch (Throwable e) { + // Anything else is jacked up XML stuff that we really can't recover + // from well + throw new IOException(e.getMessage(), e); + } + + if (missing) { + throw new FileNotFoundException("Artifact not found in Solr"); + } + } else { + final String msg = String.format("Could not connect to Solr received response code: %d %s", + conn.getResponseCode(), conn.getResponseMessage()); + LOGGER.fine(msg); + throw new IOException(msg); + } + + return null; + } +} diff --git a/dependency-check-core/src/main/java/org/owasp/dependencycheck/dependency/Dependency.java b/dependency-check-core/src/main/java/org/owasp/dependencycheck/dependency/Dependency.java index ef1117148..2f72b3d84 100644 --- a/dependency-check-core/src/main/java/org/owasp/dependencycheck/dependency/Dependency.java +++ b/dependency-check-core/src/main/java/org/owasp/dependencycheck/dependency/Dependency.java @@ -341,10 +341,12 @@ public class Dependency implements Serializable, Comparable { found = true; i.setConfidence(Confidence.HIGHEST); i.setUrl(mavenArtifact.getArtifactUrl()); + LOGGER.fine(String.format("Already found identifier %s. Confidence set to highest", i.getValue())); break; } } if (!found) { + LOGGER.fine(String.format("Adding new maven identifier %s", mavenArtifact.toString())); this.addIdentifier("maven", mavenArtifact.toString(), mavenArtifact.getArtifactUrl(), Confidence.HIGHEST); } } diff --git a/dependency-check-core/src/main/resources/META-INF/services/org.owasp.dependencycheck.analyzer.Analyzer b/dependency-check-core/src/main/resources/META-INF/services/org.owasp.dependencycheck.analyzer.Analyzer index d752f4294..f2c4509c0 100644 --- a/dependency-check-core/src/main/resources/META-INF/services/org.owasp.dependencycheck.analyzer.Analyzer +++ b/dependency-check-core/src/main/resources/META-INF/services/org.owasp.dependencycheck.analyzer.Analyzer @@ -8,6 +8,7 @@ org.owasp.dependencycheck.analyzer.CpeSuppressionAnalyzer org.owasp.dependencycheck.analyzer.DependencyBundlingAnalyzer org.owasp.dependencycheck.analyzer.NvdCveAnalyzer org.owasp.dependencycheck.analyzer.VulnerabilitySuppressionAnalyzer +org.owasp.dependencycheck.analyzer.CentralAnalyzer org.owasp.dependencycheck.analyzer.NexusAnalyzer org.owasp.dependencycheck.analyzer.NuspecAnalyzer org.owasp.dependencycheck.analyzer.AssemblyAnalyzer \ No newline at end of file diff --git a/dependency-check-core/src/main/resources/dependencycheck.properties b/dependency-check-core/src/main/resources/dependencycheck.properties index 0090e0c50..65d748ed0 100644 --- a/dependency-check-core/src/main/resources/dependencycheck.properties +++ b/dependency-check-core/src/main/resources/dependencycheck.properties @@ -58,3 +58,10 @@ analyzer.nexus.url=https://repository.sonatype.org/service/local/ # If set to true, the proxy will still ONLY be used if the proxy properties (proxy.url, proxy.port) # are configured analyzer.nexus.proxy=true + +# the URL for searching search.maven.org for SHA-1 and whether it's enabled +analyzer.solr.enabled=true +analyzer.solr.url=http://search.maven.org/solrsearch/select +# If set to true, the proxy will still ONLY be used if the proxy properties (proxy.url, proxy.port) +# are configured +analyzer.solr.proxy=true \ No newline at end of file diff --git a/dependency-check-core/src/test/java/org/owasp/dependencycheck/data/central/CentralSearchTest.java b/dependency-check-core/src/test/java/org/owasp/dependencycheck/data/central/CentralSearchTest.java new file mode 100644 index 000000000..ffcde235e --- /dev/null +++ b/dependency-check-core/src/test/java/org/owasp/dependencycheck/data/central/CentralSearchTest.java @@ -0,0 +1,63 @@ +package org.owasp.dependencycheck.data.central; + +import org.junit.Before; +import org.junit.Test; +import org.owasp.dependencycheck.BaseTest; +import org.owasp.dependencycheck.data.nexus.MavenArtifact; +import org.owasp.dependencycheck.utils.Settings; + +import java.io.FileNotFoundException; +import java.net.URL; +import java.util.List; +import java.util.logging.Logger; + +import static org.junit.Assert.*; + +/** + * Created by colezlaw on 10/13/14. + */ +public class CentralSearchTest extends BaseTest { + private static final Logger LOGGER = Logger.getLogger(CentralSearchTest.class.getName()); + private CentralSearch searcher; + + @Before + public void setUp() throws Exception { + String solrUrl = Settings.getString(Settings.KEYS.ANALYZER_SOLR_URL); + LOGGER.fine(solrUrl); + searcher = new CentralSearch(new URL(solrUrl)); + } + + @Test(expected = IllegalArgumentException.class) + public void testNullSha1() throws Exception { searcher.searchSha1(null); } + + @Test(expected = IllegalArgumentException.class) + public void testMalformedSha1() throws Exception { + searcher.searchSha1("invalid"); + } + + // This test does generate network traffic and communicates with a host + // you may not be able to reach. Remove the @Ignore annotation if you want to + // test it anyway + @Test + public void testValidSha1() throws Exception { + List ma = searcher.searchSha1("9977a8d04e75609cf01badc4eb6a9c7198c4c5ea"); + assertEquals("Incorrect group", "org.apache.maven.plugins", ma.get(0).getGroupId()); + assertEquals("Incorrect artifact", "maven-compiler-plugin", ma.get(0).getArtifactId()); + assertEquals("Incorrect version", "3.1", ma.get(0).getVersion()); + } + + // This test does generate network traffic and communicates with a host + // you may not be able to reach. Remove the @Ignore annotation if you want to + // test it anyway + @Test(expected = FileNotFoundException.class) + public void testMissingSha1() throws Exception { + searcher.searchSha1("AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"); + } + + // This test should give us multiple results back from Solr + @Test + public void testMultipleReturns() throws Exception { + List ma = searcher.searchSha1("94A9CE681A42D0352B3AD22659F67835E560D107"); + assertTrue(ma.size() > 1); + } +} diff --git a/dependency-check-core/src/test/resources/dependencycheck.properties b/dependency-check-core/src/test/resources/dependencycheck.properties index 61efb2407..07d11d566 100644 --- a/dependency-check-core/src/test/resources/dependencycheck.properties +++ b/dependency-check-core/src/test/resources/dependencycheck.properties @@ -59,3 +59,9 @@ analyzer.nexus.url=https://repository.sonatype.org/service/local/ # If set to true, the proxy will still ONLY be used if the proxy properties (proxy.url, proxy.port) # are configured analyzer.nexus.proxy=true + +# the URL for searching search.maven.org for SHA-1 and whether it's enabled +analyzer.solr.enabled=true +analyzer.solr.url=http://search.maven.org/solrsearch/select +# If set to true, the proxy will still ONLY be used if the proxy properties (proxy.url, proxy.port) +# are configured \ No newline at end of file diff --git a/dependency-check-utils/src/main/java/org/owasp/dependencycheck/utils/Settings.java b/dependency-check-utils/src/main/java/org/owasp/dependencycheck/utils/Settings.java index 7ff76b16a..5306e3073 100644 --- a/dependency-check-utils/src/main/java/org/owasp/dependencycheck/utils/Settings.java +++ b/dependency-check-utils/src/main/java/org/owasp/dependencycheck/utils/Settings.java @@ -188,6 +188,18 @@ public final class Settings { * The properties key for using the proxy to reach Nexus. */ public static final String ANALYZER_NEXUS_PROXY = "analyzer.nexus.proxy"; + /** + * The properties key for whether the Solr analyzer is enabled. + */ + public static final String ANALYZER_SOLR_ENABLED = "analyzer.solr.enabled"; + /** + * The properties key for the Solr search URL. + */ + public static final String ANALYZER_SOLR_URL = "analyzer.solr.url"; + /** + * The properties key for using the proxy to reach Solr. + */ + public static final String ANALYZER_SOLR_PROXY = "analyzer.solr.proxy"; /** * The path to mono, if available. */