From 5f38741831e34c6a4008f89dca10f04fa51b11c1 Mon Sep 17 00:00:00 2001 From: Will Stranathan Date: Thu, 16 Oct 2014 15:04:50 -0400 Subject: [PATCH] Initial checkin of work on a Solr analyzer which will replace Nexus on Internet checks Former-commit-id: 7b51d0cb1d23122bc73261424b66df24f72370cd --- .../analyzer/NexusAnalyzer.java | 39 +++- .../analyzer/SolrAnalyzer.java | 170 ++++++++++++++++++ .../dependencycheck/data/solr/SolrSearch.java | 136 ++++++++++++++ ...rg.owasp.dependencycheck.analyzer.Analyzer | 1 + .../main/resources/dependencycheck.properties | 7 + .../data/solr/SolrSearchTest.java | 57 ++++++ .../test/resources/dependencycheck.properties | 6 + .../owasp/dependencycheck/utils/Settings.java | 12 ++ 8 files changed, 427 insertions(+), 1 deletion(-) create mode 100644 dependency-check-core/src/main/java/org/owasp/dependencycheck/analyzer/SolrAnalyzer.java create mode 100644 dependency-check-core/src/main/java/org/owasp/dependencycheck/data/solr/SolrSearch.java create mode 100644 dependency-check-core/src/test/java/org/owasp/dependencycheck/data/solr/SolrSearchTest.java diff --git a/dependency-check-core/src/main/java/org/owasp/dependencycheck/analyzer/NexusAnalyzer.java b/dependency-check-core/src/main/java/org/owasp/dependencycheck/analyzer/NexusAnalyzer.java index dbee6a5d7..d5acb8e66 100644 --- a/dependency-check-core/src/main/java/org/owasp/dependencycheck/analyzer/NexusAnalyzer.java +++ b/dependency-check-core/src/main/java/org/owasp/dependencycheck/analyzer/NexusAnalyzer.java @@ -31,8 +31,11 @@ import org.owasp.dependencycheck.data.nexus.NexusSearch; import org.owasp.dependencycheck.dependency.Confidence; import org.owasp.dependencycheck.dependency.Dependency; import org.owasp.dependencycheck.dependency.Identifier; +import org.owasp.dependencycheck.utils.InvalidSettingException; import org.owasp.dependencycheck.utils.Settings; +import javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter; + /** * Analyzer which will attempt to locate a dependency on a Nexus service by SHA-1 digest of the dependency. * @@ -48,6 +51,10 @@ import org.owasp.dependencycheck.utils.Settings; * @author colezlaw */ public class NexusAnalyzer extends AbstractFileTypeAnalyzer { + /** + * The default URL - this will be used by the SolrAnalyzer to determine whether to enable this. + */ + public static final String DEFAULT_URL = "https://repository.sonatype.org/service/local/"; /** * The logger. @@ -74,6 +81,33 @@ public class NexusAnalyzer extends AbstractFileTypeAnalyzer { */ private NexusSearch searcher; + /** + * Determine whether to enable this analyzer or not. + * + * @return whether the analyzer should be enabled + */ + @Override + public boolean isEnabled() { + /* Enable this analyzer ONLY if the Nexus URL has been set to something + other than the default one (if it's the default one, we'll use the + solr one) and it's enabled by the user. + */ + boolean retval = false; + try { + if ((! DEFAULT_URL.equals(Settings.getString(Settings.KEYS.ANALYZER_NEXUS_URL))) + && Settings.getBoolean(Settings.KEYS.ANALYZER_NEXUS_ENABLED)) { + LOGGER.info("Enabling Nexus analyzer"); + retval = true; + } else { + LOGGER.info("Nexus analyzer disabled"); + } + } catch (InvalidSettingException ise) { + LOGGER.warning("Invalid setting. Disabling Nexus analyzer"); + } + + return retval; + } + /** * Initializes the analyzer once before any analysis is performed. * @@ -150,6 +184,9 @@ public class NexusAnalyzer extends AbstractFileTypeAnalyzer { */ @Override public void analyzeFileType(Dependency dependency, Engine engine) throws AnalysisException { + if (! isEnabled()) { + return; + } try { final MavenArtifact ma = searcher.searchSha1(dependency.getSha1sum()); if (ma.getGroupId() != null && !"".equals(ma.getGroupId())) { @@ -166,7 +203,7 @@ public class NexusAnalyzer extends AbstractFileTypeAnalyzer { for (Identifier i : dependency.getIdentifiers()) { if ("maven".equals(i.getType()) && i.getValue().equals(ma.toString())) { found = true; - i.setConfidence(Confidence.HIGHEST); + i.setConfidence(Confidence .HIGHEST); i.setUrl(ma.getArtifactUrl()); break; } diff --git a/dependency-check-core/src/main/java/org/owasp/dependencycheck/analyzer/SolrAnalyzer.java b/dependency-check-core/src/main/java/org/owasp/dependencycheck/analyzer/SolrAnalyzer.java new file mode 100644 index 000000000..82aee8d66 --- /dev/null +++ b/dependency-check-core/src/main/java/org/owasp/dependencycheck/analyzer/SolrAnalyzer.java @@ -0,0 +1,170 @@ +package org.owasp.dependencycheck.analyzer; + +import org.owasp.dependencycheck.Engine; +import org.owasp.dependencycheck.analyzer.exception.AnalysisException; +import org.owasp.dependencycheck.data.nexus.MavenArtifact; +import org.owasp.dependencycheck.data.solr.SolrSearch; +import org.owasp.dependencycheck.dependency.Confidence; +import org.owasp.dependencycheck.dependency.Dependency; +import org.owasp.dependencycheck.utils.InvalidSettingException; +import org.owasp.dependencycheck.utils.Settings; + +import java.io.FileNotFoundException; +import java.io.IOException; +import java.net.URL; +import java.util.Set; +import java.util.logging.Level; +import java.util.logging.Logger; + +/** + * Created by colezlaw on 10/9/14. + */ +public class SolrAnalyzer extends AbstractFileTypeAnalyzer { + /** + * The logger. + */ + private static final Logger LOGGER = Logger.getLogger(SolrAnalyzer.class.getName()); + + /** + * The name of the analyzer. + */ + private static final String ANALYZER_NAME = "Solr Analyzer"; + + /** + * The phase in which this analyzer runs. + */ + private static final AnalysisPhase ANALYSIS_PHASE = AnalysisPhase.INFORMATION_COLLECTION; + + /** + * The types of files on which this will work. + */ + private static final Set SUPPORTED_EXTENSIONS = newHashSet("jar"); + + /** + * The analyzer should be disabled if there are errors, so this is a flag + * to determine if such an error has occurred. + */ + protected boolean errorFlag = false; + + /** + * The searcher itself. + */ + private SolrSearch searcher; + + /** + * Determine whether to enable this analyzer or not. + * + * @return whether the analyzer should be enabled + */ + @Override + public boolean isEnabled() { + boolean retval = false; + + try { + if (Settings.getBoolean(Settings.KEYS.ANALYZER_SOLR_ENABLED)) { + if (!Settings.getBoolean(Settings.KEYS.ANALYZER_NEXUS_ENABLED) + || NexusAnalyzer.DEFAULT_URL.equals(Settings.getString(Settings.KEYS.ANALYZER_NEXUS_URL))) { + LOGGER.info("Enabling the Solr analyzer"); + retval = true; + } else { + LOGGER.info("Nexus analyzer is enabled, disabling Solr"); + } + } else { + LOGGER.info("Solr analyzer disabled"); + } + } catch (InvalidSettingException ise) { + LOGGER.warning("Invalid setting. Disabling the Solr analyzer"); + } + + return retval; + } + + /** + * Initializes the analyzer once before any analysis is performed. + * + * @throws Exception if there's an error during initalization + */ + @Override + public void initializeFileTypeAnalyzer() throws Exception { + LOGGER.fine("Initializing Solr analyzer"); + LOGGER.fine(String.format("Solr analyzer enabled: %s", isEnabled())); + if (isEnabled()) { + final String searchUrl = Settings.getString(Settings.KEYS.ANALYZER_SOLR_URL); + LOGGER.fine(String.format("Solr Analyzer URL: %s", searchUrl)); + searcher = new SolrSearch(new URL(searchUrl)); + } + } + + /** + * Returns the analyzer's name. + * + * @return the name of the analyzer + */ + @Override + public String getName() { + return ANALYZER_NAME; + } + + /** Returns the key used in the properties file to to reference the analyzer's enabled property. + * + * @return the analyzer's enabled property setting key. + */ + @Override + protected String getAnalyzerEnabledSettingKey() { + return Settings.KEYS.ANALYZER_SOLR_ENABLED; + } + + /** + * Returns the analysis phase under which the analyzer runs. + * + * @return the phase under which the analyzer runs + */ + @Override + public AnalysisPhase getAnalysisPhase() { + return ANALYSIS_PHASE; + } + + /** + * Returns the extensions for which this Analyzer runs. + * + * @return the extensions for which this Analyzer runs + */ + @Override + public Set getSupportedExtensions() { + return SUPPORTED_EXTENSIONS; + } + + /** + * Performs the analysis. + * + * @param dependency the dependency to analyze + * @param engine the engine + * @throws AnalysisException when there's an exception during analysis + */ + @Override + public void analyzeFileType(Dependency dependency, Engine engine) throws AnalysisException { + if (errorFlag) { + return; + } + + try { + final MavenArtifact ma = searcher.searchSha1(dependency.getSha1sum()); + if (ma.getGroupId() != null && !"".equals(ma.getGroupId())) { + dependency.getVendorEvidence().addEvidence("solr", "groupid", ma.getGroupId(), Confidence.HIGH); + } + if (ma.getArtifactId() != null && !"".equals(ma.getArtifactId())) { + dependency.getProductEvidence().addEvidence("solr", "artifactid", ma.getArtifactId(), Confidence.HIGH); + } + if (ma.getVersion() != null && !"".equals(ma.getVersion())) { + dependency.getVersionEvidence().addEvidence("solr", "version", ma.getVersion(), Confidence.HIGH); + } + } catch (IllegalArgumentException iae) { + LOGGER.info(String.format("invalid sha1-hash on %s", dependency.getFileName())); + } catch (FileNotFoundException fnfe) { + LOGGER.fine(String.format("Artifact not found in repository: '%s", dependency.getFileName())); + } catch (IOException ioe) { + LOGGER.log(Level.FINE, "Could not connect to Solr search", ioe); + errorFlag = true; + } + } +} diff --git a/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/solr/SolrSearch.java b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/solr/SolrSearch.java new file mode 100644 index 000000000..fdf1758b0 --- /dev/null +++ b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/solr/SolrSearch.java @@ -0,0 +1,136 @@ +package org.owasp.dependencycheck.data.solr; + +import org.owasp.dependencycheck.data.nexus.MavenArtifact; +import org.owasp.dependencycheck.utils.InvalidSettingException; +import org.owasp.dependencycheck.utils.Settings; +import org.owasp.dependencycheck.utils.URLConnectionFactory; +import org.w3c.dom.Document; + +import javax.xml.parsers.DocumentBuilder; +import javax.xml.parsers.DocumentBuilderFactory; +import javax.xml.xpath.XPath; +import javax.xml.xpath.XPathFactory; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.net.HttpURLConnection; +import java.net.URL; +import java.util.logging.Logger; + +/** + * Class of methods to search Maven Central via Solr. + * + * @author colezlaw + */ +public class SolrSearch { + /** + * The URL for the Solr service + */ + private final URL rootURL; + + /** + * Whether to use the Proxy when making requests + */ + private boolean useProxy; + + /** + * Used for logging. + */ + private static final Logger LOGGER = Logger.getLogger(SolrSearch.class.getName()); + + /** + * Determines whether we'll continue using the analyzer. If there's some sort + * of HTTP failure, we'll disable the analyzer. + */ + private boolean isEnabled = true; + + /** + * Creates a NexusSearch for the given repository URL. + * + * @param rootURL the URL of the repository on which searches should execute. + * Only parameters are added to this (so it should end in /select) + */ + public SolrSearch(URL rootURL) { + this.rootURL = rootURL; + try { + if (null != Settings.getString(Settings.KEYS.PROXY_SERVER) + && Settings.getBoolean(Settings.KEYS.ANALYZER_SOLR_PROXY)) { + useProxy = true; + LOGGER.fine("Using proxy"); + } else { + useProxy = false; + LOGGER.fine("Not using proxy"); + } + } catch (InvalidSettingException ise) { + useProxy = false; + } + } + + /** + * Searches the configured Solr URL for the given sha1 hash. If the artifact is found, a + * MavenArtifact is populated with the GAV. + * + * @param sha1 the SHA-1 hash string for which to search + * @return the populated Maven GAV. + * @throws IOException if it's unable to connect to the specified repository or if + * the specified artifact is not found. + */ + public MavenArtifact searchSha1(String sha1) throws IOException { + if (null == sha1 || !sha1.matches("^[0-9A-Fa-f]{40}$")) { + throw new IllegalArgumentException("Invalid SHA1 format"); + } + + final URL url = new URL(rootURL + String.format("?q=1:\"%s\"&wt=xml", sha1)); + + LOGGER.info(String.format("Searching Solr url %s", url.toString())); + + // Determine if we need to use a proxy. The rules: + // 1) If the proxy is set, AND the setting is set to true, use the proxy + // 2) Otherwise, don't use the proxy (either the proxy isn't configured, + // or proxy is specifically set to false) + final HttpURLConnection conn = URLConnectionFactory.createHttpURLConnection(url, useProxy); + + conn.setDoOutput(true); + + // JSON would be more elegant, but there's not currently a dependency + // on JSON, so don't want to add one just for this + conn.addRequestProperty("Accept", "application/xml"); + conn.connect(); + + if (conn.getResponseCode() == 200) { + boolean missing = false; + try { + final DocumentBuilder builder = DocumentBuilderFactory + .newInstance().newDocumentBuilder(); + final Document doc = builder.parse(conn.getInputStream()); + final XPath xpath = XPathFactory.newInstance().newXPath(); + final String numFound = xpath.evaluate("/response/result/@numFound", doc); + if ("0".equals(numFound)) { + missing = true; + } else { + final String g = xpath.evaluate("/response/result/doc[1]/str[@name='g']", doc); + LOGGER.finest(String.format("GroupId: %s", g)); + final String a = xpath.evaluate("/response/result/doc[1]/str[@name='a']", doc); + LOGGER.finest(String.format("ArtifactId: %s", a)); + final String v = xpath.evaluate("/response/result/doc[1]/str[@name='v']", doc); + LOGGER.finest(String.format("Version: %s", v)); + return new MavenArtifact(g, a, v); + } + } catch (Throwable e) { + // Anything else is jacked up XML stuff that we really can't recover + // from well + throw new IOException(e.getMessage(), e); + } + + if (missing) { + throw new FileNotFoundException("Artifact not found in Solr"); + } + } else { + final String msg = String.format("Could not connect to Solr received response code: %d %s", + conn.getResponseCode(), conn.getResponseMessage()); + LOGGER.fine(msg); + throw new IOException(msg); + } + + return null; + } +} diff --git a/dependency-check-core/src/main/resources/META-INF/services/org.owasp.dependencycheck.analyzer.Analyzer b/dependency-check-core/src/main/resources/META-INF/services/org.owasp.dependencycheck.analyzer.Analyzer index d752f4294..f98b34e8a 100644 --- a/dependency-check-core/src/main/resources/META-INF/services/org.owasp.dependencycheck.analyzer.Analyzer +++ b/dependency-check-core/src/main/resources/META-INF/services/org.owasp.dependencycheck.analyzer.Analyzer @@ -8,6 +8,7 @@ org.owasp.dependencycheck.analyzer.CpeSuppressionAnalyzer org.owasp.dependencycheck.analyzer.DependencyBundlingAnalyzer org.owasp.dependencycheck.analyzer.NvdCveAnalyzer org.owasp.dependencycheck.analyzer.VulnerabilitySuppressionAnalyzer +org.owasp.dependencycheck.analyzer.SolrAnalyzer org.owasp.dependencycheck.analyzer.NexusAnalyzer org.owasp.dependencycheck.analyzer.NuspecAnalyzer org.owasp.dependencycheck.analyzer.AssemblyAnalyzer \ No newline at end of file diff --git a/dependency-check-core/src/main/resources/dependencycheck.properties b/dependency-check-core/src/main/resources/dependencycheck.properties index 0090e0c50..65d748ed0 100644 --- a/dependency-check-core/src/main/resources/dependencycheck.properties +++ b/dependency-check-core/src/main/resources/dependencycheck.properties @@ -58,3 +58,10 @@ analyzer.nexus.url=https://repository.sonatype.org/service/local/ # If set to true, the proxy will still ONLY be used if the proxy properties (proxy.url, proxy.port) # are configured analyzer.nexus.proxy=true + +# the URL for searching search.maven.org for SHA-1 and whether it's enabled +analyzer.solr.enabled=true +analyzer.solr.url=http://search.maven.org/solrsearch/select +# If set to true, the proxy will still ONLY be used if the proxy properties (proxy.url, proxy.port) +# are configured +analyzer.solr.proxy=true \ No newline at end of file diff --git a/dependency-check-core/src/test/java/org/owasp/dependencycheck/data/solr/SolrSearchTest.java b/dependency-check-core/src/test/java/org/owasp/dependencycheck/data/solr/SolrSearchTest.java new file mode 100644 index 000000000..937a6d849 --- /dev/null +++ b/dependency-check-core/src/test/java/org/owasp/dependencycheck/data/solr/SolrSearchTest.java @@ -0,0 +1,57 @@ +package org.owasp.dependencycheck.data.solr; + +import org.junit.Assume; +import org.junit.Before; +import org.junit.Test; +import org.owasp.dependencycheck.BaseTest; +import org.owasp.dependencycheck.data.nexus.MavenArtifact; +import org.owasp.dependencycheck.utils.Settings; + +import java.io.FileNotFoundException; +import java.net.URL; +import java.util.logging.Logger; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; + +/** + * Created by colezlaw on 10/13/14. + */ +public class SolrSearchTest extends BaseTest { + private static final Logger LOGGER = Logger.getLogger(SolrSearchTest.class.getName()); + private SolrSearch searcher; + + @Before + public void setUp() throws Exception { + String solrUrl = Settings.getString(Settings.KEYS.ANALYZER_SOLR_URL); + LOGGER.fine(solrUrl); + searcher = new SolrSearch(new URL(solrUrl)); + } + + @Test(expected = IllegalArgumentException.class) + public void testNullSha1() throws Exception { searcher.searchSha1(null); } + + @Test(expected = IllegalArgumentException.class) + public void testMalformedSha1() throws Exception { + searcher.searchSha1("invalid"); + } + + // This test does generate network traffic and communicates with a host + // you may not be able to reach. Remove the @Ignore annotation if you want to + // test it anyway + @Test + public void testValidSha1() throws Exception { + MavenArtifact ma = searcher.searchSha1("9977a8d04e75609cf01badc4eb6a9c7198c4c5ea"); + assertEquals("Incorrect group", "org.apache.maven.plugins", ma.getGroupId()); + assertEquals("Incorrect artifact", "maven-compiler-plugin", ma.getArtifactId()); + assertEquals("Incorrect version", "3.1", ma.getVersion()); + } + + // This test does generate network traffic and communicates with a host + // you may not be able to reach. Remove the @Ignore annotation if you want to + // test it anyway + @Test(expected = FileNotFoundException.class) + public void testMissingSha1() throws Exception { + searcher.searchSha1("AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"); + } +} diff --git a/dependency-check-core/src/test/resources/dependencycheck.properties b/dependency-check-core/src/test/resources/dependencycheck.properties index 61efb2407..07d11d566 100644 --- a/dependency-check-core/src/test/resources/dependencycheck.properties +++ b/dependency-check-core/src/test/resources/dependencycheck.properties @@ -59,3 +59,9 @@ analyzer.nexus.url=https://repository.sonatype.org/service/local/ # If set to true, the proxy will still ONLY be used if the proxy properties (proxy.url, proxy.port) # are configured analyzer.nexus.proxy=true + +# the URL for searching search.maven.org for SHA-1 and whether it's enabled +analyzer.solr.enabled=true +analyzer.solr.url=http://search.maven.org/solrsearch/select +# If set to true, the proxy will still ONLY be used if the proxy properties (proxy.url, proxy.port) +# are configured \ No newline at end of file diff --git a/dependency-check-utils/src/main/java/org/owasp/dependencycheck/utils/Settings.java b/dependency-check-utils/src/main/java/org/owasp/dependencycheck/utils/Settings.java index 7ff76b16a..5306e3073 100644 --- a/dependency-check-utils/src/main/java/org/owasp/dependencycheck/utils/Settings.java +++ b/dependency-check-utils/src/main/java/org/owasp/dependencycheck/utils/Settings.java @@ -188,6 +188,18 @@ public final class Settings { * The properties key for using the proxy to reach Nexus. */ public static final String ANALYZER_NEXUS_PROXY = "analyzer.nexus.proxy"; + /** + * The properties key for whether the Solr analyzer is enabled. + */ + public static final String ANALYZER_SOLR_ENABLED = "analyzer.solr.enabled"; + /** + * The properties key for the Solr search URL. + */ + public static final String ANALYZER_SOLR_URL = "analyzer.solr.url"; + /** + * The properties key for using the proxy to reach Solr. + */ + public static final String ANALYZER_SOLR_PROXY = "analyzer.solr.proxy"; /** * The path to mono, if available. */