From 5f38741831e34c6a4008f89dca10f04fa51b11c1 Mon Sep 17 00:00:00 2001 From: Will Stranathan Date: Thu, 16 Oct 2014 15:04:50 -0400 Subject: [PATCH 1/3] Initial checkin of work on a Solr analyzer which will replace Nexus on Internet checks Former-commit-id: 7b51d0cb1d23122bc73261424b66df24f72370cd --- .../analyzer/NexusAnalyzer.java | 39 +++- .../analyzer/SolrAnalyzer.java | 170 ++++++++++++++++++ .../dependencycheck/data/solr/SolrSearch.java | 136 ++++++++++++++ ...rg.owasp.dependencycheck.analyzer.Analyzer | 1 + .../main/resources/dependencycheck.properties | 7 + .../data/solr/SolrSearchTest.java | 57 ++++++ .../test/resources/dependencycheck.properties | 6 + .../owasp/dependencycheck/utils/Settings.java | 12 ++ 8 files changed, 427 insertions(+), 1 deletion(-) create mode 100644 dependency-check-core/src/main/java/org/owasp/dependencycheck/analyzer/SolrAnalyzer.java create mode 100644 dependency-check-core/src/main/java/org/owasp/dependencycheck/data/solr/SolrSearch.java create mode 100644 dependency-check-core/src/test/java/org/owasp/dependencycheck/data/solr/SolrSearchTest.java diff --git a/dependency-check-core/src/main/java/org/owasp/dependencycheck/analyzer/NexusAnalyzer.java b/dependency-check-core/src/main/java/org/owasp/dependencycheck/analyzer/NexusAnalyzer.java index dbee6a5d7..d5acb8e66 100644 --- a/dependency-check-core/src/main/java/org/owasp/dependencycheck/analyzer/NexusAnalyzer.java +++ b/dependency-check-core/src/main/java/org/owasp/dependencycheck/analyzer/NexusAnalyzer.java @@ -31,8 +31,11 @@ import org.owasp.dependencycheck.data.nexus.NexusSearch; import org.owasp.dependencycheck.dependency.Confidence; import org.owasp.dependencycheck.dependency.Dependency; import org.owasp.dependencycheck.dependency.Identifier; +import org.owasp.dependencycheck.utils.InvalidSettingException; import org.owasp.dependencycheck.utils.Settings; +import javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter; + /** * Analyzer which will attempt to locate a dependency on a Nexus service by SHA-1 digest of the dependency. * @@ -48,6 +51,10 @@ import org.owasp.dependencycheck.utils.Settings; * @author colezlaw */ public class NexusAnalyzer extends AbstractFileTypeAnalyzer { + /** + * The default URL - this will be used by the SolrAnalyzer to determine whether to enable this. + */ + public static final String DEFAULT_URL = "https://repository.sonatype.org/service/local/"; /** * The logger. @@ -74,6 +81,33 @@ public class NexusAnalyzer extends AbstractFileTypeAnalyzer { */ private NexusSearch searcher; + /** + * Determine whether to enable this analyzer or not. + * + * @return whether the analyzer should be enabled + */ + @Override + public boolean isEnabled() { + /* Enable this analyzer ONLY if the Nexus URL has been set to something + other than the default one (if it's the default one, we'll use the + solr one) and it's enabled by the user. + */ + boolean retval = false; + try { + if ((! DEFAULT_URL.equals(Settings.getString(Settings.KEYS.ANALYZER_NEXUS_URL))) + && Settings.getBoolean(Settings.KEYS.ANALYZER_NEXUS_ENABLED)) { + LOGGER.info("Enabling Nexus analyzer"); + retval = true; + } else { + LOGGER.info("Nexus analyzer disabled"); + } + } catch (InvalidSettingException ise) { + LOGGER.warning("Invalid setting. Disabling Nexus analyzer"); + } + + return retval; + } + /** * Initializes the analyzer once before any analysis is performed. * @@ -150,6 +184,9 @@ public class NexusAnalyzer extends AbstractFileTypeAnalyzer { */ @Override public void analyzeFileType(Dependency dependency, Engine engine) throws AnalysisException { + if (! isEnabled()) { + return; + } try { final MavenArtifact ma = searcher.searchSha1(dependency.getSha1sum()); if (ma.getGroupId() != null && !"".equals(ma.getGroupId())) { @@ -166,7 +203,7 @@ public class NexusAnalyzer extends AbstractFileTypeAnalyzer { for (Identifier i : dependency.getIdentifiers()) { if ("maven".equals(i.getType()) && i.getValue().equals(ma.toString())) { found = true; - i.setConfidence(Confidence.HIGHEST); + i.setConfidence(Confidence .HIGHEST); i.setUrl(ma.getArtifactUrl()); break; } diff --git a/dependency-check-core/src/main/java/org/owasp/dependencycheck/analyzer/SolrAnalyzer.java b/dependency-check-core/src/main/java/org/owasp/dependencycheck/analyzer/SolrAnalyzer.java new file mode 100644 index 000000000..82aee8d66 --- /dev/null +++ b/dependency-check-core/src/main/java/org/owasp/dependencycheck/analyzer/SolrAnalyzer.java @@ -0,0 +1,170 @@ +package org.owasp.dependencycheck.analyzer; + +import org.owasp.dependencycheck.Engine; +import org.owasp.dependencycheck.analyzer.exception.AnalysisException; +import org.owasp.dependencycheck.data.nexus.MavenArtifact; +import org.owasp.dependencycheck.data.solr.SolrSearch; +import org.owasp.dependencycheck.dependency.Confidence; +import org.owasp.dependencycheck.dependency.Dependency; +import org.owasp.dependencycheck.utils.InvalidSettingException; +import org.owasp.dependencycheck.utils.Settings; + +import java.io.FileNotFoundException; +import java.io.IOException; +import java.net.URL; +import java.util.Set; +import java.util.logging.Level; +import java.util.logging.Logger; + +/** + * Created by colezlaw on 10/9/14. + */ +public class SolrAnalyzer extends AbstractFileTypeAnalyzer { + /** + * The logger. + */ + private static final Logger LOGGER = Logger.getLogger(SolrAnalyzer.class.getName()); + + /** + * The name of the analyzer. + */ + private static final String ANALYZER_NAME = "Solr Analyzer"; + + /** + * The phase in which this analyzer runs. + */ + private static final AnalysisPhase ANALYSIS_PHASE = AnalysisPhase.INFORMATION_COLLECTION; + + /** + * The types of files on which this will work. + */ + private static final Set SUPPORTED_EXTENSIONS = newHashSet("jar"); + + /** + * The analyzer should be disabled if there are errors, so this is a flag + * to determine if such an error has occurred. + */ + protected boolean errorFlag = false; + + /** + * The searcher itself. + */ + private SolrSearch searcher; + + /** + * Determine whether to enable this analyzer or not. + * + * @return whether the analyzer should be enabled + */ + @Override + public boolean isEnabled() { + boolean retval = false; + + try { + if (Settings.getBoolean(Settings.KEYS.ANALYZER_SOLR_ENABLED)) { + if (!Settings.getBoolean(Settings.KEYS.ANALYZER_NEXUS_ENABLED) + || NexusAnalyzer.DEFAULT_URL.equals(Settings.getString(Settings.KEYS.ANALYZER_NEXUS_URL))) { + LOGGER.info("Enabling the Solr analyzer"); + retval = true; + } else { + LOGGER.info("Nexus analyzer is enabled, disabling Solr"); + } + } else { + LOGGER.info("Solr analyzer disabled"); + } + } catch (InvalidSettingException ise) { + LOGGER.warning("Invalid setting. Disabling the Solr analyzer"); + } + + return retval; + } + + /** + * Initializes the analyzer once before any analysis is performed. + * + * @throws Exception if there's an error during initalization + */ + @Override + public void initializeFileTypeAnalyzer() throws Exception { + LOGGER.fine("Initializing Solr analyzer"); + LOGGER.fine(String.format("Solr analyzer enabled: %s", isEnabled())); + if (isEnabled()) { + final String searchUrl = Settings.getString(Settings.KEYS.ANALYZER_SOLR_URL); + LOGGER.fine(String.format("Solr Analyzer URL: %s", searchUrl)); + searcher = new SolrSearch(new URL(searchUrl)); + } + } + + /** + * Returns the analyzer's name. + * + * @return the name of the analyzer + */ + @Override + public String getName() { + return ANALYZER_NAME; + } + + /** Returns the key used in the properties file to to reference the analyzer's enabled property. + * + * @return the analyzer's enabled property setting key. + */ + @Override + protected String getAnalyzerEnabledSettingKey() { + return Settings.KEYS.ANALYZER_SOLR_ENABLED; + } + + /** + * Returns the analysis phase under which the analyzer runs. + * + * @return the phase under which the analyzer runs + */ + @Override + public AnalysisPhase getAnalysisPhase() { + return ANALYSIS_PHASE; + } + + /** + * Returns the extensions for which this Analyzer runs. + * + * @return the extensions for which this Analyzer runs + */ + @Override + public Set getSupportedExtensions() { + return SUPPORTED_EXTENSIONS; + } + + /** + * Performs the analysis. + * + * @param dependency the dependency to analyze + * @param engine the engine + * @throws AnalysisException when there's an exception during analysis + */ + @Override + public void analyzeFileType(Dependency dependency, Engine engine) throws AnalysisException { + if (errorFlag) { + return; + } + + try { + final MavenArtifact ma = searcher.searchSha1(dependency.getSha1sum()); + if (ma.getGroupId() != null && !"".equals(ma.getGroupId())) { + dependency.getVendorEvidence().addEvidence("solr", "groupid", ma.getGroupId(), Confidence.HIGH); + } + if (ma.getArtifactId() != null && !"".equals(ma.getArtifactId())) { + dependency.getProductEvidence().addEvidence("solr", "artifactid", ma.getArtifactId(), Confidence.HIGH); + } + if (ma.getVersion() != null && !"".equals(ma.getVersion())) { + dependency.getVersionEvidence().addEvidence("solr", "version", ma.getVersion(), Confidence.HIGH); + } + } catch (IllegalArgumentException iae) { + LOGGER.info(String.format("invalid sha1-hash on %s", dependency.getFileName())); + } catch (FileNotFoundException fnfe) { + LOGGER.fine(String.format("Artifact not found in repository: '%s", dependency.getFileName())); + } catch (IOException ioe) { + LOGGER.log(Level.FINE, "Could not connect to Solr search", ioe); + errorFlag = true; + } + } +} diff --git a/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/solr/SolrSearch.java b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/solr/SolrSearch.java new file mode 100644 index 000000000..fdf1758b0 --- /dev/null +++ b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/solr/SolrSearch.java @@ -0,0 +1,136 @@ +package org.owasp.dependencycheck.data.solr; + +import org.owasp.dependencycheck.data.nexus.MavenArtifact; +import org.owasp.dependencycheck.utils.InvalidSettingException; +import org.owasp.dependencycheck.utils.Settings; +import org.owasp.dependencycheck.utils.URLConnectionFactory; +import org.w3c.dom.Document; + +import javax.xml.parsers.DocumentBuilder; +import javax.xml.parsers.DocumentBuilderFactory; +import javax.xml.xpath.XPath; +import javax.xml.xpath.XPathFactory; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.net.HttpURLConnection; +import java.net.URL; +import java.util.logging.Logger; + +/** + * Class of methods to search Maven Central via Solr. + * + * @author colezlaw + */ +public class SolrSearch { + /** + * The URL for the Solr service + */ + private final URL rootURL; + + /** + * Whether to use the Proxy when making requests + */ + private boolean useProxy; + + /** + * Used for logging. + */ + private static final Logger LOGGER = Logger.getLogger(SolrSearch.class.getName()); + + /** + * Determines whether we'll continue using the analyzer. If there's some sort + * of HTTP failure, we'll disable the analyzer. + */ + private boolean isEnabled = true; + + /** + * Creates a NexusSearch for the given repository URL. + * + * @param rootURL the URL of the repository on which searches should execute. + * Only parameters are added to this (so it should end in /select) + */ + public SolrSearch(URL rootURL) { + this.rootURL = rootURL; + try { + if (null != Settings.getString(Settings.KEYS.PROXY_SERVER) + && Settings.getBoolean(Settings.KEYS.ANALYZER_SOLR_PROXY)) { + useProxy = true; + LOGGER.fine("Using proxy"); + } else { + useProxy = false; + LOGGER.fine("Not using proxy"); + } + } catch (InvalidSettingException ise) { + useProxy = false; + } + } + + /** + * Searches the configured Solr URL for the given sha1 hash. If the artifact is found, a + * MavenArtifact is populated with the GAV. + * + * @param sha1 the SHA-1 hash string for which to search + * @return the populated Maven GAV. + * @throws IOException if it's unable to connect to the specified repository or if + * the specified artifact is not found. + */ + public MavenArtifact searchSha1(String sha1) throws IOException { + if (null == sha1 || !sha1.matches("^[0-9A-Fa-f]{40}$")) { + throw new IllegalArgumentException("Invalid SHA1 format"); + } + + final URL url = new URL(rootURL + String.format("?q=1:\"%s\"&wt=xml", sha1)); + + LOGGER.info(String.format("Searching Solr url %s", url.toString())); + + // Determine if we need to use a proxy. The rules: + // 1) If the proxy is set, AND the setting is set to true, use the proxy + // 2) Otherwise, don't use the proxy (either the proxy isn't configured, + // or proxy is specifically set to false) + final HttpURLConnection conn = URLConnectionFactory.createHttpURLConnection(url, useProxy); + + conn.setDoOutput(true); + + // JSON would be more elegant, but there's not currently a dependency + // on JSON, so don't want to add one just for this + conn.addRequestProperty("Accept", "application/xml"); + conn.connect(); + + if (conn.getResponseCode() == 200) { + boolean missing = false; + try { + final DocumentBuilder builder = DocumentBuilderFactory + .newInstance().newDocumentBuilder(); + final Document doc = builder.parse(conn.getInputStream()); + final XPath xpath = XPathFactory.newInstance().newXPath(); + final String numFound = xpath.evaluate("/response/result/@numFound", doc); + if ("0".equals(numFound)) { + missing = true; + } else { + final String g = xpath.evaluate("/response/result/doc[1]/str[@name='g']", doc); + LOGGER.finest(String.format("GroupId: %s", g)); + final String a = xpath.evaluate("/response/result/doc[1]/str[@name='a']", doc); + LOGGER.finest(String.format("ArtifactId: %s", a)); + final String v = xpath.evaluate("/response/result/doc[1]/str[@name='v']", doc); + LOGGER.finest(String.format("Version: %s", v)); + return new MavenArtifact(g, a, v); + } + } catch (Throwable e) { + // Anything else is jacked up XML stuff that we really can't recover + // from well + throw new IOException(e.getMessage(), e); + } + + if (missing) { + throw new FileNotFoundException("Artifact not found in Solr"); + } + } else { + final String msg = String.format("Could not connect to Solr received response code: %d %s", + conn.getResponseCode(), conn.getResponseMessage()); + LOGGER.fine(msg); + throw new IOException(msg); + } + + return null; + } +} diff --git a/dependency-check-core/src/main/resources/META-INF/services/org.owasp.dependencycheck.analyzer.Analyzer b/dependency-check-core/src/main/resources/META-INF/services/org.owasp.dependencycheck.analyzer.Analyzer index d752f4294..f98b34e8a 100644 --- a/dependency-check-core/src/main/resources/META-INF/services/org.owasp.dependencycheck.analyzer.Analyzer +++ b/dependency-check-core/src/main/resources/META-INF/services/org.owasp.dependencycheck.analyzer.Analyzer @@ -8,6 +8,7 @@ org.owasp.dependencycheck.analyzer.CpeSuppressionAnalyzer org.owasp.dependencycheck.analyzer.DependencyBundlingAnalyzer org.owasp.dependencycheck.analyzer.NvdCveAnalyzer org.owasp.dependencycheck.analyzer.VulnerabilitySuppressionAnalyzer +org.owasp.dependencycheck.analyzer.SolrAnalyzer org.owasp.dependencycheck.analyzer.NexusAnalyzer org.owasp.dependencycheck.analyzer.NuspecAnalyzer org.owasp.dependencycheck.analyzer.AssemblyAnalyzer \ No newline at end of file diff --git a/dependency-check-core/src/main/resources/dependencycheck.properties b/dependency-check-core/src/main/resources/dependencycheck.properties index 0090e0c50..65d748ed0 100644 --- a/dependency-check-core/src/main/resources/dependencycheck.properties +++ b/dependency-check-core/src/main/resources/dependencycheck.properties @@ -58,3 +58,10 @@ analyzer.nexus.url=https://repository.sonatype.org/service/local/ # If set to true, the proxy will still ONLY be used if the proxy properties (proxy.url, proxy.port) # are configured analyzer.nexus.proxy=true + +# the URL for searching search.maven.org for SHA-1 and whether it's enabled +analyzer.solr.enabled=true +analyzer.solr.url=http://search.maven.org/solrsearch/select +# If set to true, the proxy will still ONLY be used if the proxy properties (proxy.url, proxy.port) +# are configured +analyzer.solr.proxy=true \ No newline at end of file diff --git a/dependency-check-core/src/test/java/org/owasp/dependencycheck/data/solr/SolrSearchTest.java b/dependency-check-core/src/test/java/org/owasp/dependencycheck/data/solr/SolrSearchTest.java new file mode 100644 index 000000000..937a6d849 --- /dev/null +++ b/dependency-check-core/src/test/java/org/owasp/dependencycheck/data/solr/SolrSearchTest.java @@ -0,0 +1,57 @@ +package org.owasp.dependencycheck.data.solr; + +import org.junit.Assume; +import org.junit.Before; +import org.junit.Test; +import org.owasp.dependencycheck.BaseTest; +import org.owasp.dependencycheck.data.nexus.MavenArtifact; +import org.owasp.dependencycheck.utils.Settings; + +import java.io.FileNotFoundException; +import java.net.URL; +import java.util.logging.Logger; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; + +/** + * Created by colezlaw on 10/13/14. + */ +public class SolrSearchTest extends BaseTest { + private static final Logger LOGGER = Logger.getLogger(SolrSearchTest.class.getName()); + private SolrSearch searcher; + + @Before + public void setUp() throws Exception { + String solrUrl = Settings.getString(Settings.KEYS.ANALYZER_SOLR_URL); + LOGGER.fine(solrUrl); + searcher = new SolrSearch(new URL(solrUrl)); + } + + @Test(expected = IllegalArgumentException.class) + public void testNullSha1() throws Exception { searcher.searchSha1(null); } + + @Test(expected = IllegalArgumentException.class) + public void testMalformedSha1() throws Exception { + searcher.searchSha1("invalid"); + } + + // This test does generate network traffic and communicates with a host + // you may not be able to reach. Remove the @Ignore annotation if you want to + // test it anyway + @Test + public void testValidSha1() throws Exception { + MavenArtifact ma = searcher.searchSha1("9977a8d04e75609cf01badc4eb6a9c7198c4c5ea"); + assertEquals("Incorrect group", "org.apache.maven.plugins", ma.getGroupId()); + assertEquals("Incorrect artifact", "maven-compiler-plugin", ma.getArtifactId()); + assertEquals("Incorrect version", "3.1", ma.getVersion()); + } + + // This test does generate network traffic and communicates with a host + // you may not be able to reach. Remove the @Ignore annotation if you want to + // test it anyway + @Test(expected = FileNotFoundException.class) + public void testMissingSha1() throws Exception { + searcher.searchSha1("AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"); + } +} diff --git a/dependency-check-core/src/test/resources/dependencycheck.properties b/dependency-check-core/src/test/resources/dependencycheck.properties index 61efb2407..07d11d566 100644 --- a/dependency-check-core/src/test/resources/dependencycheck.properties +++ b/dependency-check-core/src/test/resources/dependencycheck.properties @@ -59,3 +59,9 @@ analyzer.nexus.url=https://repository.sonatype.org/service/local/ # If set to true, the proxy will still ONLY be used if the proxy properties (proxy.url, proxy.port) # are configured analyzer.nexus.proxy=true + +# the URL for searching search.maven.org for SHA-1 and whether it's enabled +analyzer.solr.enabled=true +analyzer.solr.url=http://search.maven.org/solrsearch/select +# If set to true, the proxy will still ONLY be used if the proxy properties (proxy.url, proxy.port) +# are configured \ No newline at end of file diff --git a/dependency-check-utils/src/main/java/org/owasp/dependencycheck/utils/Settings.java b/dependency-check-utils/src/main/java/org/owasp/dependencycheck/utils/Settings.java index 7ff76b16a..5306e3073 100644 --- a/dependency-check-utils/src/main/java/org/owasp/dependencycheck/utils/Settings.java +++ b/dependency-check-utils/src/main/java/org/owasp/dependencycheck/utils/Settings.java @@ -188,6 +188,18 @@ public final class Settings { * The properties key for using the proxy to reach Nexus. */ public static final String ANALYZER_NEXUS_PROXY = "analyzer.nexus.proxy"; + /** + * The properties key for whether the Solr analyzer is enabled. + */ + public static final String ANALYZER_SOLR_ENABLED = "analyzer.solr.enabled"; + /** + * The properties key for the Solr search URL. + */ + public static final String ANALYZER_SOLR_URL = "analyzer.solr.url"; + /** + * The properties key for using the proxy to reach Solr. + */ + public static final String ANALYZER_SOLR_PROXY = "analyzer.solr.proxy"; /** * The path to mono, if available. */ From 931110ba6c33d6d69fff2b69f79c8851cc01e246 Mon Sep 17 00:00:00 2001 From: Will Stranathan Date: Thu, 16 Oct 2014 15:04:50 -0400 Subject: [PATCH 2/3] Initial checkin of work on a Solr analyzer which will replace Nexus on Internet checks Former-commit-id: 09337f6416fa3140c00413426e17e39c4a1fadc6 --- .../analyzer/NexusAnalyzer.java | 37 ++++ .../analyzer/SolrAnalyzer.java | 170 ++++++++++++++++++ .../dependencycheck/data/solr/SolrSearch.java | 136 ++++++++++++++ ...rg.owasp.dependencycheck.analyzer.Analyzer | 1 + .../main/resources/dependencycheck.properties | 7 + .../data/solr/SolrSearchTest.java | 57 ++++++ .../test/resources/dependencycheck.properties | 6 + .../owasp/dependencycheck/utils/Settings.java | 12 ++ 8 files changed, 426 insertions(+) create mode 100644 dependency-check-core/src/main/java/org/owasp/dependencycheck/analyzer/SolrAnalyzer.java create mode 100644 dependency-check-core/src/main/java/org/owasp/dependencycheck/data/solr/SolrSearch.java create mode 100644 dependency-check-core/src/test/java/org/owasp/dependencycheck/data/solr/SolrSearchTest.java diff --git a/dependency-check-core/src/main/java/org/owasp/dependencycheck/analyzer/NexusAnalyzer.java b/dependency-check-core/src/main/java/org/owasp/dependencycheck/analyzer/NexusAnalyzer.java index 1a21ba2b0..abc546955 100644 --- a/dependency-check-core/src/main/java/org/owasp/dependencycheck/analyzer/NexusAnalyzer.java +++ b/dependency-check-core/src/main/java/org/owasp/dependencycheck/analyzer/NexusAnalyzer.java @@ -31,8 +31,11 @@ import org.owasp.dependencycheck.data.nexus.MavenArtifact; import org.owasp.dependencycheck.data.nexus.NexusSearch; import org.owasp.dependencycheck.dependency.Confidence; import org.owasp.dependencycheck.dependency.Dependency; +import org.owasp.dependencycheck.utils.InvalidSettingException; import org.owasp.dependencycheck.utils.Settings; +import javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter; + /** * Analyzer which will attempt to locate a dependency on a Nexus service by SHA-1 digest of the dependency. * @@ -48,6 +51,10 @@ import org.owasp.dependencycheck.utils.Settings; * @author colezlaw */ public class NexusAnalyzer extends AbstractFileTypeAnalyzer { + /** + * The default URL - this will be used by the SolrAnalyzer to determine whether to enable this. + */ + public static final String DEFAULT_URL = "https://repository.sonatype.org/service/local/"; /** * The logger. @@ -74,6 +81,33 @@ public class NexusAnalyzer extends AbstractFileTypeAnalyzer { */ private NexusSearch searcher; + /** + * Determine whether to enable this analyzer or not. + * + * @return whether the analyzer should be enabled + */ + @Override + public boolean isEnabled() { + /* Enable this analyzer ONLY if the Nexus URL has been set to something + other than the default one (if it's the default one, we'll use the + solr one) and it's enabled by the user. + */ + boolean retval = false; + try { + if ((! DEFAULT_URL.equals(Settings.getString(Settings.KEYS.ANALYZER_NEXUS_URL))) + && Settings.getBoolean(Settings.KEYS.ANALYZER_NEXUS_ENABLED)) { + LOGGER.info("Enabling Nexus analyzer"); + retval = true; + } else { + LOGGER.info("Nexus analyzer disabled"); + } + } catch (InvalidSettingException ise) { + LOGGER.warning("Invalid setting. Disabling Nexus analyzer"); + } + + return retval; + } + /** * Initializes the analyzer once before any analysis is performed. * @@ -150,6 +184,9 @@ public class NexusAnalyzer extends AbstractFileTypeAnalyzer { */ @Override public void analyzeFileType(Dependency dependency, Engine engine) throws AnalysisException { + if (! isEnabled()) { + return; + } try { final MavenArtifact ma = searcher.searchSha1(dependency.getSha1sum()); dependency.addAsEvidence("nexus", ma, Confidence.HIGH); diff --git a/dependency-check-core/src/main/java/org/owasp/dependencycheck/analyzer/SolrAnalyzer.java b/dependency-check-core/src/main/java/org/owasp/dependencycheck/analyzer/SolrAnalyzer.java new file mode 100644 index 000000000..82aee8d66 --- /dev/null +++ b/dependency-check-core/src/main/java/org/owasp/dependencycheck/analyzer/SolrAnalyzer.java @@ -0,0 +1,170 @@ +package org.owasp.dependencycheck.analyzer; + +import org.owasp.dependencycheck.Engine; +import org.owasp.dependencycheck.analyzer.exception.AnalysisException; +import org.owasp.dependencycheck.data.nexus.MavenArtifact; +import org.owasp.dependencycheck.data.solr.SolrSearch; +import org.owasp.dependencycheck.dependency.Confidence; +import org.owasp.dependencycheck.dependency.Dependency; +import org.owasp.dependencycheck.utils.InvalidSettingException; +import org.owasp.dependencycheck.utils.Settings; + +import java.io.FileNotFoundException; +import java.io.IOException; +import java.net.URL; +import java.util.Set; +import java.util.logging.Level; +import java.util.logging.Logger; + +/** + * Created by colezlaw on 10/9/14. + */ +public class SolrAnalyzer extends AbstractFileTypeAnalyzer { + /** + * The logger. + */ + private static final Logger LOGGER = Logger.getLogger(SolrAnalyzer.class.getName()); + + /** + * The name of the analyzer. + */ + private static final String ANALYZER_NAME = "Solr Analyzer"; + + /** + * The phase in which this analyzer runs. + */ + private static final AnalysisPhase ANALYSIS_PHASE = AnalysisPhase.INFORMATION_COLLECTION; + + /** + * The types of files on which this will work. + */ + private static final Set SUPPORTED_EXTENSIONS = newHashSet("jar"); + + /** + * The analyzer should be disabled if there are errors, so this is a flag + * to determine if such an error has occurred. + */ + protected boolean errorFlag = false; + + /** + * The searcher itself. + */ + private SolrSearch searcher; + + /** + * Determine whether to enable this analyzer or not. + * + * @return whether the analyzer should be enabled + */ + @Override + public boolean isEnabled() { + boolean retval = false; + + try { + if (Settings.getBoolean(Settings.KEYS.ANALYZER_SOLR_ENABLED)) { + if (!Settings.getBoolean(Settings.KEYS.ANALYZER_NEXUS_ENABLED) + || NexusAnalyzer.DEFAULT_URL.equals(Settings.getString(Settings.KEYS.ANALYZER_NEXUS_URL))) { + LOGGER.info("Enabling the Solr analyzer"); + retval = true; + } else { + LOGGER.info("Nexus analyzer is enabled, disabling Solr"); + } + } else { + LOGGER.info("Solr analyzer disabled"); + } + } catch (InvalidSettingException ise) { + LOGGER.warning("Invalid setting. Disabling the Solr analyzer"); + } + + return retval; + } + + /** + * Initializes the analyzer once before any analysis is performed. + * + * @throws Exception if there's an error during initalization + */ + @Override + public void initializeFileTypeAnalyzer() throws Exception { + LOGGER.fine("Initializing Solr analyzer"); + LOGGER.fine(String.format("Solr analyzer enabled: %s", isEnabled())); + if (isEnabled()) { + final String searchUrl = Settings.getString(Settings.KEYS.ANALYZER_SOLR_URL); + LOGGER.fine(String.format("Solr Analyzer URL: %s", searchUrl)); + searcher = new SolrSearch(new URL(searchUrl)); + } + } + + /** + * Returns the analyzer's name. + * + * @return the name of the analyzer + */ + @Override + public String getName() { + return ANALYZER_NAME; + } + + /** Returns the key used in the properties file to to reference the analyzer's enabled property. + * + * @return the analyzer's enabled property setting key. + */ + @Override + protected String getAnalyzerEnabledSettingKey() { + return Settings.KEYS.ANALYZER_SOLR_ENABLED; + } + + /** + * Returns the analysis phase under which the analyzer runs. + * + * @return the phase under which the analyzer runs + */ + @Override + public AnalysisPhase getAnalysisPhase() { + return ANALYSIS_PHASE; + } + + /** + * Returns the extensions for which this Analyzer runs. + * + * @return the extensions for which this Analyzer runs + */ + @Override + public Set getSupportedExtensions() { + return SUPPORTED_EXTENSIONS; + } + + /** + * Performs the analysis. + * + * @param dependency the dependency to analyze + * @param engine the engine + * @throws AnalysisException when there's an exception during analysis + */ + @Override + public void analyzeFileType(Dependency dependency, Engine engine) throws AnalysisException { + if (errorFlag) { + return; + } + + try { + final MavenArtifact ma = searcher.searchSha1(dependency.getSha1sum()); + if (ma.getGroupId() != null && !"".equals(ma.getGroupId())) { + dependency.getVendorEvidence().addEvidence("solr", "groupid", ma.getGroupId(), Confidence.HIGH); + } + if (ma.getArtifactId() != null && !"".equals(ma.getArtifactId())) { + dependency.getProductEvidence().addEvidence("solr", "artifactid", ma.getArtifactId(), Confidence.HIGH); + } + if (ma.getVersion() != null && !"".equals(ma.getVersion())) { + dependency.getVersionEvidence().addEvidence("solr", "version", ma.getVersion(), Confidence.HIGH); + } + } catch (IllegalArgumentException iae) { + LOGGER.info(String.format("invalid sha1-hash on %s", dependency.getFileName())); + } catch (FileNotFoundException fnfe) { + LOGGER.fine(String.format("Artifact not found in repository: '%s", dependency.getFileName())); + } catch (IOException ioe) { + LOGGER.log(Level.FINE, "Could not connect to Solr search", ioe); + errorFlag = true; + } + } +} diff --git a/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/solr/SolrSearch.java b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/solr/SolrSearch.java new file mode 100644 index 000000000..fdf1758b0 --- /dev/null +++ b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/solr/SolrSearch.java @@ -0,0 +1,136 @@ +package org.owasp.dependencycheck.data.solr; + +import org.owasp.dependencycheck.data.nexus.MavenArtifact; +import org.owasp.dependencycheck.utils.InvalidSettingException; +import org.owasp.dependencycheck.utils.Settings; +import org.owasp.dependencycheck.utils.URLConnectionFactory; +import org.w3c.dom.Document; + +import javax.xml.parsers.DocumentBuilder; +import javax.xml.parsers.DocumentBuilderFactory; +import javax.xml.xpath.XPath; +import javax.xml.xpath.XPathFactory; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.net.HttpURLConnection; +import java.net.URL; +import java.util.logging.Logger; + +/** + * Class of methods to search Maven Central via Solr. + * + * @author colezlaw + */ +public class SolrSearch { + /** + * The URL for the Solr service + */ + private final URL rootURL; + + /** + * Whether to use the Proxy when making requests + */ + private boolean useProxy; + + /** + * Used for logging. + */ + private static final Logger LOGGER = Logger.getLogger(SolrSearch.class.getName()); + + /** + * Determines whether we'll continue using the analyzer. If there's some sort + * of HTTP failure, we'll disable the analyzer. + */ + private boolean isEnabled = true; + + /** + * Creates a NexusSearch for the given repository URL. + * + * @param rootURL the URL of the repository on which searches should execute. + * Only parameters are added to this (so it should end in /select) + */ + public SolrSearch(URL rootURL) { + this.rootURL = rootURL; + try { + if (null != Settings.getString(Settings.KEYS.PROXY_SERVER) + && Settings.getBoolean(Settings.KEYS.ANALYZER_SOLR_PROXY)) { + useProxy = true; + LOGGER.fine("Using proxy"); + } else { + useProxy = false; + LOGGER.fine("Not using proxy"); + } + } catch (InvalidSettingException ise) { + useProxy = false; + } + } + + /** + * Searches the configured Solr URL for the given sha1 hash. If the artifact is found, a + * MavenArtifact is populated with the GAV. + * + * @param sha1 the SHA-1 hash string for which to search + * @return the populated Maven GAV. + * @throws IOException if it's unable to connect to the specified repository or if + * the specified artifact is not found. + */ + public MavenArtifact searchSha1(String sha1) throws IOException { + if (null == sha1 || !sha1.matches("^[0-9A-Fa-f]{40}$")) { + throw new IllegalArgumentException("Invalid SHA1 format"); + } + + final URL url = new URL(rootURL + String.format("?q=1:\"%s\"&wt=xml", sha1)); + + LOGGER.info(String.format("Searching Solr url %s", url.toString())); + + // Determine if we need to use a proxy. The rules: + // 1) If the proxy is set, AND the setting is set to true, use the proxy + // 2) Otherwise, don't use the proxy (either the proxy isn't configured, + // or proxy is specifically set to false) + final HttpURLConnection conn = URLConnectionFactory.createHttpURLConnection(url, useProxy); + + conn.setDoOutput(true); + + // JSON would be more elegant, but there's not currently a dependency + // on JSON, so don't want to add one just for this + conn.addRequestProperty("Accept", "application/xml"); + conn.connect(); + + if (conn.getResponseCode() == 200) { + boolean missing = false; + try { + final DocumentBuilder builder = DocumentBuilderFactory + .newInstance().newDocumentBuilder(); + final Document doc = builder.parse(conn.getInputStream()); + final XPath xpath = XPathFactory.newInstance().newXPath(); + final String numFound = xpath.evaluate("/response/result/@numFound", doc); + if ("0".equals(numFound)) { + missing = true; + } else { + final String g = xpath.evaluate("/response/result/doc[1]/str[@name='g']", doc); + LOGGER.finest(String.format("GroupId: %s", g)); + final String a = xpath.evaluate("/response/result/doc[1]/str[@name='a']", doc); + LOGGER.finest(String.format("ArtifactId: %s", a)); + final String v = xpath.evaluate("/response/result/doc[1]/str[@name='v']", doc); + LOGGER.finest(String.format("Version: %s", v)); + return new MavenArtifact(g, a, v); + } + } catch (Throwable e) { + // Anything else is jacked up XML stuff that we really can't recover + // from well + throw new IOException(e.getMessage(), e); + } + + if (missing) { + throw new FileNotFoundException("Artifact not found in Solr"); + } + } else { + final String msg = String.format("Could not connect to Solr received response code: %d %s", + conn.getResponseCode(), conn.getResponseMessage()); + LOGGER.fine(msg); + throw new IOException(msg); + } + + return null; + } +} diff --git a/dependency-check-core/src/main/resources/META-INF/services/org.owasp.dependencycheck.analyzer.Analyzer b/dependency-check-core/src/main/resources/META-INF/services/org.owasp.dependencycheck.analyzer.Analyzer index d752f4294..f98b34e8a 100644 --- a/dependency-check-core/src/main/resources/META-INF/services/org.owasp.dependencycheck.analyzer.Analyzer +++ b/dependency-check-core/src/main/resources/META-INF/services/org.owasp.dependencycheck.analyzer.Analyzer @@ -8,6 +8,7 @@ org.owasp.dependencycheck.analyzer.CpeSuppressionAnalyzer org.owasp.dependencycheck.analyzer.DependencyBundlingAnalyzer org.owasp.dependencycheck.analyzer.NvdCveAnalyzer org.owasp.dependencycheck.analyzer.VulnerabilitySuppressionAnalyzer +org.owasp.dependencycheck.analyzer.SolrAnalyzer org.owasp.dependencycheck.analyzer.NexusAnalyzer org.owasp.dependencycheck.analyzer.NuspecAnalyzer org.owasp.dependencycheck.analyzer.AssemblyAnalyzer \ No newline at end of file diff --git a/dependency-check-core/src/main/resources/dependencycheck.properties b/dependency-check-core/src/main/resources/dependencycheck.properties index 0090e0c50..65d748ed0 100644 --- a/dependency-check-core/src/main/resources/dependencycheck.properties +++ b/dependency-check-core/src/main/resources/dependencycheck.properties @@ -58,3 +58,10 @@ analyzer.nexus.url=https://repository.sonatype.org/service/local/ # If set to true, the proxy will still ONLY be used if the proxy properties (proxy.url, proxy.port) # are configured analyzer.nexus.proxy=true + +# the URL for searching search.maven.org for SHA-1 and whether it's enabled +analyzer.solr.enabled=true +analyzer.solr.url=http://search.maven.org/solrsearch/select +# If set to true, the proxy will still ONLY be used if the proxy properties (proxy.url, proxy.port) +# are configured +analyzer.solr.proxy=true \ No newline at end of file diff --git a/dependency-check-core/src/test/java/org/owasp/dependencycheck/data/solr/SolrSearchTest.java b/dependency-check-core/src/test/java/org/owasp/dependencycheck/data/solr/SolrSearchTest.java new file mode 100644 index 000000000..937a6d849 --- /dev/null +++ b/dependency-check-core/src/test/java/org/owasp/dependencycheck/data/solr/SolrSearchTest.java @@ -0,0 +1,57 @@ +package org.owasp.dependencycheck.data.solr; + +import org.junit.Assume; +import org.junit.Before; +import org.junit.Test; +import org.owasp.dependencycheck.BaseTest; +import org.owasp.dependencycheck.data.nexus.MavenArtifact; +import org.owasp.dependencycheck.utils.Settings; + +import java.io.FileNotFoundException; +import java.net.URL; +import java.util.logging.Logger; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; + +/** + * Created by colezlaw on 10/13/14. + */ +public class SolrSearchTest extends BaseTest { + private static final Logger LOGGER = Logger.getLogger(SolrSearchTest.class.getName()); + private SolrSearch searcher; + + @Before + public void setUp() throws Exception { + String solrUrl = Settings.getString(Settings.KEYS.ANALYZER_SOLR_URL); + LOGGER.fine(solrUrl); + searcher = new SolrSearch(new URL(solrUrl)); + } + + @Test(expected = IllegalArgumentException.class) + public void testNullSha1() throws Exception { searcher.searchSha1(null); } + + @Test(expected = IllegalArgumentException.class) + public void testMalformedSha1() throws Exception { + searcher.searchSha1("invalid"); + } + + // This test does generate network traffic and communicates with a host + // you may not be able to reach. Remove the @Ignore annotation if you want to + // test it anyway + @Test + public void testValidSha1() throws Exception { + MavenArtifact ma = searcher.searchSha1("9977a8d04e75609cf01badc4eb6a9c7198c4c5ea"); + assertEquals("Incorrect group", "org.apache.maven.plugins", ma.getGroupId()); + assertEquals("Incorrect artifact", "maven-compiler-plugin", ma.getArtifactId()); + assertEquals("Incorrect version", "3.1", ma.getVersion()); + } + + // This test does generate network traffic and communicates with a host + // you may not be able to reach. Remove the @Ignore annotation if you want to + // test it anyway + @Test(expected = FileNotFoundException.class) + public void testMissingSha1() throws Exception { + searcher.searchSha1("AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"); + } +} diff --git a/dependency-check-core/src/test/resources/dependencycheck.properties b/dependency-check-core/src/test/resources/dependencycheck.properties index 61efb2407..07d11d566 100644 --- a/dependency-check-core/src/test/resources/dependencycheck.properties +++ b/dependency-check-core/src/test/resources/dependencycheck.properties @@ -59,3 +59,9 @@ analyzer.nexus.url=https://repository.sonatype.org/service/local/ # If set to true, the proxy will still ONLY be used if the proxy properties (proxy.url, proxy.port) # are configured analyzer.nexus.proxy=true + +# the URL for searching search.maven.org for SHA-1 and whether it's enabled +analyzer.solr.enabled=true +analyzer.solr.url=http://search.maven.org/solrsearch/select +# If set to true, the proxy will still ONLY be used if the proxy properties (proxy.url, proxy.port) +# are configured \ No newline at end of file diff --git a/dependency-check-utils/src/main/java/org/owasp/dependencycheck/utils/Settings.java b/dependency-check-utils/src/main/java/org/owasp/dependencycheck/utils/Settings.java index 7ff76b16a..5306e3073 100644 --- a/dependency-check-utils/src/main/java/org/owasp/dependencycheck/utils/Settings.java +++ b/dependency-check-utils/src/main/java/org/owasp/dependencycheck/utils/Settings.java @@ -188,6 +188,18 @@ public final class Settings { * The properties key for using the proxy to reach Nexus. */ public static final String ANALYZER_NEXUS_PROXY = "analyzer.nexus.proxy"; + /** + * The properties key for whether the Solr analyzer is enabled. + */ + public static final String ANALYZER_SOLR_ENABLED = "analyzer.solr.enabled"; + /** + * The properties key for the Solr search URL. + */ + public static final String ANALYZER_SOLR_URL = "analyzer.solr.url"; + /** + * The properties key for using the proxy to reach Solr. + */ + public static final String ANALYZER_SOLR_PROXY = "analyzer.solr.proxy"; /** * The path to mono, if available. */ From 17d7d47b9a8a8b14cf37e1c2277dccd80584ac4f Mon Sep 17 00:00:00 2001 From: Will Stranathan Date: Sat, 8 Nov 2014 10:53:17 -0500 Subject: [PATCH 3/3] Renamed Solr to Central search Former-commit-id: b8bdca6be89497f9baf29c8762803b4652e4974b --- ...SolrAnalyzer.java => CentralAnalyzer.java} | 27 +++++++-------- .../analyzer/NexusAnalyzer.java | 2 +- .../CentralSearch.java} | 34 ++++++++++++------- .../dependency/Dependency.java | 2 ++ ...rg.owasp.dependencycheck.analyzer.Analyzer | 2 +- .../CentralSearchTest.java} | 30 +++++++++------- 6 files changed, 56 insertions(+), 41 deletions(-) rename dependency-check-core/src/main/java/org/owasp/dependencycheck/analyzer/{SolrAnalyzer.java => CentralAnalyzer.java} (82%) rename dependency-check-core/src/main/java/org/owasp/dependencycheck/data/{solr/SolrSearch.java => central/CentralSearch.java} (77%) rename dependency-check-core/src/test/java/org/owasp/dependencycheck/data/{solr/SolrSearchTest.java => central/CentralSearchTest.java} (64%) diff --git a/dependency-check-core/src/main/java/org/owasp/dependencycheck/analyzer/SolrAnalyzer.java b/dependency-check-core/src/main/java/org/owasp/dependencycheck/analyzer/CentralAnalyzer.java similarity index 82% rename from dependency-check-core/src/main/java/org/owasp/dependencycheck/analyzer/SolrAnalyzer.java rename to dependency-check-core/src/main/java/org/owasp/dependencycheck/analyzer/CentralAnalyzer.java index 82aee8d66..76744b675 100644 --- a/dependency-check-core/src/main/java/org/owasp/dependencycheck/analyzer/SolrAnalyzer.java +++ b/dependency-check-core/src/main/java/org/owasp/dependencycheck/analyzer/CentralAnalyzer.java @@ -3,7 +3,7 @@ package org.owasp.dependencycheck.analyzer; import org.owasp.dependencycheck.Engine; import org.owasp.dependencycheck.analyzer.exception.AnalysisException; import org.owasp.dependencycheck.data.nexus.MavenArtifact; -import org.owasp.dependencycheck.data.solr.SolrSearch; +import org.owasp.dependencycheck.data.central.CentralSearch; import org.owasp.dependencycheck.dependency.Confidence; import org.owasp.dependencycheck.dependency.Dependency; import org.owasp.dependencycheck.utils.InvalidSettingException; @@ -12,6 +12,7 @@ import org.owasp.dependencycheck.utils.Settings; import java.io.FileNotFoundException; import java.io.IOException; import java.net.URL; +import java.util.List; import java.util.Set; import java.util.logging.Level; import java.util.logging.Logger; @@ -19,11 +20,11 @@ import java.util.logging.Logger; /** * Created by colezlaw on 10/9/14. */ -public class SolrAnalyzer extends AbstractFileTypeAnalyzer { +public class CentralAnalyzer extends AbstractFileTypeAnalyzer { /** * The logger. */ - private static final Logger LOGGER = Logger.getLogger(SolrAnalyzer.class.getName()); + private static final Logger LOGGER = Logger.getLogger(CentralAnalyzer.class.getName()); /** * The name of the analyzer. @@ -49,7 +50,7 @@ public class SolrAnalyzer extends AbstractFileTypeAnalyzer { /** * The searcher itself. */ - private SolrSearch searcher; + private CentralSearch searcher; /** * Determine whether to enable this analyzer or not. @@ -91,7 +92,7 @@ public class SolrAnalyzer extends AbstractFileTypeAnalyzer { if (isEnabled()) { final String searchUrl = Settings.getString(Settings.KEYS.ANALYZER_SOLR_URL); LOGGER.fine(String.format("Solr Analyzer URL: %s", searchUrl)); - searcher = new SolrSearch(new URL(searchUrl)); + searcher = new CentralSearch(new URL(searchUrl)); } } @@ -143,20 +144,16 @@ public class SolrAnalyzer extends AbstractFileTypeAnalyzer { */ @Override public void analyzeFileType(Dependency dependency, Engine engine) throws AnalysisException { - if (errorFlag) { + if (errorFlag || !isEnabled()) { return; } try { - final MavenArtifact ma = searcher.searchSha1(dependency.getSha1sum()); - if (ma.getGroupId() != null && !"".equals(ma.getGroupId())) { - dependency.getVendorEvidence().addEvidence("solr", "groupid", ma.getGroupId(), Confidence.HIGH); - } - if (ma.getArtifactId() != null && !"".equals(ma.getArtifactId())) { - dependency.getProductEvidence().addEvidence("solr", "artifactid", ma.getArtifactId(), Confidence.HIGH); - } - if (ma.getVersion() != null && !"".equals(ma.getVersion())) { - dependency.getVersionEvidence().addEvidence("solr", "version", ma.getVersion(), Confidence.HIGH); + final List mas = searcher.searchSha1(dependency.getSha1sum()); + final Confidence confidence = mas.size() > 1 ? Confidence.HIGH : Confidence.HIGHEST; + for (MavenArtifact ma : mas) { + LOGGER.fine(String.format("Central analyzer found artifact (%s) for dependency (%s)", ma.toString(), dependency.getFileName())); + dependency.addAsEvidence("central", ma, confidence); } } catch (IllegalArgumentException iae) { LOGGER.info(String.format("invalid sha1-hash on %s", dependency.getFileName())); diff --git a/dependency-check-core/src/main/java/org/owasp/dependencycheck/analyzer/NexusAnalyzer.java b/dependency-check-core/src/main/java/org/owasp/dependencycheck/analyzer/NexusAnalyzer.java index abc546955..2d9d608e5 100644 --- a/dependency-check-core/src/main/java/org/owasp/dependencycheck/analyzer/NexusAnalyzer.java +++ b/dependency-check-core/src/main/java/org/owasp/dependencycheck/analyzer/NexusAnalyzer.java @@ -90,7 +90,7 @@ public class NexusAnalyzer extends AbstractFileTypeAnalyzer { public boolean isEnabled() { /* Enable this analyzer ONLY if the Nexus URL has been set to something other than the default one (if it's the default one, we'll use the - solr one) and it's enabled by the user. + central one) and it's enabled by the user. */ boolean retval = false; try { diff --git a/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/solr/SolrSearch.java b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/central/CentralSearch.java similarity index 77% rename from dependency-check-core/src/main/java/org/owasp/dependencycheck/data/solr/SolrSearch.java rename to dependency-check-core/src/main/java/org/owasp/dependencycheck/data/central/CentralSearch.java index fdf1758b0..4c55e47a1 100644 --- a/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/solr/SolrSearch.java +++ b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/central/CentralSearch.java @@ -1,19 +1,23 @@ -package org.owasp.dependencycheck.data.solr; +package org.owasp.dependencycheck.data.central; import org.owasp.dependencycheck.data.nexus.MavenArtifact; import org.owasp.dependencycheck.utils.InvalidSettingException; import org.owasp.dependencycheck.utils.Settings; import org.owasp.dependencycheck.utils.URLConnectionFactory; import org.w3c.dom.Document; +import org.w3c.dom.NodeList; import javax.xml.parsers.DocumentBuilder; import javax.xml.parsers.DocumentBuilderFactory; import javax.xml.xpath.XPath; +import javax.xml.xpath.XPathConstants; import javax.xml.xpath.XPathFactory; import java.io.FileNotFoundException; import java.io.IOException; import java.net.HttpURLConnection; import java.net.URL; +import java.util.ArrayList; +import java.util.List; import java.util.logging.Logger; /** @@ -21,7 +25,7 @@ import java.util.logging.Logger; * * @author colezlaw */ -public class SolrSearch { +public class CentralSearch { /** * The URL for the Solr service */ @@ -35,7 +39,7 @@ public class SolrSearch { /** * Used for logging. */ - private static final Logger LOGGER = Logger.getLogger(SolrSearch.class.getName()); + private static final Logger LOGGER = Logger.getLogger(CentralSearch.class.getName()); /** * Determines whether we'll continue using the analyzer. If there's some sort @@ -49,7 +53,7 @@ public class SolrSearch { * @param rootURL the URL of the repository on which searches should execute. * Only parameters are added to this (so it should end in /select) */ - public SolrSearch(URL rootURL) { + public CentralSearch(URL rootURL) { this.rootURL = rootURL; try { if (null != Settings.getString(Settings.KEYS.PROXY_SERVER) @@ -74,7 +78,7 @@ public class SolrSearch { * @throws IOException if it's unable to connect to the specified repository or if * the specified artifact is not found. */ - public MavenArtifact searchSha1(String sha1) throws IOException { + public List searchSha1(String sha1) throws IOException { if (null == sha1 || !sha1.matches("^[0-9A-Fa-f]{40}$")) { throw new IllegalArgumentException("Invalid SHA1 format"); } @@ -107,13 +111,19 @@ public class SolrSearch { if ("0".equals(numFound)) { missing = true; } else { - final String g = xpath.evaluate("/response/result/doc[1]/str[@name='g']", doc); - LOGGER.finest(String.format("GroupId: %s", g)); - final String a = xpath.evaluate("/response/result/doc[1]/str[@name='a']", doc); - LOGGER.finest(String.format("ArtifactId: %s", a)); - final String v = xpath.evaluate("/response/result/doc[1]/str[@name='v']", doc); - LOGGER.finest(String.format("Version: %s", v)); - return new MavenArtifact(g, a, v); + ArrayList result = new ArrayList(); + NodeList docs = (NodeList)xpath.evaluate("/response/result/doc", doc, XPathConstants.NODESET); + for (int i = 0; i < docs.getLength(); i++) { + final String g = xpath.evaluate("./str[@name='g']", docs.item(i)); + LOGGER.finest(String.format("GroupId: %s", g)); + final String a = xpath.evaluate("./str[@name='a']", docs.item(i)); + LOGGER.finest(String.format("ArtifactId: %s", a)); + final String v = xpath.evaluate("./str[@name='v']", docs.item(i)); + LOGGER.finest(String.format("Version: %s", v)); + result.add(new MavenArtifact(g, a, v, url.toString())); + } + + return result; } } catch (Throwable e) { // Anything else is jacked up XML stuff that we really can't recover diff --git a/dependency-check-core/src/main/java/org/owasp/dependencycheck/dependency/Dependency.java b/dependency-check-core/src/main/java/org/owasp/dependencycheck/dependency/Dependency.java index ef1117148..2f72b3d84 100644 --- a/dependency-check-core/src/main/java/org/owasp/dependencycheck/dependency/Dependency.java +++ b/dependency-check-core/src/main/java/org/owasp/dependencycheck/dependency/Dependency.java @@ -341,10 +341,12 @@ public class Dependency implements Serializable, Comparable { found = true; i.setConfidence(Confidence.HIGHEST); i.setUrl(mavenArtifact.getArtifactUrl()); + LOGGER.fine(String.format("Already found identifier %s. Confidence set to highest", i.getValue())); break; } } if (!found) { + LOGGER.fine(String.format("Adding new maven identifier %s", mavenArtifact.toString())); this.addIdentifier("maven", mavenArtifact.toString(), mavenArtifact.getArtifactUrl(), Confidence.HIGHEST); } } diff --git a/dependency-check-core/src/main/resources/META-INF/services/org.owasp.dependencycheck.analyzer.Analyzer b/dependency-check-core/src/main/resources/META-INF/services/org.owasp.dependencycheck.analyzer.Analyzer index f98b34e8a..f2c4509c0 100644 --- a/dependency-check-core/src/main/resources/META-INF/services/org.owasp.dependencycheck.analyzer.Analyzer +++ b/dependency-check-core/src/main/resources/META-INF/services/org.owasp.dependencycheck.analyzer.Analyzer @@ -8,7 +8,7 @@ org.owasp.dependencycheck.analyzer.CpeSuppressionAnalyzer org.owasp.dependencycheck.analyzer.DependencyBundlingAnalyzer org.owasp.dependencycheck.analyzer.NvdCveAnalyzer org.owasp.dependencycheck.analyzer.VulnerabilitySuppressionAnalyzer -org.owasp.dependencycheck.analyzer.SolrAnalyzer +org.owasp.dependencycheck.analyzer.CentralAnalyzer org.owasp.dependencycheck.analyzer.NexusAnalyzer org.owasp.dependencycheck.analyzer.NuspecAnalyzer org.owasp.dependencycheck.analyzer.AssemblyAnalyzer \ No newline at end of file diff --git a/dependency-check-core/src/test/java/org/owasp/dependencycheck/data/solr/SolrSearchTest.java b/dependency-check-core/src/test/java/org/owasp/dependencycheck/data/central/CentralSearchTest.java similarity index 64% rename from dependency-check-core/src/test/java/org/owasp/dependencycheck/data/solr/SolrSearchTest.java rename to dependency-check-core/src/test/java/org/owasp/dependencycheck/data/central/CentralSearchTest.java index 937a6d849..ffcde235e 100644 --- a/dependency-check-core/src/test/java/org/owasp/dependencycheck/data/solr/SolrSearchTest.java +++ b/dependency-check-core/src/test/java/org/owasp/dependencycheck/data/central/CentralSearchTest.java @@ -1,6 +1,5 @@ -package org.owasp.dependencycheck.data.solr; +package org.owasp.dependencycheck.data.central; -import org.junit.Assume; import org.junit.Before; import org.junit.Test; import org.owasp.dependencycheck.BaseTest; @@ -9,23 +8,23 @@ import org.owasp.dependencycheck.utils.Settings; import java.io.FileNotFoundException; import java.net.URL; +import java.util.List; import java.util.logging.Logger; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.*; /** * Created by colezlaw on 10/13/14. */ -public class SolrSearchTest extends BaseTest { - private static final Logger LOGGER = Logger.getLogger(SolrSearchTest.class.getName()); - private SolrSearch searcher; +public class CentralSearchTest extends BaseTest { + private static final Logger LOGGER = Logger.getLogger(CentralSearchTest.class.getName()); + private CentralSearch searcher; @Before public void setUp() throws Exception { String solrUrl = Settings.getString(Settings.KEYS.ANALYZER_SOLR_URL); LOGGER.fine(solrUrl); - searcher = new SolrSearch(new URL(solrUrl)); + searcher = new CentralSearch(new URL(solrUrl)); } @Test(expected = IllegalArgumentException.class) @@ -41,10 +40,10 @@ public class SolrSearchTest extends BaseTest { // test it anyway @Test public void testValidSha1() throws Exception { - MavenArtifact ma = searcher.searchSha1("9977a8d04e75609cf01badc4eb6a9c7198c4c5ea"); - assertEquals("Incorrect group", "org.apache.maven.plugins", ma.getGroupId()); - assertEquals("Incorrect artifact", "maven-compiler-plugin", ma.getArtifactId()); - assertEquals("Incorrect version", "3.1", ma.getVersion()); + List ma = searcher.searchSha1("9977a8d04e75609cf01badc4eb6a9c7198c4c5ea"); + assertEquals("Incorrect group", "org.apache.maven.plugins", ma.get(0).getGroupId()); + assertEquals("Incorrect artifact", "maven-compiler-plugin", ma.get(0).getArtifactId()); + assertEquals("Incorrect version", "3.1", ma.get(0).getVersion()); } // This test does generate network traffic and communicates with a host @@ -54,4 +53,11 @@ public class SolrSearchTest extends BaseTest { public void testMissingSha1() throws Exception { searcher.searchSha1("AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"); } + + // This test should give us multiple results back from Solr + @Test + public void testMultipleReturns() throws Exception { + List ma = searcher.searchSha1("94A9CE681A42D0352B3AD22659F67835E560D107"); + assertTrue(ma.size() > 1); + } }