diff --git a/dependency-check-core/src/main/java/org/owasp/dependencycheck/analyzer/CPEAnalyzer.java b/dependency-check-core/src/main/java/org/owasp/dependencycheck/analyzer/CPEAnalyzer.java index 14177fa9b..c23ceccc4 100644 --- a/dependency-check-core/src/main/java/org/owasp/dependencycheck/analyzer/CPEAnalyzer.java +++ b/dependency-check-core/src/main/java/org/owasp/dependencycheck/analyzer/CPEAnalyzer.java @@ -27,6 +27,7 @@ import java.util.Set; import java.util.StringTokenizer; import java.util.concurrent.TimeUnit; import org.apache.commons.lang3.builder.CompareToBuilder; +import org.apache.lucene.analysis.util.CharArraySet; import org.apache.lucene.document.Document; import org.apache.lucene.index.CorruptIndexException; import org.apache.lucene.queryparser.classic.ParseException; @@ -39,6 +40,7 @@ import org.owasp.dependencycheck.data.cpe.Fields; import org.owasp.dependencycheck.data.cpe.IndexEntry; import org.owasp.dependencycheck.data.cpe.IndexException; import org.owasp.dependencycheck.data.lucene.LuceneUtils; +import org.owasp.dependencycheck.data.lucene.SearchFieldAnalyzer; import org.owasp.dependencycheck.data.nvdcve.CveDB; import org.owasp.dependencycheck.data.nvdcve.DatabaseException; import org.owasp.dependencycheck.dependency.Confidence; @@ -212,7 +214,6 @@ public class CPEAnalyzer extends AbstractAnalyzer { * @throws ParseException is thrown when the Lucene query cannot be parsed. */ protected void determineCPE(Dependency dependency) throws CorruptIndexException, IOException, ParseException { - //TODO test dojo-war against this. we should get dojo-toolkit:dojo-toolkit AND dojo-toolkit:toolkit String vendors = ""; String products = ""; for (Confidence confidence : Confidence.values()) { @@ -488,7 +489,11 @@ public class CPEAnalyzer extends AbstractAnalyzer { final String[] words = text.split("[\\s_-]"); final List list = new ArrayList<>(); String tempWord = null; + CharArraySet stopWords = SearchFieldAnalyzer.getStopWords(); for (String word : words) { + if (stopWords.contains(word)) { + continue; + } /* single letter words should be concatenated with the next word. so { "m", "core", "sample" } -> { "mcore", "sample" } @@ -561,6 +566,9 @@ public class CPEAnalyzer extends AbstractAnalyzer { protected boolean determineIdentifiers(Dependency dependency, String vendor, String product, Confidence currentConfidence) throws UnsupportedEncodingException { final Set cpes = cve.getCPEs(vendor, product); + if (cpes.isEmpty()) { + return false; + } DependencyVersion bestGuess = new DependencyVersion("-"); Confidence bestGuessConf = null; boolean hasBroadMatch = false; diff --git a/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/cpe/CpeMemoryIndex.java b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/cpe/CpeMemoryIndex.java index 3132778b4..4c1aa6a1e 100644 --- a/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/cpe/CpeMemoryIndex.java +++ b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/cpe/CpeMemoryIndex.java @@ -52,7 +52,7 @@ import org.slf4j.LoggerFactory; * * @author Jeremy Long */ -public final class CpeMemoryIndex { +public final class CpeMemoryIndex implements AutoCloseable { /** * The logger. @@ -160,6 +160,7 @@ public final class CpeMemoryIndex { /** * Closes the CPE Index. */ + @Override public synchronized void close() { if (searchingAnalyzer != null) { searchingAnalyzer.close(); @@ -206,7 +207,6 @@ public final class CpeMemoryIndex { v.setStringValue(pair.getLeft()); p.setStringValue(pair.getRight()); indexWriter.addDocument(doc); - resetFieldAnalyzer(); } } indexWriter.commit(); @@ -221,18 +221,6 @@ public final class CpeMemoryIndex { } } - /** - * Resets the product and vendor field analyzers. - */ - private void resetFieldAnalyzer() { - if (productFieldAnalyzer != null) { - productFieldAnalyzer.clear(); - } - if (vendorFieldAnalyzer != null) { - vendorFieldAnalyzer.clear(); - } - } - /** * Searches the index using the given search string. * @@ -248,7 +236,6 @@ public final class CpeMemoryIndex { throw new ParseException("Query is null or empty"); } LOGGER.debug(searchString); - resetFieldAnalyzer(); final Query query = queryParser.parse(searchString); return search(query, maxQueryResults); } @@ -263,7 +250,6 @@ public final class CpeMemoryIndex { * @throws IOException thrown if there is an IOException */ public synchronized TopDocs search(Query query, int maxQueryResults) throws CorruptIndexException, IOException { - resetFieldAnalyzer(); return indexSearcher.search(query, maxQueryResults); } diff --git a/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/lucene/FieldAnalyzer.java b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/lucene/FieldAnalyzer.java deleted file mode 100644 index 0736c9fb0..000000000 --- a/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/lucene/FieldAnalyzer.java +++ /dev/null @@ -1,84 +0,0 @@ -/* - * This file is part of dependency-check-core. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * Copyright (c) 2012 Jeremy Long. All Rights Reserved. - */ -package org.owasp.dependencycheck.data.lucene; - -import java.io.Reader; -import org.apache.lucene.analysis.Analyzer; -import org.apache.lucene.analysis.TokenStream; -import org.apache.lucene.analysis.Tokenizer; -import org.apache.lucene.analysis.core.LowerCaseFilter; -import org.apache.lucene.analysis.core.StopAnalyzer; -import org.apache.lucene.analysis.core.StopFilter; -import org.apache.lucene.analysis.miscellaneous.WordDelimiterFilter; -import org.apache.lucene.util.Version; - -/** - *

- * A Lucene Analyzer that utilizes the WhitespaceTokenizer, WordDelimiterFilter, - * LowerCaseFilter, and StopFilter. The intended purpose of this Analyzer is to - * index the CPE fields vendor and product.

- * - * @author Jeremy Long - * @deprecated the field analyzer should not be used, instead use the - * SearchFieldAnalyzer so that the token analyzing filter is used. - */ -@Deprecated -public class FieldAnalyzer extends Analyzer { - - /** - * The Lucene Version used. - */ - private final Version version; - - /** - * Creates a new FieldAnalyzer. - * - * @param version the Lucene version - */ - public FieldAnalyzer(Version version) { - this.version = version; - } - - /** - * Creates the TokenStreamComponents - * - * @param fieldName the field name being analyzed - * @param reader the reader containing the input - * @return the TokenStreamComponents - */ - @Override - protected TokenStreamComponents createComponents(String fieldName, Reader reader) { - final Tokenizer source = new AlphaNumericTokenizer(version, reader); - - TokenStream stream = source; - - stream = new WordDelimiterFilter(stream, - WordDelimiterFilter.CATENATE_WORDS - | WordDelimiterFilter.GENERATE_WORD_PARTS - | WordDelimiterFilter.GENERATE_NUMBER_PARTS - | WordDelimiterFilter.PRESERVE_ORIGINAL - | WordDelimiterFilter.SPLIT_ON_CASE_CHANGE - | WordDelimiterFilter.SPLIT_ON_NUMERICS - | WordDelimiterFilter.STEM_ENGLISH_POSSESSIVE, null); - - stream = new LowerCaseFilter(version, stream); - stream = new StopFilter(version, stream, StopAnalyzer.ENGLISH_STOP_WORDS_SET); - - return new TokenStreamComponents(source, stream); - } -} diff --git a/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/lucene/SearchFieldAnalyzer.java b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/lucene/SearchFieldAnalyzer.java index e5733f532..8b75f4c10 100644 --- a/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/lucene/SearchFieldAnalyzer.java +++ b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/lucene/SearchFieldAnalyzer.java @@ -18,6 +18,8 @@ package org.owasp.dependencycheck.data.lucene; import java.io.Reader; +import java.util.Arrays; +import java.util.List; import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.analysis.Tokenizer; @@ -25,6 +27,7 @@ import org.apache.lucene.analysis.core.LowerCaseFilter; import org.apache.lucene.analysis.core.StopAnalyzer; import org.apache.lucene.analysis.core.StopFilter; import org.apache.lucene.analysis.miscellaneous.WordDelimiterFilter; +import org.apache.lucene.analysis.util.CharArraySet; import org.apache.lucene.util.Version; /** @@ -39,9 +42,25 @@ public class SearchFieldAnalyzer extends Analyzer { */ private final Version version; /** - * A local reference to the TokenPairConcatenatingFilter so that we can clear any left over state if this analyzer is re-used. + * The list of additional stop words to use. */ - private TokenPairConcatenatingFilter concatenatingFilter; + private static final List ADDITIONAL_STOP_WORDS = Arrays.asList("software", "framework", "inc", + "com", "org", "net", "www", "consulting", "ltd", "foundation", "project"); + /** + * The set of stop words to use in the analyzer. + */ + private final CharArraySet stopWords; + + /** + * Returns the set of stop words being used. + * + * @return the set of stop words being used + */ + public static CharArraySet getStopWords() { + CharArraySet words = new CharArraySet(LuceneUtils.CURRENT_VERSION, StopAnalyzer.ENGLISH_STOP_WORDS_SET, true); + words.addAll(ADDITIONAL_STOP_WORDS); + return words; + } /** * Constructs a new SearchFieldAnalyzer. @@ -50,6 +69,7 @@ public class SearchFieldAnalyzer extends Analyzer { */ public SearchFieldAnalyzer(Version version) { this.version = version; + stopWords = getStopWords(); } /** @@ -75,22 +95,9 @@ public class SearchFieldAnalyzer extends Analyzer { stream = new LowerCaseFilter(version, stream); stream = new UrlTokenizingFilter(stream); - concatenatingFilter = new TokenPairConcatenatingFilter(stream); - stream = concatenatingFilter; - stream = new StopFilter(version, stream, StopAnalyzer.ENGLISH_STOP_WORDS_SET); + stream = new StopFilter(version, stream, stopWords); + stream = new TokenPairConcatenatingFilter(stream); return new TokenStreamComponents(source, stream); } - - /** - *

- * Resets the analyzer and clears any internal state data that may have been left-over from previous uses of the analyzer.

- *

- * If this analyzer is re-used this method must be called between uses.

- */ - public void clear() { - if (concatenatingFilter != null) { - concatenatingFilter.clear(); - } - } } diff --git a/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/lucene/TokenPairConcatenatingFilter.java b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/lucene/TokenPairConcatenatingFilter.java index 22f6f369a..0d966b528 100644 --- a/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/lucene/TokenPairConcatenatingFilter.java +++ b/dependency-check-core/src/main/java/org/owasp/dependencycheck/data/lucene/TokenPairConcatenatingFilter.java @@ -25,9 +25,11 @@ import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; /** *

- * Takes a TokenStream and adds additional tokens by concatenating pairs of words.

+ * Takes a TokenStream and adds additional tokens by concatenating pairs of + * words.

*

- * Example: "Spring Framework Core" -> "Spring SpringFramework Framework FrameworkCore Core".

+ * Example: "Spring Framework Core" -> "Spring SpringFramework + * Framework FrameworkCore Core".

* * @author Jeremy Long */ @@ -46,24 +48,6 @@ public final class TokenPairConcatenatingFilter extends TokenFilter { */ private final LinkedList words; - /** - * Returns the previous word. This is needed in the test cases. - * - * @return te previous word - */ - protected String getPreviousWord() { - return previousWord; - } - - /** - * Returns the words list. This is needed in the test cases. - * - * @return the words list - */ - protected LinkedList getWords() { - return words; - } - /** * Constructs a new TokenPairConcatenatingFilter. * @@ -75,8 +59,9 @@ public final class TokenPairConcatenatingFilter extends TokenFilter { } /** - * Increments the underlying TokenStream and sets CharTermAttributes to construct an expanded set of tokens by concatenating - * tokens with the previous token. + * Increments the underlying TokenStream and sets CharTermAttributes to + * construct an expanded set of tokens by concatenating tokens with the + * previous token. * * @return whether or not we have hit the end of the TokenStream * @throws IOException is thrown when an IOException occurs @@ -112,11 +97,18 @@ public final class TokenPairConcatenatingFilter extends TokenFilter { /** *

- * Resets the Filter and clears any internal state data that may have been left-over from previous uses of the Filter.

+ * Resets the Filter and clears any internal state data that may have been + * left-over from previous uses of the Filter.

*

- * If this Filter is re-used this method must be called between uses.

+ * If this Filter is re-used this method must be called between + * uses.

+ * + * @throws java.io.IOException thrown if there is an error resetting the + * filter */ - public void clear() { + @Override + public void end() throws IOException { + super.end(); previousWord = null; words.clear(); } @@ -158,5 +150,4 @@ public final class TokenPairConcatenatingFilter extends TokenFilter { } return !(this.words != other.words && (this.words == null || !this.words.equals(other.words))); } - } diff --git a/dependency-check-core/src/test/java/org/owasp/dependencycheck/data/lucene/FieldAnalyzerTest.java b/dependency-check-core/src/test/java/org/owasp/dependencycheck/data/lucene/FieldAnalyzerTest.java index f98107ba2..c6916a1b5 100644 --- a/dependency-check-core/src/test/java/org/owasp/dependencycheck/data/lucene/FieldAnalyzerTest.java +++ b/dependency-check-core/src/test/java/org/owasp/dependencycheck/data/lucene/FieldAnalyzerTest.java @@ -52,7 +52,7 @@ public class FieldAnalyzerTest extends BaseTest { @Test public void testAnalyzers() throws Exception { - Analyzer analyzer = new FieldAnalyzer(LuceneUtils.CURRENT_VERSION); + Analyzer analyzer = new SearchFieldAnalyzer(LuceneUtils.CURRENT_VERSION); Directory index = new RAMDirectory(); String field1 = "product"; @@ -93,12 +93,10 @@ public class FieldAnalyzerTest extends BaseTest { assertEquals("springframework", searcher.doc(hits[0].doc).get(field1)); assertEquals("springsource", searcher.doc(hits[0].doc).get(field2)); - searchAnalyzerProduct.clear(); //ensure we don't have anything left over from the previous search. - searchAnalyzerVendor.clear(); querystr = "product:(Apache Struts) vendor:(Apache)"; Query q2 = parser.parse(querystr); assertFalse("second parsing contains previousWord from the TokenPairConcatenatingFilter", q2.toString().contains("core")); - + querystr = "product:( x-stream^5 ) AND vendor:( thoughtworks.xstream )"; Query q3 = parser.parse(querystr); collector = TopScoreDocCollector.create(hitsPerPage, true); diff --git a/dependency-check-core/src/test/java/org/owasp/dependencycheck/data/lucene/TokenPairConcatenatingFilterTest.java b/dependency-check-core/src/test/java/org/owasp/dependencycheck/data/lucene/TokenPairConcatenatingFilterTest.java index d8f63b546..74715e9b0 100644 --- a/dependency-check-core/src/test/java/org/owasp/dependencycheck/data/lucene/TokenPairConcatenatingFilterTest.java +++ b/dependency-check-core/src/test/java/org/owasp/dependencycheck/data/lucene/TokenPairConcatenatingFilterTest.java @@ -26,7 +26,6 @@ import org.apache.lucene.analysis.core.WhitespaceTokenizer; import org.junit.After; import org.junit.Before; -import org.junit.Test; /** * @@ -55,22 +54,4 @@ public class TokenPairConcatenatingFilterTest extends BaseTokenStreamTestCase { assertTokenStreamContents(filter, new String[]{"one", "onetwo", "two", "twothree", "three"}); } - - /** - * Test of clear method, of class TokenPairConcatenatingFilter. - * - * @throws java.io.IOException - */ - @Test - public void testClear() throws IOException { - - TokenStream ts = new WhitespaceTokenizer(LuceneUtils.CURRENT_VERSION, new StringReader("one two three")); - TokenPairConcatenatingFilter filter = new TokenPairConcatenatingFilter(ts); - assertTokenStreamContents(filter, new String[]{"one", "onetwo", "two", "twothree", "three"}); - - assertNotNull(filter.getPreviousWord()); - filter.clear(); - assertNull(filter.getPreviousWord()); - assertTrue(filter.getWords().isEmpty()); - } } diff --git a/dependency-check-maven/src/main/java/org/owasp/dependencycheck/maven/CheckMojo.java b/dependency-check-maven/src/main/java/org/owasp/dependencycheck/maven/CheckMojo.java index 75b20e8e4..04d36feac 100644 --- a/dependency-check-maven/src/main/java/org/owasp/dependencycheck/maven/CheckMojo.java +++ b/dependency-check-maven/src/main/java/org/owasp/dependencycheck/maven/CheckMojo.java @@ -62,6 +62,7 @@ public class CheckMojo extends BaseDependencyCheckMojo { */ @Override public boolean canGenerateReport() { + populateSettings(); boolean isCapable = false; for (Artifact a : getProject().getArtifacts()) { if (!getArtifactScopeExcluded().passes(a.getScope())) { diff --git a/dependency-check-utils/src/main/java/org/owasp/dependencycheck/utils/Downloader.java b/dependency-check-utils/src/main/java/org/owasp/dependencycheck/utils/Downloader.java old mode 100644 new mode 100755 index 4f9bca2a9..14dc720a3 --- a/dependency-check-utils/src/main/java/org/owasp/dependencycheck/utils/Downloader.java +++ b/dependency-check-utils/src/main/java/org/owasp/dependencycheck/utils/Downloader.java @@ -154,7 +154,7 @@ public final class Downloader { } if ("Connection reset".equalsIgnoreCase(ex.getMessage())) { final String msg = format("TLS Connection Reset%nPlease see " - + "http://jeremylong.github.io/DependencyCheck/general/tlsfailures.html " + + "http://jeremylong.github.io/DependencyCheck/data/tlsfailure.html " + "for more information regarding how to resolve the issue."); LOGGER.error(msg); throw new DownloadFailedException(msg, ex);