reverted to lucene 4.7.2 - new versions of lucene are built using JDK 1.7+ and cause issues for the dependency-check Maven Plugin

Former-commit-id: 514cc4922c7f00f55b1dcd102f6d45491e90a5d8

dependency-check-core/pom.xml

@@ -340,6 +340,7 @@ Copyright (c) 2012 Jeremy Long. All Rights Reserved.
         </plugins>
     </reporting>
     <dependencies>
+        <!-- Note, to stay compatible with Jenkins installations only JARs compiled to 1.6 can be used -->
         <dependency>
             <groupId>org.owasp</groupId>
             <artifactId>dependency-check-utils</artifactId>
@@ -679,6 +680,8 @@ Copyright (c) 2012 Jeremy Long. All Rights Reserved.
         </profile>
     </profiles>
     <properties>
-        <apache.lucene.version>4.10.3</apache.lucene.version>
+        <!-- new versions of lucene are compiled with JDK 1.7 and cannot be used ubiquitously in Jenkins
+        this, we cannot upgrade beyond 4.7.2 -->
+        <apache.lucene.version>4.7.2</apache.lucene.version>
     </properties>
 </project>

dependency-check-core/src/main/java/org/owasp/dependencycheck/data/cpe/CpeMemoryIndex.java

@@ -48,8 +48,8 @@ import org.owasp.dependencycheck.data.nvdcve.DatabaseException;
 import org.owasp.dependencycheck.utils.Pair;
 
 /**
- * An in memory lucene index that contains the vendor/product combinations from the CPE (application) identifiers within
+ * An in memory lucene index that contains the vendor/product combinations from the CPE (application) identifiers within the NVD
- * the NVD CVE data.
+ * CVE data.
  *
  * @author Jeremy Long <jeremy.long@owasp.org>
  */
@@ -125,7 +125,7 @@ public final class CpeMemoryIndex {
                 }
                 indexSearcher = new IndexSearcher(indexReader);
                 searchingAnalyzer = createSearchingAnalyzer();
-                queryParser = new QueryParser(Fields.DOCUMENT_KEY, searchingAnalyzer);
+                queryParser = new QueryParser(LuceneUtils.CURRENT_VERSION, Fields.DOCUMENT_KEY, searchingAnalyzer);
                 openState = true;
             }
         }
@@ -153,7 +153,7 @@ public final class CpeMemoryIndex {
     private Analyzer createIndexingAnalyzer() {
         final Map fieldAnalyzers = new HashMap();
         fieldAnalyzers.put(Fields.DOCUMENT_KEY, new KeywordAnalyzer());
-        return new PerFieldAnalyzerWrapper(new FieldAnalyzer(), fieldAnalyzers);
+        return new PerFieldAnalyzerWrapper(new FieldAnalyzer(LuceneUtils.CURRENT_VERSION), fieldAnalyzers);
     }
 
     /**
@@ -165,12 +165,12 @@ public final class CpeMemoryIndex {
     private Analyzer createSearchingAnalyzer() {
         final Map<String, Analyzer> fieldAnalyzers = new HashMap<String, Analyzer>();
         fieldAnalyzers.put(Fields.DOCUMENT_KEY, new KeywordAnalyzer());
-        productSearchFieldAnalyzer = new SearchFieldAnalyzer();
+        productSearchFieldAnalyzer = new SearchFieldAnalyzer(LuceneUtils.CURRENT_VERSION);
-        vendorSearchFieldAnalyzer = new SearchFieldAnalyzer();
+        vendorSearchFieldAnalyzer = new SearchFieldAnalyzer(LuceneUtils.CURRENT_VERSION);
         fieldAnalyzers.put(Fields.PRODUCT, productSearchFieldAnalyzer);
         fieldAnalyzers.put(Fields.VENDOR, vendorSearchFieldAnalyzer);
 
-        return new PerFieldAnalyzerWrapper(new FieldAnalyzer(), fieldAnalyzers);
+        return new PerFieldAnalyzerWrapper(new FieldAnalyzer(LuceneUtils.CURRENT_VERSION), fieldAnalyzers);
     }
 
     /**

dependency-check-core/src/main/java/org/owasp/dependencycheck/data/lucene/AlphaNumericTokenizer.java

@@ -19,6 +19,7 @@ package org.owasp.dependencycheck.data.lucene;
 
 import java.io.Reader;
 import org.apache.lucene.analysis.util.CharTokenizer;
+import org.apache.lucene.util.Version;
 
 /**
  * Tokenizes the input breaking it into tokens when non-alpha/numeric characters are found.
@@ -30,10 +31,22 @@ public class AlphaNumericTokenizer extends CharTokenizer {
     /**
      * Constructs a new AlphaNumericTokenizer.
      *
+     * @param matchVersion the lucene version
      * @param in the Reader
      */
-    public AlphaNumericTokenizer(Reader in) {
+    public AlphaNumericTokenizer(Version matchVersion, Reader in) {
-        super(in);
+        super(matchVersion, in);
+    }
+
+    /**
+     * Constructs a new AlphaNumericTokenizer.
+     *
+     * @param matchVersion the lucene version
+     * @param factory the AttributeFactory
+     * @param in the Reader
+     */
+    public AlphaNumericTokenizer(Version matchVersion, AttributeFactory factory, Reader in) {
+        super(matchVersion, factory, in);
     }
 
     /**

dependency-check-core/src/main/java/org/owasp/dependencycheck/data/lucene/FieldAnalyzer.java

@@ -25,21 +25,30 @@ import org.apache.lucene.analysis.core.LowerCaseFilter;
 import org.apache.lucene.analysis.core.StopAnalyzer;
 import org.apache.lucene.analysis.core.StopFilter;
 import org.apache.lucene.analysis.miscellaneous.WordDelimiterFilter;
+import org.apache.lucene.util.Version;
 
 /**
  * <p>
- * A Lucene Analyzer that utilizes the WhitespaceTokenizer, WordDelimiterFilter, LowerCaseFilter, and StopFilter. The
+ * A Lucene Analyzer that utilizes the WhitespaceTokenizer, WordDelimiterFilter, LowerCaseFilter, and StopFilter. The intended
- * intended purpose of this Analyzer is to index the CPE fields vendor and product.</p>
+ * purpose of this Analyzer is to index the CPE fields vendor and product.</p>
  *
  * @author Jeremy Long <jeremy.long@owasp.org>
  */
 public class FieldAnalyzer extends Analyzer {
 
+    /**
+     * The Lucene Version used.
+     */
+    private final Version version;
+
     /**
      * Creates a new FieldAnalyzer.
      *
+     * @param version the Lucene version
      */
-    public FieldAnalyzer() { }
+    public FieldAnalyzer(Version version) {
+        this.version = version;
+    }
 
     /**
      * Creates the TokenStreamComponents
@@ -50,7 +59,7 @@ public class FieldAnalyzer extends Analyzer {
      */
     @Override
     protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
-        final Tokenizer source = new AlphaNumericTokenizer(reader);
+        final Tokenizer source = new AlphaNumericTokenizer(version, reader);
 
         TokenStream stream = source;
 
@@ -63,8 +72,8 @@ public class FieldAnalyzer extends Analyzer {
                 | WordDelimiterFilter.SPLIT_ON_NUMERICS
                 | WordDelimiterFilter.STEM_ENGLISH_POSSESSIVE, null);
 
-        stream = new LowerCaseFilter(stream);
+        stream = new LowerCaseFilter(version, stream);
-        stream = new StopFilter(stream, StopAnalyzer.ENGLISH_STOP_WORDS_SET);
+        stream = new StopFilter(version, stream, StopAnalyzer.ENGLISH_STOP_WORDS_SET);
 
         return new TokenStreamComponents(source, stream);
     }

dependency-check-core/src/main/java/org/owasp/dependencycheck/data/lucene/LuceneUtils.java

@@ -29,10 +29,10 @@ import org.apache.lucene.util.Version;
 public final class LuceneUtils {
 
     /**
-     * The current version of Lucene being used. Declaring this one place so an upgrade doesn't require hunting through
+     * The current version of Lucene being used. Declaring this one place so an upgrade doesn't require hunting through the code
-     * the code base.
+     * base.
      */
-    public static final Version CURRENT_VERSION = Version.LATEST;
+    public static final Version CURRENT_VERSION = Version.LUCENE_47;
 
     /**
      * Private constructor as this is a utility class.

dependency-check-core/src/main/java/org/owasp/dependencycheck/data/lucene/SearchFieldAnalyzer.java

@@ -25,6 +25,7 @@ import org.apache.lucene.analysis.core.LowerCaseFilter;
 import org.apache.lucene.analysis.core.StopAnalyzer;
 import org.apache.lucene.analysis.core.StopFilter;
 import org.apache.lucene.analysis.miscellaneous.WordDelimiterFilter;
+import org.apache.lucene.util.Version;
 
 /**
  * A Lucene field analyzer used to analyzer queries against the CPE data.
@@ -34,16 +35,22 @@ import org.apache.lucene.analysis.miscellaneous.WordDelimiterFilter;
 public class SearchFieldAnalyzer extends Analyzer {
 
     /**
-     * A local reference to the TokenPairConcatenatingFilter so that we can clear any left over state if this analyzer
+     * The Lucene Version used.
-     * is re-used.
+     */
+    private final Version version;
+    /**
+     * A local reference to the TokenPairConcatenatingFilter so that we can clear any left over state if this analyzer is re-used.
      */
     private TokenPairConcatenatingFilter concatenatingFilter;
 
     /**
      * Constructs a new SearchFieldAnalyzer.
      *
+     * @param version the Lucene version
      */
-    public SearchFieldAnalyzer() {  }
+    public SearchFieldAnalyzer(Version version) {
+        this.version = version;
+    }
 
     /**
      * Creates a the TokenStreamComponents used to analyze the stream.
@@ -54,7 +61,7 @@ public class SearchFieldAnalyzer extends Analyzer {
      */
     @Override
     protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
-        final Tokenizer source = new AlphaNumericTokenizer(reader);
+        final Tokenizer source = new AlphaNumericTokenizer(version, reader);
 
         TokenStream stream = source;
 
@@ -66,19 +73,18 @@ public class SearchFieldAnalyzer extends Analyzer {
                 | WordDelimiterFilter.SPLIT_ON_NUMERICS
                 | WordDelimiterFilter.STEM_ENGLISH_POSSESSIVE, null);
 
-        stream = new LowerCaseFilter(stream);
+        stream = new LowerCaseFilter(version, stream);
         stream = new UrlTokenizingFilter(stream);
         concatenatingFilter = new TokenPairConcatenatingFilter(stream);
         stream = concatenatingFilter;
-        stream = new StopFilter(stream, StopAnalyzer.ENGLISH_STOP_WORDS_SET);
+        stream = new StopFilter(version, stream, StopAnalyzer.ENGLISH_STOP_WORDS_SET);
 
         return new TokenStreamComponents(source, stream);
     }
 
     /**
      * <p>
-     * Resets the analyzer and clears any internal state data that may have been left-over from previous uses of the
+     * Resets the analyzer and clears any internal state data that may have been left-over from previous uses of the analyzer.</p>
-     * analyzer.</p>
      * <p>
      * <b>If this analyzer is re-used this method must be called between uses.</b></p>
      */

dependency-check-core/src/test/java/org/owasp/dependencycheck/data/lucene/FieldAnalyzerTest.java

@@ -69,7 +69,7 @@ public class FieldAnalyzerTest {
     @Test
     public void testAnalyzers() throws Exception {
 
-        Analyzer analyzer = new FieldAnalyzer();
+        Analyzer analyzer = new FieldAnalyzer(LuceneUtils.CURRENT_VERSION);
         Directory index = new RAMDirectory();
 
         String field1 = "product";
@@ -83,13 +83,13 @@ public class FieldAnalyzerTest {
         //Analyzer searchingAnalyzer = new SearchFieldAnalyzer(LuceneUtils.CURRENT_VERSION);
         String querystr = "product:\"(Spring Framework Core)\" vendor:(SpringSource)";
 
-        SearchFieldAnalyzer searchAnalyzerProduct = new SearchFieldAnalyzer();
+        SearchFieldAnalyzer searchAnalyzerProduct = new SearchFieldAnalyzer(LuceneUtils.CURRENT_VERSION);
-        SearchFieldAnalyzer searchAnalyzerVendor = new SearchFieldAnalyzer();
+        SearchFieldAnalyzer searchAnalyzerVendor = new SearchFieldAnalyzer(LuceneUtils.CURRENT_VERSION);
         HashMap<String, Analyzer> map = new HashMap<String, Analyzer>();
         map.put(field1, searchAnalyzerProduct);
         map.put(field2, searchAnalyzerVendor);
-        PerFieldAnalyzerWrapper wrapper = new PerFieldAnalyzerWrapper(new StandardAnalyzer(), map);
+        PerFieldAnalyzerWrapper wrapper = new PerFieldAnalyzerWrapper(new StandardAnalyzer(LuceneUtils.CURRENT_VERSION), map);
-        QueryParser parser = new QueryParser(field1, wrapper);
+        QueryParser parser = new QueryParser(LuceneUtils.CURRENT_VERSION, field1, wrapper);
 
         Query q = parser.parse(querystr);
         //System.out.println(q.toString());

dependency-check-core/src/test/java/org/owasp/dependencycheck/data/lucene/TokenPairConcatenatingFilterTest.java

@@ -61,7 +61,7 @@ public class TokenPairConcatenatingFilterTest extends BaseTokenStreamTestCase {
      * test some examples
      */
     public void testExamples() throws IOException {
-        Tokenizer wsTokenizer = new WhitespaceTokenizer(new StringReader("one two three"));
+        Tokenizer wsTokenizer = new WhitespaceTokenizer(LuceneUtils.CURRENT_VERSION, new StringReader("one two three"));
         TokenStream filter = new TokenPairConcatenatingFilter(wsTokenizer);
         assertTokenStreamContents(filter,
                 new String[]{"one", "onetwo", "two", "twothree", "three"});
@@ -75,7 +75,7 @@ public class TokenPairConcatenatingFilterTest extends BaseTokenStreamTestCase {
     @Test
     public void testClear() throws IOException {
 
-        TokenStream ts = new WhitespaceTokenizer(new StringReader("one two three"));
+        TokenStream ts = new WhitespaceTokenizer(LuceneUtils.CURRENT_VERSION, new StringReader("one two three"));
         TokenPairConcatenatingFilter filter = new TokenPairConcatenatingFilter(ts);
         assertTokenStreamContents(filter, new String[]{"one", "onetwo", "two", "twothree", "three"});