1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.owasp.dependencycheck.data.lucene;
19
20 import java.io.IOException;
21 import java.util.HashMap;
22 import org.apache.lucene.analysis.Analyzer;
23 import org.apache.lucene.analysis.miscellaneous.PerFieldAnalyzerWrapper;
24 import org.apache.lucene.analysis.standard.StandardAnalyzer;
25 import org.apache.lucene.document.Document;
26 import org.apache.lucene.document.Field;
27 import org.apache.lucene.document.TextField;
28 import org.apache.lucene.index.DirectoryReader;
29 import org.apache.lucene.index.IndexReader;
30 import org.apache.lucene.index.IndexWriter;
31 import org.apache.lucene.index.IndexWriterConfig;
32 import org.apache.lucene.queryparser.classic.QueryParser;
33 import org.apache.lucene.search.IndexSearcher;
34 import org.apache.lucene.search.Query;
35 import org.apache.lucene.search.ScoreDoc;
36 import org.apache.lucene.search.TopScoreDocCollector;
37 import org.apache.lucene.store.Directory;
38 import org.apache.lucene.store.RAMDirectory;
39
40 import static org.junit.Assert.assertEquals;
41 import static org.junit.Assert.assertFalse;
42
43 import org.junit.Test;
44 import org.owasp.dependencycheck.BaseTest;
45
46
47
48
49
50 public class FieldAnalyzerTest extends BaseTest {
51
52 @Test
53 public void testAnalyzers() throws Exception {
54
55 Analyzer analyzer = new FieldAnalyzer(LuceneUtils.CURRENT_VERSION);
56 Directory index = new RAMDirectory();
57
58 String field1 = "product";
59 String text1 = "springframework";
60
61 String field2 = "vendor";
62 String text2 = "springsource";
63
64 IndexWriter w = createIndex(analyzer, index);
65 addDoc(w, field1, text1, field2, text2);
66 text1 = "x-stream";
67 text2 = "xstream";
68 addDoc(w, field1, text1, field2, text2);
69 w.close();
70
71
72 String querystr = "product:\"(Spring Framework Core)\" vendor:(SpringSource)";
73
74 SearchFieldAnalyzer searchAnalyzerProduct = new SearchFieldAnalyzer(LuceneUtils.CURRENT_VERSION);
75 SearchFieldAnalyzer searchAnalyzerVendor = new SearchFieldAnalyzer(LuceneUtils.CURRENT_VERSION);
76 HashMap<String, Analyzer> map = new HashMap<String, Analyzer>();
77 map.put(field1, searchAnalyzerProduct);
78 map.put(field2, searchAnalyzerVendor);
79 PerFieldAnalyzerWrapper wrapper = new PerFieldAnalyzerWrapper(new StandardAnalyzer(LuceneUtils.CURRENT_VERSION), map);
80 QueryParser parser = new QueryParser(LuceneUtils.CURRENT_VERSION, field1, wrapper);
81
82 Query q = parser.parse(querystr);
83
84 int hitsPerPage = 10;
85
86 IndexReader reader = DirectoryReader.open(index);
87 IndexSearcher searcher = new IndexSearcher(reader);
88 TopScoreDocCollector collector = TopScoreDocCollector.create(hitsPerPage, true);
89 searcher.search(q, collector);
90 ScoreDoc[] hits = collector.topDocs().scoreDocs;
91
92 assertEquals("Did not find 1 document?", 1, hits.length);
93 assertEquals("springframework", searcher.doc(hits[0].doc).get(field1));
94 assertEquals("springsource", searcher.doc(hits[0].doc).get(field2));
95
96 searchAnalyzerProduct.clear();
97 searchAnalyzerVendor.clear();
98 querystr = "product:(Apache Struts) vendor:(Apache)";
99 Query q2 = parser.parse(querystr);
100 assertFalse("second parsing contains previousWord from the TokenPairConcatenatingFilter", q2.toString().contains("core"));
101
102 querystr = "product:( x-stream^5 ) AND vendor:( thoughtworks.xstream )";
103 Query q3 = parser.parse(querystr);
104 collector = TopScoreDocCollector.create(hitsPerPage, true);
105 searcher.search(q3, collector);
106 hits = collector.topDocs().scoreDocs;
107 assertEquals("x-stream", searcher.doc(hits[0].doc).get(field1));
108 assertEquals("xstream", searcher.doc(hits[0].doc).get(field2));
109 }
110
111 private IndexWriter createIndex(Analyzer analyzer, Directory index) throws IOException {
112 IndexWriterConfig config = new IndexWriterConfig(LuceneUtils.CURRENT_VERSION, analyzer);
113 return new IndexWriter(index, config);
114 }
115
116 private static void addDoc(IndexWriter w, String field1, String text1, String field2, String text2) throws IOException {
117 Document doc = new Document();
118 doc.add(new TextField(field1, text1, Field.Store.YES));
119 doc.add(new TextField(field2, text2, Field.Store.YES));
120 w.addDocument(doc);
121 }
122 }