View Javadoc

    1   /*
    2    * To change this template, choose Tools | Templates
    3    * and open the template in the editor.
    4    */
    5   package org.owasp.dependencycheck.data.lucene;
    6
    7   import java.io.IOException;
    8   import java.io.Reader;
    9   import org.apache.lucene.analysis.Analyzer;
    10  import org.apache.lucene.analysis.Analyzer.TokenStreamComponents;
    11  import org.apache.lucene.analysis.BaseTokenStreamTestCase;
    12  import static org.apache.lucene.analysis.BaseTokenStreamTestCase.checkOneTerm;
    13  import org.apache.lucene.analysis.MockTokenizer;
    14  import org.apache.lucene.analysis.Tokenizer;
    15  import org.apache.lucene.analysis.core.KeywordTokenizer;
    16  import org.junit.After;
    17  import org.junit.AfterClass;
    18  import org.junit.Before;
    19  import org.junit.BeforeClass;
    20
    21  /**
    22   *
    23   * @author Jeremy Long (jeremy.long@owasp.org)
    24   */
    25  public class UrlTokenizingFilterTest extends BaseTokenStreamTestCase {
    26
    27      private Analyzer analyzer;
    28
    29      public UrlTokenizingFilterTest() {
    30          analyzer = new Analyzer() {
    31              @Override
    32              protected TokenStreamComponents createComponents(String fieldName,
    33                      Reader reader) {
    34                  Tokenizer source = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
    35                  return new TokenStreamComponents(source, new UrlTokenizingFilter(source));
    36              }
    37          };
    38      }
    39
    40      @BeforeClass
    41      public static void setUpClass() {
    42      }
    43
    44      @AfterClass
    45      public static void tearDownClass() {
    46      }
    47
    48      @Before
    49      public void setUp() throws Exception {
    50          super.setUp();
    51      }
    52
    53      @After
    54      public void tearDown() throws Exception {
    55          super.tearDown();
    56      }
    57
    58      /**
    59       * test some example domains
    60       */
    61      public void testExamples() throws IOException {
    62          String[] expected = new String[2];
    63          expected[0] = "domain";
    64          expected[1] = "test";
    65          assertAnalyzesTo(analyzer, "http://www.domain.com/test.php", expected);
    66          checkOneTerm(analyzer, "https://apache.org", "apache");
    67      }
    68
    69      /**
    70       * copied from
    71       * http://svn.apache.org/repos/asf/lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/en/TestEnglishMinimalStemFilter.java
    72       * blast some random strings through the analyzer
    73       */
    74      public void testRandomStrings() throws Exception {
    75          checkRandomData(random(), analyzer, 1000 * RANDOM_MULTIPLIER);
    76      }
    77
    78      /**
    79       * copied from
    80       * http://svn.apache.org/repos/asf/lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/en/TestEnglishMinimalStemFilter.java
    81       *
    82       * @throws IOException
    83       */
    84      public void testEmptyTerm() throws IOException {
    85          Analyzer a = new Analyzer() {
    86              @Override
    87              protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
    88                  Tokenizer tokenizer = new KeywordTokenizer(reader);
    89                  return new TokenStreamComponents(tokenizer, new UrlTokenizingFilter(tokenizer));
    90              }
    91          };
    92          checkOneTermReuse(a, "", "");
    93      }
    94  }