View Javadoc

    1   /*
    2    * This file is part of dependency-check-core.
    3    *
    4    * Dependency-check-core is free software: you can redistribute it and/or modify it
    5    * under the terms of the GNU General Public License as published by the Free
    6    * Software Foundation, either version 3 of the License, or (at your option) any
    7    * later version.
    8    *
    9    * Dependency-check-core is distributed in the hope that it will be useful, but
    10   * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
    11   * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
    12   * details.
    13   *
    14   * You should have received a copy of the GNU General Public License along with
    15   * dependency-check-core. If not, see http://www.gnu.org/licenses/.
    16   *
    17   * Copyright (c) 2012 Jeremy Long. All Rights Reserved.
    18   */
    19  package org.owasp.dependencycheck.data.lucene;
    20
    21  import java.io.Reader;
    22  import org.apache.lucene.analysis.Analyzer;
    23  import org.apache.lucene.analysis.TokenStream;
    24  import org.apache.lucene.analysis.Tokenizer;
    25  import org.apache.lucene.analysis.core.LowerCaseFilter;
    26  import org.apache.lucene.analysis.core.StopAnalyzer;
    27  import org.apache.lucene.analysis.core.StopFilter;
    28  import org.apache.lucene.analysis.miscellaneous.WordDelimiterFilter;
    29  import org.apache.lucene.util.Version;
    30
    31  /**
    32   * A Lucene field analyzer used to analyzer queries against the CPE data.
    33   *
    34   * @author Jeremy Long (jeremy.long@owasp.org)
    35   */
    36  public class SearchFieldAnalyzer extends Analyzer {
    37
    38      /**
    39       * The Lucene Version used.
    40       */
    41      private final Version version;
    42      /**
    43       * A local reference to the TokenPairConcatenatingFilter so that we can
    44       * clear any left over state if this analyzer is re-used.
    45       */
    46      private TokenPairConcatenatingFilter concatenatingFilter;
    47
    48      /**
    49       * Constructs a new SearchFieldAnalyzer.
    50       *
    51       * @param version the Lucene version
    52       */
    53      public SearchFieldAnalyzer(Version version) {
    54          this.version = version;
    55      }
    56
    57      /**
    58       * Creates a the TokenStreamComponents used to analyze the stream.
    59       *
    60       * @param fieldName the field that this lucene analyzer will process
    61       * @param reader a reader containing the tokens
    62       * @return the token stream filter chain
    63       */
    64      @Override
    65      protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
    66          final Tokenizer source = new AlphaNumericTokenizer(version, reader);
    67
    68          TokenStream stream = source;
    69
    70          stream = new WordDelimiterFilter(stream,
    71                  WordDelimiterFilter.GENERATE_WORD_PARTS
    72                  | WordDelimiterFilter.GENERATE_NUMBER_PARTS
    73                  | WordDelimiterFilter.PRESERVE_ORIGINAL
    74                  | WordDelimiterFilter.SPLIT_ON_CASE_CHANGE
    75                  | WordDelimiterFilter.SPLIT_ON_NUMERICS
    76                  | WordDelimiterFilter.STEM_ENGLISH_POSSESSIVE, null);
    77
    78          stream = new LowerCaseFilter(version, stream);
    79          stream = new UrlTokenizingFilter(stream);
    80          concatenatingFilter = new TokenPairConcatenatingFilter(stream);
    81          stream = concatenatingFilter;
    82          stream = new StopFilter(version, stream, StopAnalyzer.ENGLISH_STOP_WORDS_SET);
    83
    84          return new TokenStreamComponents(source, stream);
    85      }
    86
    87      /**
    88       * <p>Resets the analyzer and clears any internal state data that may have
    89       * been left-over from previous uses of the analyzer.</p>
    90       * <p><b>If this analyzer is re-used this method must be called between
    91       * uses.</b></p>
    92       */
    93      public void clear() {
    94          if (concatenatingFilter != null) {
    95              concatenatingFilter.clear();
    96          }
    97      }
    98  }