View Javadoc

    1   /*
    2    * This file is part of dependency-check-core.
    3    *
    4    * Dependency-check-core is free software: you can redistribute it and/or modify it
    5    * under the terms of the GNU General Public License as published by the Free
    6    * Software Foundation, either version 3 of the License, or (at your option) any
    7    * later version.
    8    *
    9    * Dependency-check-core is distributed in the hope that it will be useful, but
    10   * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
    11   * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
    12   * details.
    13   *
    14   * You should have received a copy of the GNU General Public License along with
    15   * dependency-check-core. If not, see http://www.gnu.org/licenses/.
    16   *
    17   * Copyright (c) 2012 Jeremy Long. All Rights Reserved.
    18   */
    19  package org.owasp.dependencycheck.dependency;
    20
    21  import java.net.MalformedURLException;
    22  import java.util.HashSet;
    23  import java.util.Iterator;
    24  import java.util.List;
    25  import java.util.Set;
    26  import java.util.TreeSet;
    27  import java.util.logging.Level;
    28  import java.util.logging.Logger;
    29  import org.apache.commons.lang.StringUtils;
    30  import org.owasp.dependencycheck.utils.DependencyVersion;
    31  import org.owasp.dependencycheck.utils.DependencyVersionUtil;
    32  import org.owasp.dependencycheck.utils.Filter;
    33  import org.owasp.dependencycheck.utils.UrlStringUtils;
    34
    35  /**
    36   * Used to maintain a collection of Evidence.
    37   *
    38   * @author Jeremy Long (jeremy.long@owasp.org)
    39   */
    40  public class EvidenceCollection implements Iterable<Evidence> {
    41
    42      /**
    43       * Used to iterate over highest confidence evidence contained in the
    44       * collection.
    45       */
    46      private static final Filter<Evidence> HIGHEST_CONFIDENCE = new Filter<Evidence>() {
    47          public boolean passes(Evidence evidence) {
    48              return evidence.getConfidence() == Evidence.Confidence.HIGHEST;
    49          }
    50      };
    51      /**
    52       * Used to iterate over high confidence evidence contained in the
    53       * collection.
    54       */
    55      private static final Filter<Evidence> HIGH_CONFIDENCE = new Filter<Evidence>() {
    56          public boolean passes(Evidence evidence) {
    57              return evidence.getConfidence() == Evidence.Confidence.HIGH;
    58          }
    59      };
    60      /**
    61       * Used to iterate over medium confidence evidence contained in the
    62       * collection.
    63       */
    64      private static final Filter<Evidence> MEDIUM_CONFIDENCE = new Filter<Evidence>() {
    65          public boolean passes(Evidence evidence) {
    66              return evidence.getConfidence() == Evidence.Confidence.MEDIUM;
    67          }
    68      };
    69      /**
    70       * Used to iterate over low confidence evidence contained in the collection.
    71       */
    72      private static final Filter<Evidence> LOW_CONFIDENCE = new Filter<Evidence>() {
    73          public boolean passes(Evidence evidence) {
    74              return evidence.getConfidence() == Evidence.Confidence.LOW;
    75          }
    76      };
    77      /**
    78       * Used to iterate over evidence that has was used (aka read) from the
    79       * collection.
    80       */
    81      private static final Filter<Evidence> EVIDENCE_USED = new Filter<Evidence>() {
    82          public boolean passes(Evidence evidence) {
    83              return evidence.isUsed();
    84          }
    85      };
    86
    87      /**
    88       * Used to iterate over evidence of the specified confidence.
    89       *
    90       * @param confidence the confidence level for the evidence to be iterated
    91       * over.
    92       * @return Iterable<Evidence> an iterable collectoin of evidence
    93       */
    94      public final Iterable<Evidence> iterator(Evidence.Confidence confidence) {
    95          if (confidence == Evidence.Confidence.HIGHEST) {
    96              return EvidenceCollection.HIGHEST_CONFIDENCE.filter(this.list);
    97          } else if (confidence == Evidence.Confidence.HIGH) {
    98              return EvidenceCollection.HIGH_CONFIDENCE.filter(this.list);
    99          } else if (confidence == Evidence.Confidence.MEDIUM) {
    100             return EvidenceCollection.MEDIUM_CONFIDENCE.filter(this.list);
    101         } else {
    102             return EvidenceCollection.LOW_CONFIDENCE.filter(this.list);
    103         }
    104     }
    105     /**
    106      * A collection of evidence.
    107      */
    108     private final Set<Evidence> list;
    109     /**
    110      * A collection of strings used to adjust Lucene's term weighting.
    111      */
    112     private final Set<String> weightedStrings;
    113
    114     /**
    115      * Creates a new EvidenceCollection.
    116      */
    117     public EvidenceCollection() {
    118         list = new TreeSet<Evidence>();
    119         weightedStrings = new HashSet<String>();
    120     }
    121
    122     /**
    123      * Adds evidence to the collection.
    124      *
    125      * @param e Evidence.
    126      */
    127     public void addEvidence(Evidence e) {
    128         list.add(e);
    129     }
    130
    131     /**
    132      * Creates an Evidence object from the parameters and adds the resulting
    133      * object to the collection.
    134      *
    135      * @param source the source of the Evidence.
    136      * @param name the name of the Evidence.
    137      * @param value the value of the Evidence.
    138      * @param confidence the confidence of the Evidence.
    139      */
    140     public void addEvidence(String source, String name, String value, Evidence.Confidence confidence) {
    141         final Evidence e = new Evidence(source, name, value, confidence);
    142         addEvidence(e);
    143     }
    144
    145     /**
    146      * Adds term to the weighting collection. The terms added here are used
    147      * later to boost the score of other terms. This is a way of combining
    148      * evidence from multiple sources to boost the confidence of the given
    149      * evidence.
    150      *
    151      * Example: The term 'Apache' is found in the manifest of a JAR and is added
    152      * to the Collection. When we parse the package names within the JAR file we
    153      * may add these package names to the "weighted" strings collection to boost
    154      * the score in the Lucene query. That way when we construct the Lucene
    155      * query we find the term Apache in the collection AND in the weighted
    156      * strings; as such, we will boost the confidence of the term Apache.
    157      *
    158      * @param str to add to the weighting collection.
    159      */
    160     public void addWeighting(String str) {
    161         weightedStrings.add(str);
    162     }
    163
    164     /**
    165      * Returns a set of Weightings - a list of terms that are believed to be of
    166      * higher confidence when also found in another location.
    167      *
    168      * @return Set<String>
    169      */
    170     public Set<String> getWeighting() {
    171         return weightedStrings;
    172     }
    173
    174     /**
    175      * Returns the set of evidence.
    176      *
    177      * @return the set of evidence.
    178      */
    179     public Set<Evidence> getEvidence() {
    180         return list;
    181     }
    182
    183     /**
    184      * Returns the set of evidence from a given source.
    185      *
    186      * @param source the source of the evidence
    187      * @return the set of evidence.
    188      */
    189     public Set<Evidence> getEvidence(String source) {
    190         if (source == null) {
    191             return null;
    192         }
    193         final Set<Evidence> ret = new HashSet<Evidence>();
    194         for (Evidence e : list) {
    195             if (source.equals(e.getSource())) {
    196                 ret.add(e);
    197             }
    198         }
    199         return ret;
    200     }
    201
    202     /**
    203      * Returns the set of evidence from a given source and name.
    204      *
    205      * @param source the source of the evidence
    206      * @param name the name of the evidence to return
    207      * @return the set of evidence.
    208      */
    209     public Set<Evidence> getEvidence(String source, String name) {
    210         if (source == null || name == null) {
    211             return null;
    212         }
    213         final Set<Evidence> ret = new HashSet<Evidence>();
    214         for (Evidence e : list) {
    215             if (source.equals(e.getSource()) && name.equals(e.getName())) {
    216                 ret.add(e);
    217             }
    218         }
    219         return ret;
    220     }
    221
    222     /**
    223      * Implements the iterator interface for the Evidence Collection.
    224      *
    225      * @return an Iterator<Evidence>.
    226      */
    227     public Iterator<Evidence> iterator() {
    228         return list.iterator();
    229     }
    230
    231     /**
    232      * Used to determine if a given string was used (aka read).
    233      *
    234      * @param text the string to search for.
    235      * @return whether or not the string was used.
    236      */
    237     public boolean containsUsedString(String text) {
    238         if (text == null) {
    239             return false;
    240         }
    241         final String textToTest = text.toLowerCase();
    242
    243         for (Evidence e : EvidenceCollection.EVIDENCE_USED.filter(this)) {
    244             //TODO consider changing the regex to only compare alpha-numeric (i.e. strip everything else)
    245             final String value = urlCorrection(e.getValue().toLowerCase()).replaceAll("[\\s_-]", "");
    246             if (value.contains(textToTest)) {
    247                 return true;
    248             }
    249         }
    250         return false;
    251     }
    252
    253     /**
    254      * Used to determine if a given version was used (aka read) from the
    255      * EvidenceCollection.
    256      *
    257      * @param version the version to search for within the collected evidence.
    258      * @return whether or not the string was used.
    259      */
    260     public boolean containsUsedVersion(DependencyVersion version) {
    261         if (version == null) {
    262             return false;
    263         }
    264
    265         for (Evidence e : EvidenceCollection.EVIDENCE_USED.filter(this)) {
    266             final DependencyVersion value = DependencyVersionUtil.parseVersion(e.getValue());
    267             if (value != null && value.matchesAtLeastThreeLevels(version)) {
    268                 return true;
    269             }
    270         }
    271         return false;
    272     }
    273
    274     /**
    275      * Returns whether or not the collection contains evidence of a specified
    276      * Confidence.
    277      *
    278      * @param confidence A Confidence value.
    279      * @return boolean.
    280      */
    281     public boolean contains(Evidence.Confidence confidence) {
    282         for (Evidence e : list) {
    283             if (e.getConfidence().equals(confidence)) {
    284                 return true;
    285             }
    286         }
    287         return false;
    288     }
    289
    290     /**
    291      * Merges multiple EvidenceCollections together, only merging evidence that
    292      * was used, into a new EvidenceCollection.
    293      *
    294      * @param ec One or more EvidenceCollections.
    295      * @return a new EvidenceCollection containing the used evidence.
    296      */
    297     public static EvidenceCollection mergeUsed(EvidenceCollection... ec) {
    298         final EvidenceCollection ret = new EvidenceCollection();
    299         for (EvidenceCollection col : ec) {
    300             for (Evidence e : col.list) {
    301                 if (e.isUsed()) {
    302                     ret.addEvidence(e);
    303                 }
    304             }
    305         }
    306         return ret;
    307     }
    308
    309     /**
    310      * Merges multiple EvidenceCollections together.
    311      *
    312      * @param ec One or more EvidenceCollections.
    313      * @return a new EvidenceCollection.
    314      */
    315     public static EvidenceCollection merge(EvidenceCollection... ec) {
    316         final EvidenceCollection ret = new EvidenceCollection();
    317         for (EvidenceCollection col : ec) {
    318             ret.list.addAll(col.list);
    319             ret.weightedStrings.addAll(col.weightedStrings);
    320         }
    321         return ret;
    322     }
    323
    324     /**
    325      * Returns a string of evidence 'values'.
    326      *
    327      * @return a string containing the evidence.
    328      */
    329     @Override
    330     public String toString() {
    331         final StringBuilder sb = new StringBuilder();
    332         for (Evidence e : this.list) {
    333             sb.append(e.getValue()).append(' ');
    334         }
    335         return sb.toString();
    336     }
    337
    338     /**
    339      * Returns the number of elements in the EvidenceCollection.
    340      *
    341      * @return the number of elements in the collection.
    342      */
    343     public int size() {
    344         return list.size();
    345     }
    346
    347     /**
    348      * <p>Takes a string that may contain a fully qualified domain and it will
    349      * return the string having removed the query string, the protocol, the
    350      * sub-domain of 'www', and the file extension of the path.</p>
    351      * <p>This is useful for checking if the evidence contains a specific
    352      * string. The presence of the protocol, file extension, etc. may produce
    353      * false positives.
    354      *
    355      * <p>Example, given the following input:</p>
    356      * <code>'Please visit https://www.somedomain.com/path1/path2/file.php?id=439'</code>
    357      * <p>The function would return:</p>
    358      * <code>'Please visit somedomain path1 path2 file'</code>
    359      *
    360      * @param value the value that may contain a url
    361      * @return the modified string
    362      */
    363     private String urlCorrection(String value) {
    364         if (value == null || !UrlStringUtils.containsUrl(value)) {
    365             return value;
    366         }
    367         final StringBuilder sb = new StringBuilder(value.length());
    368         final String[] parts = value.split("\\s");
    369         for (String part : parts) {
    370             if (UrlStringUtils.isUrl(part)) {
    371                 try {
    372                     final List<String> data = UrlStringUtils.extractImportantUrlData(part);
    373                     sb.append(' ').append(StringUtils.join(data, ' '));
    374                 } catch (MalformedURLException ex) {
    375                     Logger.getLogger(EvidenceCollection.class.getName()).log(Level.INFO, "error parsing " + part, ex);
    376                     sb.append(' ').append(part);
    377                 }
    378             } else {
    379                 sb.append(' ').append(part);
    380             }
    381         }
    382         return sb.toString().trim();
    383     }
    384 }