View Javadoc

    1   /*
    2    * This file is part of dependency-check-core.
    3    *
    4    * Dependency-check-core is free software: you can redistribute it and/or modify it
    5    * under the terms of the GNU General Public License as published by the Free
    6    * Software Foundation, either version 3 of the License, or (at your option) any
    7    * later version.
    8    *
    9    * Dependency-check-core is distributed in the hope that it will be useful, but
    10   * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
    11   * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
    12   * details.
    13   *
    14   * You should have received a copy of the GNU General Public License along with
    15   * dependency-check-core. If not, see http://www.gnu.org/licenses/.
    16   *
    17   * Copyright (c) 2012 Jeremy Long. All Rights Reserved.
    18   */
    19  package org.owasp.dependencycheck.analyzer;
    20
    21  import java.io.File;
    22  import java.util.Enumeration;
    23  import java.util.logging.Level;
    24  import java.util.logging.Logger;
    25  import javax.xml.bind.JAXBException;
    26  import javax.xml.parsers.ParserConfigurationException;
    27  import org.owasp.dependencycheck.Engine;
    28  import org.owasp.dependencycheck.dependency.Dependency;
    29  import org.owasp.dependencycheck.dependency.Evidence;
    30  import org.owasp.dependencycheck.dependency.EvidenceCollection;
    31  import java.io.IOException;
    32  import java.io.InputStreamReader;
    33  import java.io.Reader;
    34  import java.util.ArrayList;
    35  import java.util.HashMap;
    36  import java.util.List;
    37  import java.util.Map;
    38  import java.util.Map.Entry;
    39  import java.util.Properties;
    40  import java.util.Set;
    41  import java.util.StringTokenizer;
    42  import java.util.jar.Attributes;
    43  import java.util.jar.JarEntry;
    44  import java.util.jar.JarFile;
    45  import java.util.jar.Manifest;
    46  import java.util.regex.Pattern;
    47  import java.util.zip.ZipEntry;
    48  import javax.xml.bind.JAXBContext;
    49  import javax.xml.bind.JAXBElement;
    50  import javax.xml.bind.Unmarshaller;
    51  import javax.xml.parsers.SAXParser;
    52  import javax.xml.parsers.SAXParserFactory;
    53  import javax.xml.transform.sax.SAXSource;
    54  import org.jsoup.Jsoup;
    55  import org.owasp.dependencycheck.jaxb.pom.MavenNamespaceFilter;
    56  import org.owasp.dependencycheck.jaxb.pom.generated.License;
    57  import org.owasp.dependencycheck.jaxb.pom.generated.Model;
    58  import org.owasp.dependencycheck.jaxb.pom.generated.Organization;
    59  import org.owasp.dependencycheck.utils.NonClosingStream;
    60  import org.xml.sax.InputSource;
    61  import org.xml.sax.SAXException;
    62  import org.xml.sax.XMLFilter;
    63  import org.xml.sax.XMLReader;
    64
    65  /**
    66   *
    67   * Used to load a JAR file and collect information that can be used to determine
    68   * the associated CPE.
    69   *
    70   * @author Jeremy Long (jeremy.long@owasp.org)
    71   */
    72  public class JarAnalyzer extends AbstractAnalyzer implements Analyzer {
    73
    74      //<editor-fold defaultstate="collapsed" desc="Constants and Member Variables">
    75      /**
    76       * The system independent newline character.
    77       */
    78      private static final String NEWLINE = System.getProperty("line.separator");
    79      /**
    80       * A list of values in the manifest to ignore as they only result in false
    81       * positives.
    82       */
    83      private static final Set<String> IGNORE_VALUES = newHashSet(
    84              "Sun Java System Application Server");
    85      /**
    86       * A list of elements in the manifest to ignore.
    87       */
    88      private static final Set<String> IGNORE_KEYS = newHashSet(
    89              "built-by",
    90              "created-by",
    91              "builtby",
    92              "createdby",
    93              "build-jdk",
    94              "buildjdk",
    95              "ant-version",
    96              "antversion",
    97              "import-package",
    98              "export-package",
    99              "importpackage",
    100             "exportpackage",
    101             "sealed",
    102             "manifest-version",
    103             "archiver-version",
    104             "manifestversion",
    105             "archiverversion",
    106             "classpath",
    107             "class-path",
    108             "tool",
    109             "bundle-manifestversion",
    110             "bundlemanifestversion",
    111             "include-resource");
    112     /**
    113      * item in some manifest, should be considered medium confidence.
    114      */
    115     private static final String BUNDLE_VERSION = "Bundle-Version"; //: 2.1.2
    116     /**
    117      * item in some manifest, should be considered medium confidence.
    118      */
    119     private static final String BUNDLE_DESCRIPTION = "Bundle-Description"; //: Apache Struts 2
    120     /**
    121      * item in some manifest, should be considered medium confidence.
    122      */
    123     private static final String BUNDLE_NAME = "Bundle-Name"; //: Struts 2 Core
    124     /**
    125      * item in some manifest, should be considered medium confidence.
    126      */
    127     private static final String BUNDLE_VENDOR = "Bundle-Vendor"; //: Apache Software Foundation
    128     /**
    129      * A pattern to detect HTML within text.
    130      */
    131     private static final Pattern HTML_DETECTION_PATTERN = Pattern.compile("\\<[a-z]+.*/?\\>", Pattern.CASE_INSENSITIVE);
    132     /**
    133      * The unmarshaller used to parse the pom.xml from a JAR file.
    134      */
    135     private Unmarshaller pomUnmarshaller;
    136     //</editor-fold>
    137
    138     /**
    139      * Constructs a new JarAnalyzer.
    140      */
    141     public JarAnalyzer() {
    142         try {
    143             final JAXBContext jaxbContext = JAXBContext.newInstance("org.owasp.dependencycheck.jaxb.pom.generated");
    144             pomUnmarshaller = jaxbContext.createUnmarshaller();
    145         } catch (JAXBException ex) { //guess we will just have a null pointer exception later...
    146             Logger.getLogger(JarAnalyzer.class.getName()).log(Level.SEVERE, "Unable to load parser. See the log for more details.");
    147             Logger.getLogger(JarAnalyzer.class.getName()).log(Level.FINE, null, ex);
    148         }
    149     }
    150     //<editor-fold defaultstate="collapsed" desc="All standard implmentation details of Analyzer">
    151     /**
    152      * The name of the analyzer.
    153      */
    154     private static final String ANALYZER_NAME = "Jar Analyzer";
    155     /**
    156      * The phase that this analyzer is intended to run in.
    157      */
    158     private static final AnalysisPhase ANALYSIS_PHASE = AnalysisPhase.INFORMATION_COLLECTION;
    159     /**
    160      * The set of file extensions supported by this analyzer.
    161      */
    162     private static final Set<String> EXTENSIONS = newHashSet("jar", "war");
    163
    164     /**
    165      * Returns a list of file EXTENSIONS supported by this analyzer.
    166      *
    167      * @return a list of file EXTENSIONS supported by this analyzer.
    168      */
    169     public Set<String> getSupportedExtensions() {
    170         return EXTENSIONS;
    171     }
    172
    173     /**
    174      * Returns the name of the analyzer.
    175      *
    176      * @return the name of the analyzer.
    177      */
    178     public String getName() {
    179         return ANALYZER_NAME;
    180     }
    181
    182     /**
    183      * Returns whether or not this analyzer can process the given extension.
    184      *
    185      * @param extension the file extension to test for support.
    186      * @return whether or not the specified file extension is supported by this
    187      * analyzer.
    188      */
    189     public boolean supportsExtension(String extension) {
    190         return EXTENSIONS.contains(extension);
    191     }
    192
    193     /**
    194      * Returns the phase that the analyzer is intended to run in.
    195      *
    196      * @return the phase that the analyzer is intended to run in.
    197      */
    198     public AnalysisPhase getAnalysisPhase() {
    199         return ANALYSIS_PHASE;
    200     }
    201     //</editor-fold>
    202
    203     /**
    204      * Loads a specified JAR file and collects information from the manifest and
    205      * checksums to identify the correct CPE information.
    206      *
    207      * @param dependency the dependency to analyze.
    208      * @param engine the engine that is scanning the dependencies
    209      * @throws AnalysisException is thrown if there is an error reading the JAR
    210      * file.
    211      */
    212     @Override
    213     public void analyze(Dependency dependency, Engine engine) throws AnalysisException {
    214         try {
    215             final ArrayList<ClassNameInformation> classNames = collectClassNames(dependency);
    216             final String fileName = dependency.getFileName().toLowerCase();
    217             if (classNames.isEmpty()
    218                     && (fileName.endsWith("-sources.jar")
    219                     || fileName.endsWith("-javadoc.jar")
    220                     || fileName.endsWith("-src.jar")
    221                     || fileName.endsWith("-doc.jar"))) {
    222                 engine.getDependencies().remove(dependency);
    223             }
    224             final boolean hasManifest = parseManifest(dependency, classNames);
    225             final boolean hasPOM = analyzePOM(dependency, classNames);
    226             final boolean addPackagesAsEvidence = !(hasManifest && hasPOM);
    227             analyzePackageNames(classNames, dependency, addPackagesAsEvidence);
    228         } catch (IOException ex) {
    229             throw new AnalysisException("Exception occurred reading the JAR file.", ex);
    230         }
    231     }
    232
    233     /**
    234      * Attempts to find a pom.xml within the JAR file. If found it extracts
    235      * information and adds it to the evidence. This will attempt to interpolate
    236      * the strings contained within the pom.properties if one exists.
    237      *
    238      * @param dependency the dependency being analyzed
    239      * @param classes a collection of class name information
    240      * @throws AnalysisException is thrown if there is an exception parsing the
    241      * pom
    242      * @return whether or not evidence was added to the dependency
    243      */
    244     protected boolean analyzePOM(Dependency dependency, ArrayList<ClassNameInformation> classes) throws AnalysisException {
    245         boolean foundSomething = false;
    246         final JarFile jar;
    247         try {
    248             jar = new JarFile(dependency.getActualFilePath());
    249         } catch (IOException ex) {
    250             final String msg = String.format("Unable to read JarFile '%s'.", dependency.getActualFilePath());
    251             final AnalysisException ax = new AnalysisException(msg, ex);
    252             dependency.getAnalysisExceptions().add(ax);
    253             Logger.getLogger(JarAnalyzer.class.getName()).log(Level.WARNING, msg);
    254             Logger.getLogger(JarAnalyzer.class.getName()).log(Level.FINE, null, ex);
    255             return false;
    256         }
    257         List<String> pomEntries;
    258         try {
    259             pomEntries = retrievePomListing(jar);
    260         } catch (IOException ex) {
    261             final String msg = String.format("Unable to read Jar file entries in '%s'.", dependency.getActualFilePath());
    262             final AnalysisException ax = new AnalysisException(msg, ex);
    263             dependency.getAnalysisExceptions().add(ax);
    264             Logger.getLogger(JarAnalyzer.class.getName()).log(Level.WARNING, msg);
    265             Logger.getLogger(JarAnalyzer.class.getName()).log(Level.INFO, msg, ex);
    266             return false;
    267         }
    268         if (pomEntries.isEmpty()) {
    269             return false;
    270         }
    271         if (pomEntries.size() > 1) { //need to sort out which pom we will use
    272             pomEntries = filterPomEntries(pomEntries, classes);
    273         }
    274         for (String path : pomEntries) {
    275             Properties pomProperties = null;
    276             try {
    277                 pomProperties = retrievePomProperties(path, jar);
    278             } catch (IOException ex) {
    279                 Logger.getLogger(JarAnalyzer.class.getName()).log(Level.FINEST, "ignore this, failed reading a non-existent pom.properties", ex);
    280             }
    281             Model pom = null;
    282             try {
    283                 pom = retrievePom(path, jar);
    284             } catch (JAXBException ex) {
    285                 final String msg = String.format("Unable to parse POM '%s' in '%s'",
    286                         path, dependency.getFilePath());
    287                 final AnalysisException ax = new AnalysisException(msg, ex);
    288                 dependency.getAnalysisExceptions().add(ax);
    289                 Logger.getLogger(JarAnalyzer.class.getName()).log(Level.FINE, msg, ax);
    290             } catch (IOException ex) {
    291                 final String msg = String.format("Unable to retrieve POM '%s' in '%s'",
    292                         path, dependency.getFilePath());
    293                 Logger.getLogger(JarAnalyzer.class.getName()).log(Level.FINE, msg, ex);
    294             }
    295             foundSomething = setPomEvidence(dependency, pom, pomProperties, classes) || foundSomething;
    296         }
    297         return foundSomething;
    298     }
    299
    300     /**
    301      * Given a path to a pom.xml within a JarFile, this method attempts to load
    302      * a sibling pom.properties if one exists.
    303      *
    304      * @param path the path to the pom.xml within the JarFile
    305      * @param jar the JarFile to load the pom.properties from
    306      * @return a Properties object or null if no pom.properties was found
    307      * @throws IOException thrown if there is an exception reading the
    308      * pom.properties
    309      */
    310     @edu.umd.cs.findbugs.annotations.SuppressWarnings(value = "OS_OPEN_STREAM",
    311             justification = "The reader is closed by closing the zipEntry")
    312     private Properties retrievePomProperties(String path, final JarFile jar) throws IOException {
    313         Properties pomProperties = null;
    314         final String propPath = path.substring(0, path.length() - 7) + "pom.properies";
    315         final ZipEntry propEntry = jar.getEntry(propPath);
    316         if (propEntry != null) {
    317             final Reader reader = new InputStreamReader(jar.getInputStream(propEntry), "UTF-8");
    318             pomProperties = new Properties();
    319             pomProperties.load(reader);
    320         }
    321         return pomProperties;
    322     }
    323
    324     /**
    325      * Searches a JarFile for pom.xml entries and returns a listing of these
    326      * entries.
    327      *
    328      * @param jar the JarFile to search
    329      * @return a list of pom.xml entries
    330      * @throws IOException thrown if there is an exception reading a JarEntryf
    331      */
    332     private List<String> retrievePomListing(final JarFile jar) throws IOException {
    333         final List<String> pomEntries = new ArrayList<String>();
    334         final Enumeration<JarEntry> entries = jar.entries();
    335         while (entries.hasMoreElements()) {
    336             final JarEntry entry = entries.nextElement();
    337             final String entryName = (new File(entry.getName())).getName().toLowerCase();
    338             if (!entry.isDirectory() && "pom.xml".equals(entryName)) {
    339                 pomEntries.add(entry.getName());
    340             }
    341         }
    342         return pomEntries;
    343     }
    344
    345     /**
    346      * Retrieves the specified POM from a jar file and converts it to a Model.
    347      *
    348      * @param path the path to the pom.xml file within the jar file
    349      * @param jar the jar file to extract the pom from
    350      * @return returns a
    351      * {@link org.owasp.dependencycheck.analyzer.pom.generated.Model} object
    352      * @throws JAXBException is thrown if there is an exception parsing the pom
    353      * @throws IOException is thrown if there is an exception reading the jar
    354      */
    355     private Model retrievePom(String path, JarFile jar) throws JAXBException, IOException {
    356         final ZipEntry entry = jar.getEntry(path);
    357         if (entry != null) { //should never be null
    358             Model m = null;
    359             try {
    360                 final XMLFilter filter = new MavenNamespaceFilter();
    361                 final SAXParserFactory spf = SAXParserFactory.newInstance();
    362                 final SAXParser sp = spf.newSAXParser();
    363                 final XMLReader xr = sp.getXMLReader();
    364                 filter.setParent(xr);
    365                 final NonClosingStream stream = new NonClosingStream(jar.getInputStream(entry));
    366                 final InputStreamReader reader = new InputStreamReader(stream, "UTF-8");
    367                 final InputSource xml = new InputSource(reader);
    368                 final SAXSource source = new SAXSource(filter, xml);
    369                 final JAXBElement<Model> el = pomUnmarshaller.unmarshal(source, Model.class);
    370                 m = el.getValue();
    371             } catch (ParserConfigurationException ex) {
    372                 final String msg = String.format("Unable to parse pom '%s' in jar '%s'", path, jar.getName());
    373                 Logger.getLogger(JarAnalyzer.class.getName()).log(Level.FINE, msg, ex);
    374             } catch (SAXException ex) {
    375                 final String msg = String.format("Unable to parse pom '%s' in jar '%s'", path, jar.getName());
    376                 Logger.getLogger(JarAnalyzer.class.getName()).log(Level.FINE, msg, ex);
    377             }
    378             return m;
    379         }
    380         return null;
    381     }
    382
    383     /**
    384      * Sets evidence from the pom on the supplied dependency.
    385      *
    386      * @param dependency the dependency to set data on
    387      * @param pom the information from the pom
    388      * @param pomProperties the pom properties file (null if none exists)
    389      * @param classes a collection of ClassNameInformation - containing data
    390      * about the fully qualified class names within the JAR file being analyzed
    391      * @return true if there was evidence within the pom that we could use;
    392      * otherwise false
    393      */
    394     private boolean setPomEvidence(Dependency dependency, Model pom, Properties pomProperties, ArrayList<ClassNameInformation> classes) {
    395         boolean foundSomething = false;
    396         if (pom == null) {
    397             return foundSomething;
    398         }
    399         String groupid = interpolateString(pom.getGroupId(), pomProperties);
    400         if (groupid != null && !groupid.isEmpty()) {
    401             if (groupid.startsWith("org.") || groupid.startsWith("com.")) {
    402                 groupid = groupid.substring(4);
    403             }
    404             foundSomething = true;
    405             dependency.getVendorEvidence().addEvidence("pom", "groupid", groupid, Evidence.Confidence.HIGH);
    406             dependency.getProductEvidence().addEvidence("pom", "groupid", groupid, Evidence.Confidence.LOW);
    407             addMatchingValues(classes, groupid, dependency.getVendorEvidence());
    408             addMatchingValues(classes, groupid, dependency.getProductEvidence());
    409         }
    410         String artifactid = interpolateString(pom.getArtifactId(), pomProperties);
    411         if (artifactid != null && !artifactid.isEmpty()) {
    412             if (artifactid.startsWith("org.") || artifactid.startsWith("com.")) {
    413                 artifactid = artifactid.substring(4);
    414             }
    415             foundSomething = true;
    416             dependency.getProductEvidence().addEvidence("pom", "artifactid", artifactid, Evidence.Confidence.HIGH);
    417             dependency.getVendorEvidence().addEvidence("pom", "artifactid", artifactid, Evidence.Confidence.LOW);
    418             addMatchingValues(classes, artifactid, dependency.getVendorEvidence());
    419             addMatchingValues(classes, artifactid, dependency.getProductEvidence());
    420         }
    421         //version
    422         final String version = interpolateString(pom.getVersion(), pomProperties);
    423         if (version != null && !version.isEmpty()) {
    424             foundSomething = true;
    425             dependency.getVersionEvidence().addEvidence("pom", "version", version, Evidence.Confidence.HIGHEST);
    426         }
    427         // org name
    428         final Organization org = pom.getOrganization();
    429         if (org != null && org.getName() != null) {
    430             foundSomething = true;
    431             final String orgName = interpolateString(org.getName(), pomProperties);
    432             if (orgName != null && !orgName.isEmpty()) {
    433                 dependency.getVendorEvidence().addEvidence("pom", "organization name", orgName, Evidence.Confidence.HIGH);
    434                 addMatchingValues(classes, orgName, dependency.getVendorEvidence());
    435             }
    436         }
    437         //pom name
    438         final String pomName = interpolateString(pom.getName(), pomProperties);
    439         if (pomName != null && !pomName.isEmpty()) {
    440             foundSomething = true;
    441             dependency.getProductEvidence().addEvidence("pom", "name", pomName, Evidence.Confidence.HIGH);
    442             dependency.getVendorEvidence().addEvidence("pom", "name", pomName, Evidence.Confidence.HIGH);
    443             addMatchingValues(classes, pomName, dependency.getVendorEvidence());
    444             addMatchingValues(classes, pomName, dependency.getProductEvidence());
    445         }
    446
    447         //Description
    448         if (pom.getDescription() != null) {
    449             foundSomething = true;
    450             final String description = interpolateString(pom.getDescription(), pomProperties);
    451             if (description != null && !description.isEmpty()) {
    452                 addDescription(dependency, description, "pom", "description");
    453                 addMatchingValues(classes, description, dependency.getVendorEvidence());
    454                 addMatchingValues(classes, description, dependency.getProductEvidence());
    455             }
    456         }
    457
    458         //license
    459         if (pom.getLicenses() != null) {
    460             String license = null;
    461             for (License lic : pom.getLicenses().getLicense()) {
    462                 String tmp = null;
    463                 if (lic.getName() != null) {
    464                     tmp = interpolateString(lic.getName(), pomProperties);
    465                 }
    466                 if (lic.getUrl() != null) {
    467                     if (tmp == null) {
    468                         tmp = interpolateString(lic.getUrl(), pomProperties);
    469                     } else {
    470                         tmp += ": " + interpolateString(lic.getUrl(), pomProperties);
    471                     }
    472                 }
    473                 if (tmp == null) {
    474                     continue;
    475                 }
    476                 if (HTML_DETECTION_PATTERN.matcher(tmp).find()) {
    477                     tmp = Jsoup.parse(tmp).text();
    478                 }
    479                 if (license == null) {
    480                     license = tmp;
    481                 } else {
    482                     license += "\n" + tmp;
    483                 }
    484             }
    485             if (license != null) {
    486                 dependency.setLicense(license);
    487             }
    488         }
    489         return foundSomething;
    490     }
    491
    492     /**
    493      * Analyzes the path information of the classes contained within the
    494      * JarAnalyzer to try and determine possible vendor or product names. If any
    495      * are found they are stored in the packageVendor and packageProduct
    496      * hashSets.
    497      *
    498      * @param classNames a list of class names
    499      * @param dependency a dependency to analyze
    500      * @param addPackagesAsEvidence a flag indicating whether or not package
    501      * names should be added as evidence.
    502      */
    503     protected void analyzePackageNames(ArrayList<ClassNameInformation> classNames,
    504             Dependency dependency, boolean addPackagesAsEvidence) {
    505         final HashMap<String, Integer> vendorIdentifiers = new HashMap<String, Integer>();
    506         final HashMap<String, Integer> productIdentifiers = new HashMap<String, Integer>();
    507         analyzeFullyQualifiedClassNames(classNames, vendorIdentifiers, productIdentifiers);
    508
    509         final int classCount = classNames.size();
    510         final EvidenceCollection vendor = dependency.getVendorEvidence();
    511         final EvidenceCollection product = dependency.getProductEvidence();
    512
    513         for (Map.Entry<String, Integer> entry : vendorIdentifiers.entrySet()) {
    514             final float ratio = entry.getValue() / (float) classCount;
    515             if (ratio > 0.5) {
    516                 //TODO remove weighting
    517                 vendor.addWeighting(entry.getKey());
    518                 if (addPackagesAsEvidence && entry.getKey().length() > 1) {
    519                     vendor.addEvidence("jar", "package", entry.getKey(), Evidence.Confidence.LOW);
    520                 }
    521             }
    522         }
    523         for (Map.Entry<String, Integer> entry : productIdentifiers.entrySet()) {
    524             final float ratio = entry.getValue() / (float) classCount;
    525             if (ratio > 0.5) {
    526                 product.addWeighting(entry.getKey());
    527                 if (addPackagesAsEvidence && entry.getKey().length() > 1) {
    528                     product.addEvidence("jar", "package", entry.getKey(), Evidence.Confidence.LOW);
    529                 }
    530             }
    531         }
    532     }
    533
    534     /**
    535      * <p>Reads the manifest from the JAR file and collects the entries. Some
    536      * vendorKey entries are:</p> <ul><li>Implementation Title</li>
    537      * <li>Implementation Version</li> <li>Implementation Vendor</li>
    538      * <li>Implementation VendorId</li> <li>Bundle Name</li> <li>Bundle
    539      * Version</li> <li>Bundle Vendor</li> <li>Bundle Description</li> <li>Main
    540      * Class</li> </ul>
    541      * However, all but a handful of specific entries are read in.
    542      *
    543      * @param dependency A reference to the dependency
    544      * @param classInformation a collection of class information
    545      * @return whether evidence was identified parsing the manifest
    546      * @throws IOException if there is an issue reading the JAR file
    547      */
    548     protected boolean parseManifest(Dependency dependency, ArrayList<ClassNameInformation> classInformation) throws IOException {
    549         boolean foundSomething = false;
    550         JarFile jar = null;
    551         try {
    552             jar = new JarFile(dependency.getActualFilePath());
    553
    554             final Manifest manifest = jar.getManifest();
    555             if (manifest == null) {
    556                 //don't log this for javadoc or sources jar files
    557                 if (!dependency.getFileName().toLowerCase().endsWith("-sources.jar")
    558                         && !dependency.getFileName().toLowerCase().endsWith("-javadoc.jar")
    559                         && !dependency.getFileName().toLowerCase().endsWith("-src.jar")
    560                         && !dependency.getFileName().toLowerCase().endsWith("-doc.jar")) {
    561                     Logger.getLogger(JarAnalyzer.class.getName()).log(Level.INFO,
    562                             String.format("Jar file '%s' does not contain a manifest.",
    563                             dependency.getFileName()));
    564                 }
    565                 return false;
    566             }
    567             final Attributes atts = manifest.getMainAttributes();
    568
    569             final EvidenceCollection vendorEvidence = dependency.getVendorEvidence();
    570             final EvidenceCollection productEvidence = dependency.getProductEvidence();
    571             final EvidenceCollection versionEvidence = dependency.getVersionEvidence();
    572
    573             final String source = "Manifest";
    574
    575             for (Entry<Object, Object> entry : atts.entrySet()) {
    576                 String key = entry.getKey().toString();
    577                 String value = atts.getValue(key);
    578                 if (HTML_DETECTION_PATTERN.matcher(value).find()) {
    579                     value = Jsoup.parse(value).text();
    580                 }
    581                 if (IGNORE_VALUES.contains(value)) {
    582                     continue;
    583                 } else if (key.equalsIgnoreCase(Attributes.Name.IMPLEMENTATION_TITLE.toString())) {
    584                     foundSomething = true;
    585                     productEvidence.addEvidence(source, key, value, Evidence.Confidence.HIGH);
    586                     addMatchingValues(classInformation, value, productEvidence);
    587                 } else if (key.equalsIgnoreCase(Attributes.Name.IMPLEMENTATION_VERSION.toString())) {
    588                     foundSomething = true;
    589                     versionEvidence.addEvidence(source, key, value, Evidence.Confidence.HIGH);
    590                 } else if (key.equalsIgnoreCase(Attributes.Name.IMPLEMENTATION_VENDOR.toString())) {
    591                     foundSomething = true;
    592                     vendorEvidence.addEvidence(source, key, value, Evidence.Confidence.HIGH);
    593                     addMatchingValues(classInformation, value, vendorEvidence);
    594                 } else if (key.equalsIgnoreCase(Attributes.Name.IMPLEMENTATION_VENDOR_ID.toString())) {
    595                     foundSomething = true;
    596                     vendorEvidence.addEvidence(source, key, value, Evidence.Confidence.MEDIUM);
    597                     addMatchingValues(classInformation, value, vendorEvidence);
    598                 } else if (key.equalsIgnoreCase(BUNDLE_DESCRIPTION)) {
    599                     foundSomething = true;
    600                     addDescription(dependency, value, "manifest", key);
    601                     //productEvidence.addEvidence(source, key, value, Evidence.Confidence.MEDIUM);
    602                     addMatchingValues(classInformation, value, productEvidence);
    603                 } else if (key.equalsIgnoreCase(BUNDLE_NAME)) {
    604                     foundSomething = true;
    605                     productEvidence.addEvidence(source, key, value, Evidence.Confidence.MEDIUM);
    606                     addMatchingValues(classInformation, value, productEvidence);
    607                 } else if (key.equalsIgnoreCase(BUNDLE_VENDOR)) {
    608                     foundSomething = true;
    609                     vendorEvidence.addEvidence(source, key, value, Evidence.Confidence.HIGH);
    610                     addMatchingValues(classInformation, value, vendorEvidence);
    611                 } else if (key.equalsIgnoreCase(BUNDLE_VERSION)) {
    612                     foundSomething = true;
    613                     versionEvidence.addEvidence(source, key, value, Evidence.Confidence.HIGH);
    614                 } else if (key.equalsIgnoreCase(Attributes.Name.MAIN_CLASS.toString())) {
    615                     continue;
    616                     //skipping main class as if this has important information to add
    617                     // it will be added during class name analysis...  if other fields
    618                     // have the information from the class name then they will get added...
    619 //                    foundSomething = true;
    620 //                    productEvidence.addEvidence(source, key, value, Evidence.Confidence.MEDIUM);
    621 //                    vendorEvidence.addEvidence(source, key, value, Evidence.Confidence.MEDIUM);
    622 //                    addMatchingValues(classInformation, value, vendorEvidence);
    623 //                    addMatchingValues(classInformation, value, productEvidence);
    624                 } else {
    625                     key = key.toLowerCase();
    626
    627                     if (!IGNORE_KEYS.contains(key)
    628                             && !key.endsWith("jdk")
    629                             && !key.contains("lastmodified")
    630                             && !key.endsWith("package")
    631                             && !key.endsWith("classpath")
    632                             && !key.endsWith("class-path")
    633                             && !key.endsWith("-scm") //todo change this to a regex?
    634                             && !key.startsWith("scm-")
    635                             && !isImportPackage(key, value)
    636                             && !isPackage(key, value)) {
    637
    638                         foundSomething = true;
    639                         if (key.contains("version")) {
    640                             if (key.contains("specification")) {
    641                                 versionEvidence.addEvidence(source, key, value, Evidence.Confidence.LOW);
    642                             } else {
    643                                 versionEvidence.addEvidence(source, key, value, Evidence.Confidence.MEDIUM);
    644                             }
    645
    646                         } else if (key.contains("title")) {
    647                             productEvidence.addEvidence(source, key, value, Evidence.Confidence.MEDIUM);
    648                             addMatchingValues(classInformation, value, productEvidence);
    649                         } else if (key.contains("vendor")) {
    650                             if (key.contains("specification")) {
    651                                 vendorEvidence.addEvidence(source, key, value, Evidence.Confidence.LOW);
    652                             } else {
    653                                 vendorEvidence.addEvidence(source, key, value, Evidence.Confidence.MEDIUM);
    654                                 addMatchingValues(classInformation, value, vendorEvidence);
    655                             }
    656                         } else if (key.contains("name")) {
    657                             productEvidence.addEvidence(source, key, value, Evidence.Confidence.MEDIUM);
    658                             vendorEvidence.addEvidence(source, key, value, Evidence.Confidence.MEDIUM);
    659                             addMatchingValues(classInformation, value, vendorEvidence);
    660                             addMatchingValues(classInformation, value, productEvidence);
    661                         } else if (key.contains("license")) {
    662                             addLicense(dependency, value);
    663                         } else {
    664                             if (key.contains("description")) {
    665                                 addDescription(dependency, value, "manifest", key);
    666                             } else {
    667                                 productEvidence.addEvidence(source, key, value, Evidence.Confidence.LOW);
    668                                 vendorEvidence.addEvidence(source, key, value, Evidence.Confidence.LOW);
    669                                 addMatchingValues(classInformation, value, vendorEvidence);
    670                                 addMatchingValues(classInformation, value, productEvidence);
    671                                 if (value.matches(".*\\d.*")) {
    672                                     final StringTokenizer tokenizer = new StringTokenizer(value, " ");
    673                                     while (tokenizer.hasMoreElements()) {
    674                                         final String s = tokenizer.nextToken();
    675                                         if (s.matches("^[0-9.]+$")) {
    676                                             versionEvidence.addEvidence(source, key, s, Evidence.Confidence.LOW);
    677                                         }
    678                                     }
    679                                 }
    680                             }
    681                         }
    682                     }
    683                 }
    684             }
    685         } finally {
    686             if (jar != null) {
    687                 jar.close();
    688             }
    689         }
    690         return foundSomething;
    691     }
    692
    693     /**
    694      * Adds a description to the given dependency.
    695      *
    696      * @param dependency a dependency
    697      * @param description the description
    698      * @param source the source of the evidence
    699      * @param key the "name" of the evidence
    700      */
    701     private void addDescription(Dependency dependency, String description, String source, String key) {
    702         if (dependency.getDescription() == null) {
    703             dependency.setDescription(description);
    704         }
    705         String desc;
    706         if (HTML_DETECTION_PATTERN.matcher(description).find()) {
    707             desc = Jsoup.parse(description).text();
    708         } else {
    709             desc = description;
    710         }
    711         dependency.setDescription(desc);
    712         if (desc.length() > 100) {
    713             final int posSuchAs = desc.toLowerCase().indexOf("such as ", 100);
    714             final int posLike = desc.toLowerCase().indexOf("like ", 100);
    715             int pos = -1;
    716             if (posLike > 0 && posSuchAs > 0) {
    717                 pos = posLike > posSuchAs ? posLike : posSuchAs;
    718             } else if (posLike > 0) {
    719                 pos = posLike;
    720             } else if (posSuchAs > 0) {
    721                 pos = posSuchAs;
    722             }
    723             String descToUse = desc;
    724             if (pos > 0) {
    725                 final StringBuilder sb = new StringBuilder(pos + 3);
    726                 sb.append(desc.substring(0, pos));
    727                 sb.append("...");
    728                 descToUse = sb.toString();
    729             }
    730             dependency.getProductEvidence().addEvidence(source, key, descToUse, Evidence.Confidence.LOW);
    731             dependency.getVendorEvidence().addEvidence(source, key, descToUse, Evidence.Confidence.LOW);
    732         } else {
    733             dependency.getProductEvidence().addEvidence(source, key, desc, Evidence.Confidence.MEDIUM);
    734             dependency.getVendorEvidence().addEvidence(source, key, desc, Evidence.Confidence.MEDIUM);
    735         }
    736     }
    737
    738     /**
    739      * Adds a license to the given dependency.
    740      *
    741      * @param d a dependency
    742      * @param license the license
    743      */
    744     private void addLicense(Dependency d, String license) {
    745         if (d.getLicense() == null) {
    746             d.setLicense(license);
    747         } else if (!d.getLicense().contains(license)) {
    748             d.setLicense(d.getLicense() + NEWLINE + license);
    749         }
    750     }
    751
    752     /**
    753      * The initialize method does nothing for this Analyzer.
    754      */
    755     public void initialize() {
    756         //do nothing
    757     }
    758
    759     /**
    760      * The close method does nothing for this Analyzer.
    761      */
    762     public void close() {
    763         //do nothing
    764     }
    765
    766     /**
    767      * <p>A utility function that will interpolate strings based on values given
    768      * in the properties file. It will also interpolate the strings contained
    769      * within the properties file so that properties can reference other
    770      * properties.</p>
    771      * <p><b>Note:</b> if there is no property found the reference will be
    772      * removed. In other words, if the interpolated string will be replaced with
    773      * an empty string.
    774      * </p>
    775      * <p>Example:</p>
    776      * <code>
    777      * Properties p = new Properties();
    778      * p.setProperty("key", "value");
    779      * String s = interpolateString("'${key}' and '${nothing}'", p);
    780      * System.out.println(s);
    781      * </code>
    782      * <p>Will result in:</p>
    783      * <code>
    784      * 'value' and ''
    785      * </code>
    786      *
    787      * @param text the string that contains references to properties.
    788      * @param properties a collection of properties that may be referenced
    789      * within the text.
    790      * @return the interpolated text.
    791      */
    792     protected String interpolateString(String text, Properties properties) {
    793         Properties props = properties;
    794         if (text == null) {
    795             return text;
    796         }
    797         if (props == null) {
    798             props = new Properties();
    799         }
    800
    801         final int pos = text.indexOf("${");
    802         if (pos < 0) {
    803             return text;
    804         }
    805         final int end = text.indexOf("}");
    806         if (end < pos) {
    807             return text;
    808         }
    809
    810         final String propName = text.substring(pos + 2, end);
    811         String propValue = interpolateString(props.getProperty(propName), props);
    812         if (propValue == null) {
    813             propValue = "";
    814         }
    815         final StringBuilder sb = new StringBuilder(propValue.length() + text.length());
    816         sb.append(text.subSequence(0, pos));
    817         sb.append(propValue);
    818         sb.append(text.substring(end + 1));
    819         return interpolateString(sb.toString(), props); //yes yes, this should be a loop...
    820     }
    821
    822     /**
    823      * Determines if the key value pair from the manifest is for an "import"
    824      * type entry for package names.
    825      *
    826      * @param key the key from the manifest
    827      * @param value the value from the manifest
    828      * @return true or false depending on if it is believed the entry is an
    829      * "import" entry
    830      */
    831     private boolean isImportPackage(String key, String value) {
    832         final Pattern packageRx = Pattern.compile("^((([a-zA-Z_#\\$0-9]\\.)+)\\s*\\;\\s*)+$");
    833         if (packageRx.matcher(value).matches()) {
    834             return (key.contains("import") || key.contains("include"));
    835         }
    836         return false;
    837     }
    838
    839     /**
    840      * Cycles through an enumeration of JarEntries, contained within the
    841      * dependency, and returns a list of the class names. This does not include
    842      * core Java package names (i.e. java.* or javax.*).
    843      *
    844      * @param dependency the dependency being analyzed
    845      * @return an list of fully qualified class names
    846      */
    847     private ArrayList<ClassNameInformation> collectClassNames(Dependency dependency) {
    848         final ArrayList<ClassNameInformation> classNames = new ArrayList<ClassNameInformation>();
    849         JarFile jar = null;
    850         try {
    851             jar = new JarFile(dependency.getActualFilePath());
    852             final Enumeration entries = jar.entries();
    853             while (entries.hasMoreElements()) {
    854                 final JarEntry entry = (JarEntry) entries.nextElement();
    855                 final String name = entry.getName().toLowerCase();
    856                 //no longer stripping "|com\\.sun" - there are some com.sun jar files with CVEs.
    857                 if (name.endsWith(".class") && !name.matches("^javax?\\..*$")) {
    858                     final ClassNameInformation className = new ClassNameInformation(name.substring(0, name.length() - 6));
    859                     classNames.add(className);
    860                 }
    861             }
    862         } catch (IOException ex) {
    863             final String msg = String.format("Unable to open jar file '%s'.", dependency.getFileName());
    864             Logger.getLogger(JarAnalyzer.class.getName()).log(Level.WARNING, msg);
    865             Logger.getLogger(JarAnalyzer.class.getName()).log(Level.FINE, null, ex);
    866         } finally {
    867             if (jar != null) {
    868                 try {
    869                     jar.close();
    870                 } catch (IOException ex) {
    871                     Logger.getLogger(JarAnalyzer.class.getName()).log(Level.FINEST, null, ex);
    872                 }
    873             }
    874         }
    875         return classNames;
    876     }
    877
    878     /**
    879      * Cycles through the list of class names and places the package levels 0-3
    880      * into the provided maps for vendor and product. This is helpful when
    881      * analyzing vendor/product as many times this is included in the package
    882      * name.
    883      *
    884      * @param classNames a list of class names
    885      * @param vendor HashMap of possible vendor names from package names (e.g.
    886      * owasp)
    887      * @param product HashMap of possible product names from package names (e.g.
    888      * dependencycheck)
    889      */
    890     private void analyzeFullyQualifiedClassNames(ArrayList<ClassNameInformation> classNames,
    891             HashMap<String, Integer> vendor, HashMap<String, Integer> product) {
    892         for (ClassNameInformation entry : classNames) {
    893             final ArrayList<String> list = entry.getPackageStructure();
    894             addEntry(vendor, list.get(0));
    895
    896             if (list.size() == 2) {
    897                 addEntry(product, list.get(1));
    898             }
    899             if (list.size() == 3) {
    900                 addEntry(vendor, list.get(1));
    901                 addEntry(product, list.get(1));
    902                 addEntry(product, list.get(2));
    903             }
    904             if (list.size() >= 4) {
    905                 addEntry(vendor, list.get(1));
    906                 addEntry(vendor, list.get(2));
    907                 addEntry(product, list.get(1));
    908                 addEntry(product, list.get(2));
    909                 addEntry(product, list.get(3));
    910             }
    911         }
    912     }
    913
    914     /**
    915      * Adds an entry to the specified collection and sets the Integer (e.g. the
    916      * count) to 1. If the entry already exists in the collection then the
    917      * Integer is incremented by 1.
    918      *
    919      * @param collection a collection of strings and their occurrence count
    920      * @param key the key to add to the collection
    921      */
    922     private void addEntry(HashMap<String, Integer> collection, String key) {
    923         if (collection.containsKey(key)) {
    924             collection.put(key, collection.get(key) + 1);
    925         } else {
    926             collection.put(key, 1);
    927         }
    928     }
    929
    930     /**
    931      * Cycles through the collection of class name information to see if parts
    932      * of the package names are contained in the provided value. If found, it
    933      * will be added as the HIGHEST confidence evidence because we have more
    934      * then one source corroborating the value.
    935      *
    936      * @param classes a collection of class name information
    937      * @param value the value to check to see if it contains a package name
    938      * @param evidence the evidence collection to add new entries too
    939      */
    940     private void addMatchingValues(ArrayList<ClassNameInformation> classes, String value, EvidenceCollection evidence) {
    941         if (value == null || value.isEmpty()) {
    942             return;
    943         }
    944         final String text = value.toLowerCase();
    945         for (ClassNameInformation cni : classes) {
    946             for (String key : cni.getPackageStructure()) {
    947                 if (text.contains(key)) { //note, package structure elements are already lowercase.
    948                     evidence.addEvidence("jar", "package name", key, Evidence.Confidence.HIGHEST);
    949                 }
    950             }
    951         }
    952     }
    953
    954     /**
    955      * <p><b>This is currently a failed implementation.</b> Part of the issue is
    956      * I was trying to solve the wrong problem. Instead of multiple POMs being
    957      * in the JAR to just add information about dependencies - I didn't realize
    958      * until later that I was looking at an uber-jar (aka fat-jar) that included
    959      * all of its dependencies.</p>
    960      * <p>I'm leaving this method in the source tree, entirely commented out
    961      * until a solution https://github.com/jeremylong/DependencyCheck/issues/11
    962      * has been implemented.</p>
    963      * <p>Takes a list of pom entries from a JAR file and attempts to filter it
    964      * down to the pom related to the jar (rather then the pom entry for a
    965      * dependency).</p>
    966      *
    967      * @param pomEntries a list of pom entries
    968      * @param classes a list of fully qualified classes from the JAR file
    969      * @return the list of pom entries that are associated with the jar being
    970      * analyzed rather then the dependent poms
    971      */
    972     private List<String> filterPomEntries(List<String> pomEntries, ArrayList<ClassNameInformation> classes) {
    973         return pomEntries;
    974 //        final HashMap<String, Integer> usePoms = new HashMap<String, Integer>();
    975 //        final ArrayList<String> possiblePoms = new ArrayList<String>();
    976 //        for (String entry : pomEntries) {
    977 //            //todo validate that the starts with is correct... or does it start with a ./ or /?
    978 //            // is it different on different platforms?
    979 //            if (entry.startsWith("META-INF/maven/")) {
    980 //                //trim the meta-inf/maven and pom.xml...
    981 //                final String pomPath = entry.substring(15, entry.length() - 8).toLowerCase();
    982 //                final String[] parts = pomPath.split("/");
    983 //                if (parts == null || parts.length != 2) { //misplaced pom?
    984 //                    //TODO add logging to FINE
    985 //                    possiblePoms.add(entry);
    986 //                }
    987 //                parts[0] = parts[0].replace('.', '/');
    988 //                parts[1] = parts[1].replace('.', '/');
    989 //                for (ClassNameInformation cni : classes) {
    990 //                    final String name = cni.getName();
    991 //                    if (StringUtils.containsIgnoreCase(name, parts[0])) {
    992 //                        addEntry(usePoms, entry);
    993 //                    }
    994 //                    if (StringUtils.containsIgnoreCase(name, parts[1])) {
    995 //                        addEntry(usePoms, entry);
    996 //                    }
    997 //                }
    998 //            } else { // we have a JAR file with an incorrect POM layout...
    999 //                //TODO add logging to FINE
    1000 //                possiblePoms.add(entry);
    1001 //            }
    1002 //        }
    1003 //        List<String> retValue;
    1004 //        if (usePoms.isEmpty()) {
    1005 //            if (possiblePoms.isEmpty()) {
    1006 //                retValue = pomEntries;
    1007 //            } else {
    1008 //                retValue = possiblePoms;
    1009 //            }
    1010 //        } else {
    1011 //            retValue = new ArrayList<String>();
    1012 //            int maxCount = 0;
    1013 //            for (Map.Entry<String, Integer> entry : usePoms.entrySet()) {
    1014 //                final int current = entry.getValue().intValue();
    1015 //                if (current > maxCount) {
    1016 //                    maxCount = current;
    1017 //                    retValue.clear();
    1018 //                    retValue.add(entry.getKey());
    1019 //                } else if (current == maxCount) {
    1020 //                    retValue.add(entry.getKey());
    1021 //                }
    1022 //            }
    1023 //        }
    1024 //        return retValue;
    1025     }
    1026
    1027     /**
    1028      * Simple check to see if the attribute from a manifest is just a package
    1029      * name.
    1030      *
    1031      * @param key the key of the value to check
    1032      * @param value the value to check
    1033      * @return true if the value looks like a java package name, otherwise false
    1034      */
    1035     private boolean isPackage(String key, String value) {
    1036
    1037         return !key.matches(".*(version|title|vendor|name|license|description).*")
    1038                 && value.matches("^([a-zA-Z_][a-zA-Z0-9_\\$]*(\\.[a-zA-Z_][a-zA-Z0-9_\\$]*)*)?$");
    1039     }
    1040
    1041     /**
    1042      * Stores information about a class name.
    1043      */
    1044     protected static class ClassNameInformation {
    1045
    1046         /**
    1047          * Stores information about a given class name. This class will keep the
    1048          * fully qualified class name and a list of the important parts of the
    1049          * package structure. Up to the first four levels of the package
    1050          * structure are stored, excluding a leading "org" or "com". Example:
    1051          * <code>ClassNameInformation obj = new ClassNameInformation("org.owasp.dependencycheck.analyzer.JarAnalyzer");
    1052          * System.out.println(obj.getName());
    1053          * for (String p : obj.getPackageStructure())
    1054          *     System.out.println(p);
    1055          * </code> Would result in:
    1056          * <code>org.owasp.dependencycheck.analyzer.JarAnalyzer
    1057          * owasp
    1058          * dependencycheck
    1059          * analyzer
    1060          * jaranalyzer</code>
    1061          *
    1062          * @param className a fully qualified class name
    1063          */
    1064         ClassNameInformation(String className) {
    1065             name = className;
    1066             if (name.contains("/")) {
    1067                 final String[] tmp = className.toLowerCase().split("/");
    1068                 int start = 0;
    1069                 int end = 3;
    1070                 if ("com".equals(tmp[0]) || "org".equals(tmp[0])) {
    1071                     start = 1;
    1072                     end = 4;
    1073                 }
    1074                 if (tmp.length <= end) {
    1075                     end = tmp.length - 1;
    1076                 }
    1077                 for (int i = start; i <= end; i++) {
    1078                     packageStructure.add(tmp[i]);
    1079                 }
    1080             } else {
    1081                 packageStructure.add(name);
    1082             }
    1083         }
    1084         /**
    1085          * The fully qualified class name.
    1086          */
    1087         private String name;
    1088
    1089         /**
    1090          * Get the value of name
    1091          *
    1092          * @return the value of name
    1093          */
    1094         public String getName() {
    1095             return name;
    1096         }
    1097
    1098         /**
    1099          * Set the value of name
    1100          *
    1101          * @param name new value of name
    1102          */
    1103         public void setName(String name) {
    1104             this.name = name;
    1105         }
    1106         /**
    1107          * Up to the first four levels of the package structure, excluding a
    1108          * leading "org" or "com".
    1109          */
    1110         private ArrayList<String> packageStructure = new ArrayList<String>();
    1111
    1112         /**
    1113          * Get the value of packageStructure
    1114          *
    1115          * @return the value of packageStructure
    1116          */
    1117         public ArrayList<String> getPackageStructure() {
    1118             return packageStructure;
    1119         }
    1120     }
    1121 }