Coverage Report - org.owasp.dependencycheck.analyzer.PythonDistributionAnalyzer
 
Classes in this File Line Coverage Branch Coverage Complexity
PythonDistributionAnalyzer
86%
80/92
65%
30/46
3.308
 
 1  
 /*
 2  
  * This file is part of dependency-check-core.
 3  
  *
 4  
  * Licensed under the Apache License, Version 2.0 (the "License");
 5  
  * you may not use this file except in compliance with the License.
 6  
  * You may obtain a copy of the License at
 7  
  *
 8  
  *     http://www.apache.org/licenses/LICENSE-2.0
 9  
  *
 10  
  * Unless required by applicable law or agreed to in writing, software
 11  
  * distributed under the License is distributed on an "AS IS" BASIS,
 12  
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 13  
  * See the License for the specific language governing permissions and
 14  
  * limitations under the License.
 15  
  *
 16  
  * Copyright (c) 2015 Institute for Defense Analyses. All Rights Reserved.
 17  
  */
 18  
 package org.owasp.dependencycheck.analyzer;
 19  
 
 20  
 import java.io.BufferedInputStream;
 21  
 import java.io.File;
 22  
 import java.io.FileFilter;
 23  
 import java.io.FileInputStream;
 24  
 import java.io.FileNotFoundException;
 25  
 import java.io.FilenameFilter;
 26  
 import org.apache.commons.io.filefilter.NameFileFilter;
 27  
 import org.apache.commons.io.filefilter.SuffixFileFilter;
 28  
 import org.apache.commons.io.input.AutoCloseInputStream;
 29  
 import org.apache.commons.lang.StringUtils;
 30  
 import org.owasp.dependencycheck.Engine;
 31  
 import org.owasp.dependencycheck.analyzer.exception.AnalysisException;
 32  
 import org.owasp.dependencycheck.dependency.Confidence;
 33  
 import org.owasp.dependencycheck.dependency.Dependency;
 34  
 import org.owasp.dependencycheck.dependency.EvidenceCollection;
 35  
 import org.slf4j.Logger;
 36  
 import org.slf4j.LoggerFactory;
 37  
 
 38  
 import javax.mail.MessagingException;
 39  
 import javax.mail.internet.InternetHeaders;
 40  
 import org.owasp.dependencycheck.utils.ExtractionException;
 41  
 import org.owasp.dependencycheck.utils.ExtractionUtil;
 42  
 import org.owasp.dependencycheck.utils.FileFilterBuilder;
 43  
 import org.owasp.dependencycheck.utils.FileUtils;
 44  
 import org.owasp.dependencycheck.utils.Settings;
 45  
 import org.owasp.dependencycheck.utils.UrlStringUtils;
 46  
 
 47  
 /**
 48  
  * Used to analyze a Wheel or egg distribution files, or their contents in unzipped form, and collect information that can be used
 49  
  * to determine the associated CPE.
 50  
  *
 51  
  * @author Dale Visser <dvisser@ida.org>
 52  
  */
 53  88
 public class PythonDistributionAnalyzer extends AbstractFileTypeAnalyzer {
 54  
 
 55  
     /**
 56  
      * Name of egg metatdata files to analyze.
 57  
      */
 58  
     private static final String PKG_INFO = "PKG-INFO";
 59  
 
 60  
     /**
 61  
      * Name of wheel metadata files to analyze.
 62  
      */
 63  
     private static final String METADATA = "METADATA";
 64  
 
 65  
     /**
 66  
      * The logger.
 67  
      */
 68  8
     private static final Logger LOGGER = LoggerFactory
 69  
             .getLogger(PythonDistributionAnalyzer.class);
 70  
 
 71  
     /**
 72  
      * The count of directories created during analysis. This is used for creating temporary directories.
 73  
      */
 74  8
     private static int dirCount = 0;
 75  
 
 76  
     /**
 77  
      * The name of the analyzer.
 78  
      */
 79  
     private static final String ANALYZER_NAME = "Python Distribution Analyzer";
 80  
     /**
 81  
      * The phase that this analyzer is intended to run in.
 82  
      */
 83  8
     private static final AnalysisPhase ANALYSIS_PHASE = AnalysisPhase.INFORMATION_COLLECTION;
 84  
 
 85  
     /**
 86  
      * The set of file extensions supported by this analyzer.
 87  
      */
 88  8
     private static final String[] EXTENSIONS = {"whl", "egg", "zip"};
 89  
 
 90  
     /**
 91  
      * Used to match on egg archive candidate extensions.
 92  
      */
 93  8
     private static final FileFilter EGG_OR_ZIP = FileFilterBuilder.newInstance().addExtensions("egg", "zip").build();
 94  
 
 95  
     /**
 96  
      * Used to detect files with a .whl extension.
 97  
      */
 98  8
     private static final FileFilter WHL_FILTER = FileFilterBuilder.newInstance().addExtensions("whl").build();
 99  
 
 100  
     /**
 101  
      * The parent directory for the individual directories per archive.
 102  
      */
 103  
     private File tempFileLocation;
 104  
 
 105  
     /**
 106  
      * Filter that detects *.dist-info files (but doesn't verify they are directories.
 107  
      */
 108  8
     private static final FilenameFilter DIST_INFO_FILTER = new SuffixFileFilter(
 109  
             ".dist-info");
 110  
 
 111  
     /**
 112  
      * Filter that detects files named "METADATA".
 113  
      */
 114  8
     private static final FilenameFilter EGG_INFO_FILTER = new NameFileFilter(
 115  
             "EGG-INFO");
 116  
 
 117  
     /**
 118  
      * Filter that detects files named "METADATA".
 119  
      */
 120  8
     private static final NameFileFilter METADATA_FILTER = new NameFileFilter(
 121  
             METADATA);
 122  
 
 123  
     /**
 124  
      * Filter that detects files named "PKG-INFO".
 125  
      */
 126  8
     private static final NameFileFilter PKG_INFO_FILTER = new NameFileFilter(
 127  
             PKG_INFO);
 128  
 
 129  
     /**
 130  
      * The file filter used to determine which files this analyzer supports.
 131  
      */
 132  8
     private static final FileFilter FILTER = FileFilterBuilder.newInstance().addFileFilters(
 133  
             METADATA_FILTER, PKG_INFO_FILTER).addExtensions(EXTENSIONS).build();
 134  
 
 135  
     /**
 136  
      * Returns the FileFilter
 137  
      *
 138  
      * @return the FileFilter
 139  
      */
 140  
     @Override
 141  
     protected FileFilter getFileFilter() {
 142  6864
         return FILTER;
 143  
     }
 144  
 
 145  
     /**
 146  
      * Returns the name of the analyzer.
 147  
      *
 148  
      * @return the name of the analyzer.
 149  
      */
 150  
     @Override
 151  
     public String getName() {
 152  40
         return ANALYZER_NAME;
 153  
     }
 154  
 
 155  
     /**
 156  
      * Returns the phase that the analyzer is intended to run in.
 157  
      *
 158  
      * @return the phase that the analyzer is intended to run in.
 159  
      */
 160  
     @Override
 161  
     public AnalysisPhase getAnalysisPhase() {
 162  16
         return ANALYSIS_PHASE;
 163  
     }
 164  
 
 165  
     /**
 166  
      * Returns the key used in the properties file to reference the analyzer's enabled property.
 167  
      *
 168  
      * @return the analyzer's enabled property setting key
 169  
      */
 170  
     @Override
 171  
     protected String getAnalyzerEnabledSettingKey() {
 172  88
         return Settings.KEYS.ANALYZER_PYTHON_DISTRIBUTION_ENABLED;
 173  
     }
 174  
 
 175  
     @Override
 176  
     protected void analyzeFileType(Dependency dependency, Engine engine)
 177  
             throws AnalysisException {
 178  48
         final File actualFile = dependency.getActualFile();
 179  48
         if (WHL_FILTER.accept(actualFile)) {
 180  8
             collectMetadataFromArchiveFormat(dependency, DIST_INFO_FILTER,
 181  
                     METADATA_FILTER);
 182  40
         } else if (EGG_OR_ZIP.accept(actualFile)) {
 183  16
             collectMetadataFromArchiveFormat(dependency, EGG_INFO_FILTER,
 184  
                     PKG_INFO_FILTER);
 185  
         } else {
 186  24
             final String name = actualFile.getName();
 187  24
             final boolean metadata = METADATA.equals(name);
 188  24
             if (metadata || PKG_INFO.equals(name)) {
 189  24
                 final File parent = actualFile.getParentFile();
 190  24
                 final String parentName = parent.getName();
 191  24
                 dependency.setDisplayFileName(parentName + "/" + name);
 192  24
                 if (parent.isDirectory()
 193  
                         && (metadata && parentName.endsWith(".dist-info")
 194  
                         || parentName.endsWith(".egg-info") || "EGG-INFO"
 195  
                         .equals(parentName))) {
 196  24
                     collectWheelMetadata(dependency, actualFile);
 197  
                 }
 198  
             }
 199  
         }
 200  48
     }
 201  
 
 202  
     /**
 203  
      * Collects the meta data from an archive.
 204  
      *
 205  
      * @param dependency the archive being scanned
 206  
      * @param folderFilter the filter to apply to the folder
 207  
      * @param metadataFilter the filter to apply to the meta data
 208  
      * @throws AnalysisException thrown when there is a problem analyzing the dependency
 209  
      */
 210  
     private void collectMetadataFromArchiveFormat(Dependency dependency,
 211  
             FilenameFilter folderFilter, FilenameFilter metadataFilter)
 212  
             throws AnalysisException {
 213  24
         final File temp = getNextTempDirectory();
 214  24
         LOGGER.debug("{} exists? {}", temp, temp.exists());
 215  
         try {
 216  24
             ExtractionUtil.extractFilesUsingFilter(
 217  
                     new File(dependency.getActualFilePath()), temp,
 218  
                     metadataFilter);
 219  0
         } catch (ExtractionException ex) {
 220  0
             throw new AnalysisException(ex);
 221  24
         }
 222  
 
 223  24
         collectWheelMetadata(
 224  
                 dependency,
 225  
                 getMatchingFile(getMatchingFile(temp, folderFilter),
 226  
                         metadataFilter));
 227  24
     }
 228  
 
 229  
     /**
 230  
      * Makes sure a usable temporary directory is available.
 231  
      *
 232  
      * @throws Exception an AnalyzeException is thrown when the temp directory cannot be created
 233  
      */
 234  
     @Override
 235  
     protected void initializeFileTypeAnalyzer() throws Exception {
 236  64
         final File baseDir = Settings.getTempDirectory();
 237  64
         tempFileLocation = File.createTempFile("check", "tmp", baseDir);
 238  64
         if (!tempFileLocation.delete()) {
 239  0
             final String msg = String.format(
 240  
                     "Unable to delete temporary file '%s'.",
 241  
                     tempFileLocation.getAbsolutePath());
 242  0
             throw new AnalysisException(msg);
 243  
         }
 244  64
         if (!tempFileLocation.mkdirs()) {
 245  0
             final String msg = String.format(
 246  
                     "Unable to create directory '%s'.",
 247  
                     tempFileLocation.getAbsolutePath());
 248  0
             throw new AnalysisException(msg);
 249  
         }
 250  64
     }
 251  
 
 252  
     /**
 253  
      * Deletes any files extracted from the Wheel during analysis.
 254  
      */
 255  
     @Override
 256  
     public void close() {
 257  72
         if (tempFileLocation != null && tempFileLocation.exists()) {
 258  64
             LOGGER.debug("Attempting to delete temporary files");
 259  64
             final boolean success = FileUtils.delete(tempFileLocation);
 260  64
             if (!success) {
 261  8
                 LOGGER.warn(
 262  
                         "Failed to delete some temporary files, see the log for more details");
 263  
             }
 264  
         }
 265  72
     }
 266  
 
 267  
     /**
 268  
      * Gathers evidence from the METADATA file.
 269  
      *
 270  
      * @param dependency the dependency being analyzed
 271  
      * @param file a reference to the manifest/properties file
 272  
      * @throws AnalysisException thrown when there is an error
 273  
      */
 274  
     private static void collectWheelMetadata(Dependency dependency, File file)
 275  
             throws AnalysisException {
 276  48
         final InternetHeaders headers = getManifestProperties(file);
 277  48
         addPropertyToEvidence(headers, dependency.getVersionEvidence(),
 278  
                 "Version", Confidence.HIGHEST);
 279  48
         addPropertyToEvidence(headers, dependency.getProductEvidence(), "Name",
 280  
                 Confidence.HIGHEST);
 281  48
         final String url = headers.getHeader("Home-page", null);
 282  48
         final EvidenceCollection vendorEvidence = dependency
 283  
                 .getVendorEvidence();
 284  48
         if (StringUtils.isNotBlank(url)) {
 285  48
             if (UrlStringUtils.isUrl(url)) {
 286  48
                 vendorEvidence.addEvidence(METADATA, "vendor", url,
 287  
                         Confidence.MEDIUM);
 288  
             }
 289  
         }
 290  48
         addPropertyToEvidence(headers, vendorEvidence, "Author", Confidence.LOW);
 291  48
         final String summary = headers.getHeader("Summary", null);
 292  48
         if (StringUtils.isNotBlank(summary)) {
 293  48
             JarAnalyzer
 294  
                     .addDescription(dependency, summary, METADATA, "summary");
 295  
         }
 296  48
     }
 297  
 
 298  
     /**
 299  
      * Adds a value to the evidence collection.
 300  
      *
 301  
      * @param headers the properties collection
 302  
      * @param evidence the evidence collection to add the value
 303  
      * @param property the property name
 304  
      * @param confidence the confidence of the evidence
 305  
      */
 306  
     private static void addPropertyToEvidence(InternetHeaders headers,
 307  
             EvidenceCollection evidence, String property, Confidence confidence) {
 308  144
         final String value = headers.getHeader(property, null);
 309  144
         LOGGER.debug("Property: {}, Value: {}", property, value);
 310  144
         if (StringUtils.isNotBlank(value)) {
 311  144
             evidence.addEvidence(METADATA, property, value, confidence);
 312  
         }
 313  144
     }
 314  
 
 315  
     /**
 316  
      * Returns a list of files that match the given filter, this does not recursively scan the directory.
 317  
      *
 318  
      * @param folder the folder to filter
 319  
      * @param filter the filter to apply to the files in the directory
 320  
      * @return the list of Files in the directory that match the provided filter
 321  
      */
 322  
     private static File getMatchingFile(File folder, FilenameFilter filter) {
 323  48
         File result = null;
 324  48
         final File[] matches = folder.listFiles(filter);
 325  48
         if (null != matches && 1 == matches.length) {
 326  48
             result = matches[0];
 327  
         }
 328  48
         return result;
 329  
     }
 330  
 
 331  
     /**
 332  
      * Reads the manifest entries from the provided file.
 333  
      *
 334  
      * @param manifest the manifest
 335  
      * @return the manifest entries
 336  
      */
 337  
     private static InternetHeaders getManifestProperties(File manifest) {
 338  48
         final InternetHeaders result = new InternetHeaders();
 339  48
         if (null == manifest) {
 340  0
             LOGGER.debug("Manifest file not found.");
 341  
         } else {
 342  
             try {
 343  48
                 result.load(new AutoCloseInputStream(new BufferedInputStream(
 344  
                         new FileInputStream(manifest))));
 345  0
             } catch (MessagingException e) {
 346  0
                 LOGGER.warn(e.getMessage(), e);
 347  0
             } catch (FileNotFoundException e) {
 348  0
                 LOGGER.warn(e.getMessage(), e);
 349  48
             }
 350  
         }
 351  48
         return result;
 352  
     }
 353  
 
 354  
     /**
 355  
      * Retrieves the next temporary destingation directory for extracting an archive.
 356  
      *
 357  
      * @return a directory
 358  
      * @throws AnalysisException thrown if unable to create temporary directory
 359  
      */
 360  
     private File getNextTempDirectory() throws AnalysisException {
 361  
         File directory;
 362  
 
 363  
         // getting an exception for some directories not being able to be
 364  
         // created; might be because the directory already exists?
 365  
         do {
 366  24
             dirCount += 1;
 367  24
             directory = new File(tempFileLocation, String.valueOf(dirCount));
 368  24
         } while (directory.exists());
 369  24
         if (!directory.mkdirs()) {
 370  0
             throw new AnalysisException(String.format(
 371  
                     "Unable to create temp directory '%s'.",
 372  
                     directory.getAbsolutePath()));
 373  
         }
 374  24
         return directory;
 375  
     }
 376  
 }