Coverage Report - org.owasp.dependencycheck.analyzer.PythonDistributionAnalyzer
 
Classes in this File Line Coverage Branch Coverage Complexity
PythonDistributionAnalyzer
85%
89/104
65%
30/46
3.308
 
 1  
 /*
 2  
  * This file is part of dependency-check-core.
 3  
  *
 4  
  * Licensed under the Apache License, Version 2.0 (the "License");
 5  
  * you may not use this file except in compliance with the License.
 6  
  * You may obtain a copy of the License at
 7  
  *
 8  
  *     http://www.apache.org/licenses/LICENSE-2.0
 9  
  *
 10  
  * Unless required by applicable law or agreed to in writing, software
 11  
  * distributed under the License is distributed on an "AS IS" BASIS,
 12  
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 13  
  * See the License for the specific language governing permissions and
 14  
  * limitations under the License.
 15  
  *
 16  
  * Copyright (c) 2015 Institute for Defense Analyses. All Rights Reserved.
 17  
  */
 18  
 package org.owasp.dependencycheck.analyzer;
 19  
 
 20  
 import java.io.BufferedInputStream;
 21  
 import java.io.File;
 22  
 import java.io.FileFilter;
 23  
 import java.io.FileInputStream;
 24  
 import java.io.FileNotFoundException;
 25  
 import java.io.FilenameFilter;
 26  
 import org.apache.commons.io.filefilter.NameFileFilter;
 27  
 import org.apache.commons.io.filefilter.SuffixFileFilter;
 28  
 import org.apache.commons.io.input.AutoCloseInputStream;
 29  
 import org.apache.commons.lang3.StringUtils;
 30  
 import org.owasp.dependencycheck.Engine;
 31  
 import org.owasp.dependencycheck.analyzer.exception.AnalysisException;
 32  
 import org.owasp.dependencycheck.dependency.Confidence;
 33  
 import org.owasp.dependencycheck.dependency.Dependency;
 34  
 import org.owasp.dependencycheck.dependency.EvidenceCollection;
 35  
 import org.slf4j.Logger;
 36  
 import org.slf4j.LoggerFactory;
 37  
 
 38  
 import javax.mail.MessagingException;
 39  
 import javax.mail.internet.InternetHeaders;
 40  
 import org.owasp.dependencycheck.utils.ExtractionException;
 41  
 import org.owasp.dependencycheck.utils.ExtractionUtil;
 42  
 import org.owasp.dependencycheck.utils.FileFilterBuilder;
 43  
 import org.owasp.dependencycheck.utils.FileUtils;
 44  
 import org.owasp.dependencycheck.utils.Settings;
 45  
 import org.owasp.dependencycheck.utils.UrlStringUtils;
 46  
 
 47  
 /**
 48  
  * Used to analyze a Wheel or egg distribution files, or their contents in unzipped form, and collect information that can be used
 49  
  * to determine the associated CPE.
 50  
  *
 51  
  * @author Dale Visser
 52  
  */
 53  
 @Experimental
 54  28
 public class PythonDistributionAnalyzer extends AbstractFileTypeAnalyzer {
 55  
 
 56  
     /**
 57  
      * Name of egg metadata files to analyze.
 58  
      */
 59  
     private static final String PKG_INFO = "PKG-INFO";
 60  
 
 61  
     /**
 62  
      * Name of wheel metadata files to analyze.
 63  
      */
 64  
     private static final String METADATA = "METADATA";
 65  
 
 66  
     /**
 67  
      * The logger.
 68  
      */
 69  2
     private static final Logger LOGGER = LoggerFactory
 70  2
             .getLogger(PythonDistributionAnalyzer.class);
 71  
 
 72  
     /**
 73  
      * The count of directories created during analysis. This is used for creating temporary directories.
 74  
      */
 75  2
     private static int dirCount = 0;
 76  
 
 77  
     /**
 78  
      * The name of the analyzer.
 79  
      */
 80  
     private static final String ANALYZER_NAME = "Python Distribution Analyzer";
 81  
     /**
 82  
      * The phase that this analyzer is intended to run in.
 83  
      */
 84  2
     private static final AnalysisPhase ANALYSIS_PHASE = AnalysisPhase.INFORMATION_COLLECTION;
 85  
 
 86  
     /**
 87  
      * The set of file extensions supported by this analyzer.
 88  
      */
 89  2
     private static final String[] EXTENSIONS = {"whl", "egg", "zip"};
 90  
 
 91  
     /**
 92  
      * Used to match on egg archive candidate extensions.
 93  
      */
 94  2
     private static final FileFilter EGG_OR_ZIP = FileFilterBuilder.newInstance().addExtensions("egg", "zip").build();
 95  
 
 96  
     /**
 97  
      * Used to detect files with a .whl extension.
 98  
      */
 99  2
     private static final FileFilter WHL_FILTER = FileFilterBuilder.newInstance().addExtensions("whl").build();
 100  
 
 101  
     /**
 102  
      * The parent directory for the individual directories per archive.
 103  
      */
 104  
     private File tempFileLocation;
 105  
 
 106  
     /**
 107  
      * Filter that detects *.dist-info files (but doesn't verify they are directories.
 108  
      */
 109  2
     private static final FilenameFilter DIST_INFO_FILTER = new SuffixFileFilter(
 110  
             ".dist-info");
 111  
 
 112  
     /**
 113  
      * Filter that detects files named "METADATA".
 114  
      */
 115  2
     private static final FilenameFilter EGG_INFO_FILTER = new NameFileFilter(
 116  
             "EGG-INFO");
 117  
 
 118  
     /**
 119  
      * Filter that detects files named "METADATA".
 120  
      */
 121  2
     private static final NameFileFilter METADATA_FILTER = new NameFileFilter(
 122  
             METADATA);
 123  
 
 124  
     /**
 125  
      * Filter that detects files named "PKG-INFO".
 126  
      */
 127  2
     private static final NameFileFilter PKG_INFO_FILTER = new NameFileFilter(
 128  
             PKG_INFO);
 129  
 
 130  
     /**
 131  
      * The file filter used to determine which files this analyzer supports.
 132  
      */
 133  4
     private static final FileFilter FILTER = FileFilterBuilder.newInstance().addFileFilters(
 134  2
             METADATA_FILTER, PKG_INFO_FILTER).addExtensions(EXTENSIONS).build();
 135  
 
 136  
     /**
 137  
      * Returns the FileFilter
 138  
      *
 139  
      * @return the FileFilter
 140  
      */
 141  
     @Override
 142  
     protected FileFilter getFileFilter() {
 143  1728
         return FILTER;
 144  
     }
 145  
 
 146  
     /**
 147  
      * Returns the name of the analyzer.
 148  
      *
 149  
      * @return the name of the analyzer.
 150  
      */
 151  
     @Override
 152  
     public String getName() {
 153  30
         return ANALYZER_NAME;
 154  
     }
 155  
 
 156  
     /**
 157  
      * Returns the phase that the analyzer is intended to run in.
 158  
      *
 159  
      * @return the phase that the analyzer is intended to run in.
 160  
      */
 161  
     @Override
 162  
     public AnalysisPhase getAnalysisPhase() {
 163  8
         return ANALYSIS_PHASE;
 164  
     }
 165  
 
 166  
     /**
 167  
      * Returns the key used in the properties file to reference the analyzer's enabled property.
 168  
      *
 169  
      * @return the analyzer's enabled property setting key
 170  
      */
 171  
     @Override
 172  
     protected String getAnalyzerEnabledSettingKey() {
 173  28
         return Settings.KEYS.ANALYZER_PYTHON_DISTRIBUTION_ENABLED;
 174  
     }
 175  
 
 176  
     @Override
 177  
     protected void analyzeFileType(Dependency dependency, Engine engine)
 178  
             throws AnalysisException {
 179  12
         final File actualFile = dependency.getActualFile();
 180  12
         if (WHL_FILTER.accept(actualFile)) {
 181  2
             collectMetadataFromArchiveFormat(dependency, DIST_INFO_FILTER,
 182  
                     METADATA_FILTER);
 183  10
         } else if (EGG_OR_ZIP.accept(actualFile)) {
 184  4
             collectMetadataFromArchiveFormat(dependency, EGG_INFO_FILTER,
 185  
                     PKG_INFO_FILTER);
 186  
         } else {
 187  6
             final String name = actualFile.getName();
 188  6
             final boolean metadata = METADATA.equals(name);
 189  6
             if (metadata || PKG_INFO.equals(name)) {
 190  6
                 final File parent = actualFile.getParentFile();
 191  6
                 final String parentName = parent.getName();
 192  6
                 dependency.setDisplayFileName(parentName + "/" + name);
 193  6
                 if (parent.isDirectory()
 194  2
                         && (metadata && parentName.endsWith(".dist-info")
 195  4
                         || parentName.endsWith(".egg-info") || "EGG-INFO"
 196  2
                         .equals(parentName))) {
 197  6
                     collectWheelMetadata(dependency, actualFile);
 198  
                 }
 199  
             }
 200  
         }
 201  12
     }
 202  
 
 203  
     /**
 204  
      * Collects the meta data from an archive.
 205  
      *
 206  
      * @param dependency the archive being scanned
 207  
      * @param folderFilter the filter to apply to the folder
 208  
      * @param metadataFilter the filter to apply to the meta data
 209  
      * @throws AnalysisException thrown when there is a problem analyzing the dependency
 210  
      */
 211  
     private void collectMetadataFromArchiveFormat(Dependency dependency,
 212  
             FilenameFilter folderFilter, FilenameFilter metadataFilter)
 213  
             throws AnalysisException {
 214  6
         final File temp = getNextTempDirectory();
 215  6
         LOGGER.debug("{} exists? {}", temp, temp.exists());
 216  
         try {
 217  12
             ExtractionUtil.extractFilesUsingFilter(
 218  6
                     new File(dependency.getActualFilePath()), temp,
 219  
                     metadataFilter);
 220  0
         } catch (ExtractionException ex) {
 221  0
             throw new AnalysisException(ex);
 222  6
         }
 223  
 
 224  12
         collectWheelMetadata(
 225  
                 dependency,
 226  6
                 getMatchingFile(getMatchingFile(temp, folderFilter),
 227  
                         metadataFilter));
 228  6
     }
 229  
 
 230  
     /**
 231  
      * Makes sure a usable temporary directory is available.
 232  
      *
 233  
      * @throws Exception an AnalyzeException is thrown when the temp directory cannot be created
 234  
      */
 235  
     @Override
 236  
     protected void initializeFileTypeAnalyzer() throws Exception {
 237  16
         final File baseDir = Settings.getTempDirectory();
 238  16
         tempFileLocation = File.createTempFile("check", "tmp", baseDir);
 239  16
         if (!tempFileLocation.delete()) {
 240  0
             final String msg = String.format(
 241  
                     "Unable to delete temporary file '%s'.",
 242  0
                     tempFileLocation.getAbsolutePath());
 243  0
             throw new AnalysisException(msg);
 244  
         }
 245  16
         if (!tempFileLocation.mkdirs()) {
 246  0
             final String msg = String.format(
 247  
                     "Unable to create directory '%s'.",
 248  0
                     tempFileLocation.getAbsolutePath());
 249  0
             throw new AnalysisException(msg);
 250  
         }
 251  16
     }
 252  
 
 253  
     /**
 254  
      * Deletes any files extracted from the Wheel during analysis.
 255  
      */
 256  
     @Override
 257  
     public void close() {
 258  20
         if (tempFileLocation != null && tempFileLocation.exists()) {
 259  16
             LOGGER.debug("Attempting to delete temporary files");
 260  16
             final boolean success = FileUtils.delete(tempFileLocation);
 261  16
             if (!success) {
 262  2
                 LOGGER.warn(
 263  
                         "Failed to delete some temporary files, see the log for more details");
 264  
             }
 265  
         }
 266  20
     }
 267  
 
 268  
     /**
 269  
      * Gathers evidence from the METADATA file.
 270  
      *
 271  
      * @param dependency the dependency being analyzed
 272  
      * @param file a reference to the manifest/properties file
 273  
      */
 274  
     private static void collectWheelMetadata(Dependency dependency, File file) {
 275  12
         final InternetHeaders headers = getManifestProperties(file);
 276  12
         addPropertyToEvidence(headers, dependency.getVersionEvidence(),
 277  
                 "Version", Confidence.HIGHEST);
 278  12
         addPropertyToEvidence(headers, dependency.getProductEvidence(), "Name",
 279  
                 Confidence.HIGHEST);
 280  12
         final String url = headers.getHeader("Home-page", null);
 281  12
         final EvidenceCollection vendorEvidence = dependency
 282  12
                 .getVendorEvidence();
 283  12
         if (StringUtils.isNotBlank(url)) {
 284  12
             if (UrlStringUtils.isUrl(url)) {
 285  12
                 vendorEvidence.addEvidence(METADATA, "vendor", url,
 286  
                         Confidence.MEDIUM);
 287  
             }
 288  
         }
 289  12
         addPropertyToEvidence(headers, vendorEvidence, "Author", Confidence.LOW);
 290  12
         final String summary = headers.getHeader("Summary", null);
 291  12
         if (StringUtils.isNotBlank(summary)) {
 292  12
             JarAnalyzer
 293  12
                     .addDescription(dependency, summary, METADATA, "summary");
 294  
         }
 295  12
     }
 296  
 
 297  
     /**
 298  
      * Adds a value to the evidence collection.
 299  
      *
 300  
      * @param headers the properties collection
 301  
      * @param evidence the evidence collection to add the value
 302  
      * @param property the property name
 303  
      * @param confidence the confidence of the evidence
 304  
      */
 305  
     private static void addPropertyToEvidence(InternetHeaders headers,
 306  
             EvidenceCollection evidence, String property, Confidence confidence) {
 307  36
         final String value = headers.getHeader(property, null);
 308  36
         LOGGER.debug("Property: {}, Value: {}", property, value);
 309  36
         if (StringUtils.isNotBlank(value)) {
 310  36
             evidence.addEvidence(METADATA, property, value, confidence);
 311  
         }
 312  36
     }
 313  
 
 314  
     /**
 315  
      * Returns a list of files that match the given filter, this does not recursively scan the directory.
 316  
      *
 317  
      * @param folder the folder to filter
 318  
      * @param filter the filter to apply to the files in the directory
 319  
      * @return the list of Files in the directory that match the provided filter
 320  
      */
 321  
     private static File getMatchingFile(File folder, FilenameFilter filter) {
 322  12
         File result = null;
 323  12
         final File[] matches = folder.listFiles(filter);
 324  12
         if (null != matches && 1 == matches.length) {
 325  12
             result = matches[0];
 326  
         }
 327  12
         return result;
 328  
     }
 329  
 
 330  
     /**
 331  
      * Reads the manifest entries from the provided file.
 332  
      *
 333  
      * @param manifest the manifest
 334  
      * @return the manifest entries
 335  
      */
 336  
     private static InternetHeaders getManifestProperties(File manifest) {
 337  12
         final InternetHeaders result = new InternetHeaders();
 338  12
         if (null == manifest) {
 339  0
             LOGGER.debug("Manifest file not found.");
 340  
         } else {
 341  
             try {
 342  12
                 result.load(new AutoCloseInputStream(new BufferedInputStream(
 343  
                         new FileInputStream(manifest))));
 344  0
             } catch (MessagingException e) {
 345  0
                 LOGGER.warn(e.getMessage(), e);
 346  0
             } catch (FileNotFoundException e) {
 347  0
                 LOGGER.warn(e.getMessage(), e);
 348  12
             }
 349  
         }
 350  12
         return result;
 351  
     }
 352  
 
 353  
     /**
 354  
      * Retrieves the next temporary destination directory for extracting an archive.
 355  
      *
 356  
      * @return a directory
 357  
      * @throws AnalysisException thrown if unable to create temporary directory
 358  
      */
 359  
     private File getNextTempDirectory() throws AnalysisException {
 360  
         File directory;
 361  
 
 362  
         // getting an exception for some directories not being able to be
 363  
         // created; might be because the directory already exists?
 364  
         do {
 365  6
             dirCount += 1;
 366  6
             directory = new File(tempFileLocation, String.valueOf(dirCount));
 367  6
         } while (directory.exists());
 368  6
         if (!directory.mkdirs()) {
 369  0
             throw new AnalysisException(String.format(
 370  
                     "Unable to create temp directory '%s'.",
 371  0
                     directory.getAbsolutePath()));
 372  
         }
 373  6
         return directory;
 374  
     }
 375  
 }