Coverage Report - org.owasp.dependencycheck.analyzer.PythonDistributionAnalyzer
 
Classes in this File Line Coverage Branch Coverage Complexity
PythonDistributionAnalyzer
81%
90/110
65%
30/46
3.462
 
 1  
 /*
 2  
  * This file is part of dependency-check-core.
 3  
  *
 4  
  * Licensed under the Apache License, Version 2.0 (the "License");
 5  
  * you may not use this file except in compliance with the License.
 6  
  * You may obtain a copy of the License at
 7  
  *
 8  
  *     http://www.apache.org/licenses/LICENSE-2.0
 9  
  *
 10  
  * Unless required by applicable law or agreed to in writing, software
 11  
  * distributed under the License is distributed on an "AS IS" BASIS,
 12  
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 13  
  * See the License for the specific language governing permissions and
 14  
  * limitations under the License.
 15  
  *
 16  
  * Copyright (c) 2015 Institute for Defense Analyses. All Rights Reserved.
 17  
  */
 18  
 package org.owasp.dependencycheck.analyzer;
 19  
 
 20  
 import java.io.BufferedInputStream;
 21  
 import java.io.File;
 22  
 import java.io.FileFilter;
 23  
 import java.io.FileInputStream;
 24  
 import java.io.FileNotFoundException;
 25  
 import java.io.FilenameFilter;
 26  
 import java.io.IOException;
 27  
 import org.apache.commons.io.filefilter.NameFileFilter;
 28  
 import org.apache.commons.io.filefilter.SuffixFileFilter;
 29  
 import org.apache.commons.io.input.AutoCloseInputStream;
 30  
 import org.apache.commons.lang3.StringUtils;
 31  
 import org.owasp.dependencycheck.Engine;
 32  
 import org.owasp.dependencycheck.analyzer.exception.AnalysisException;
 33  
 import org.owasp.dependencycheck.dependency.Confidence;
 34  
 import org.owasp.dependencycheck.dependency.Dependency;
 35  
 import org.owasp.dependencycheck.dependency.EvidenceCollection;
 36  
 import org.slf4j.Logger;
 37  
 import org.slf4j.LoggerFactory;
 38  
 
 39  
 import javax.mail.MessagingException;
 40  
 import javax.mail.internet.InternetHeaders;
 41  
 import org.owasp.dependencycheck.exception.InitializationException;
 42  
 import org.owasp.dependencycheck.utils.ExtractionException;
 43  
 import org.owasp.dependencycheck.utils.ExtractionUtil;
 44  
 import org.owasp.dependencycheck.utils.FileFilterBuilder;
 45  
 import org.owasp.dependencycheck.utils.FileUtils;
 46  
 import org.owasp.dependencycheck.utils.Settings;
 47  
 import org.owasp.dependencycheck.utils.UrlStringUtils;
 48  
 
 49  
 /**
 50  
  * Used to analyze a Wheel or egg distribution files, or their contents in
 51  
  * unzipped form, and collect information that can be used to determine the
 52  
  * associated CPE.
 53  
  *
 54  
  * @author Dale Visser
 55  
  */
 56  
 @Experimental
 57  14
 public class PythonDistributionAnalyzer extends AbstractFileTypeAnalyzer {
 58  
 
 59  
     /**
 60  
      * Name of egg metadata files to analyze.
 61  
      */
 62  
     private static final String PKG_INFO = "PKG-INFO";
 63  
 
 64  
     /**
 65  
      * Name of wheel metadata files to analyze.
 66  
      */
 67  
     private static final String METADATA = "METADATA";
 68  
 
 69  
     /**
 70  
      * The logger.
 71  
      */
 72  1
     private static final Logger LOGGER = LoggerFactory
 73  1
             .getLogger(PythonDistributionAnalyzer.class);
 74  
 
 75  
     /**
 76  
      * The count of directories created during analysis. This is used for
 77  
      * creating temporary directories.
 78  
      */
 79  1
     private static int dirCount = 0;
 80  
 
 81  
     /**
 82  
      * The name of the analyzer.
 83  
      */
 84  
     private static final String ANALYZER_NAME = "Python Distribution Analyzer";
 85  
     /**
 86  
      * The phase that this analyzer is intended to run in.
 87  
      */
 88  1
     private static final AnalysisPhase ANALYSIS_PHASE = AnalysisPhase.INFORMATION_COLLECTION;
 89  
 
 90  
     /**
 91  
      * The set of file extensions supported by this analyzer.
 92  
      */
 93  1
     private static final String[] EXTENSIONS = {"whl", "egg", "zip"};
 94  
 
 95  
     /**
 96  
      * Used to match on egg archive candidate extensions.
 97  
      */
 98  1
     private static final FileFilter EGG_OR_ZIP = FileFilterBuilder.newInstance().addExtensions("egg", "zip").build();
 99  
 
 100  
     /**
 101  
      * Used to detect files with a .whl extension.
 102  
      */
 103  1
     private static final FileFilter WHL_FILTER = FileFilterBuilder.newInstance().addExtensions("whl").build();
 104  
 
 105  
     /**
 106  
      * The parent directory for the individual directories per archive.
 107  
      */
 108  
     private File tempFileLocation;
 109  
 
 110  
     /**
 111  
      * Filter that detects *.dist-info files (but doesn't verify they are
 112  
      * directories.
 113  
      */
 114  1
     private static final FilenameFilter DIST_INFO_FILTER = new SuffixFileFilter(
 115  
             ".dist-info");
 116  
 
 117  
     /**
 118  
      * Filter that detects files named "METADATA".
 119  
      */
 120  1
     private static final FilenameFilter EGG_INFO_FILTER = new NameFileFilter(
 121  
             "EGG-INFO");
 122  
 
 123  
     /**
 124  
      * Filter that detects files named "METADATA".
 125  
      */
 126  1
     private static final NameFileFilter METADATA_FILTER = new NameFileFilter(
 127  
             METADATA);
 128  
 
 129  
     /**
 130  
      * Filter that detects files named "PKG-INFO".
 131  
      */
 132  1
     private static final NameFileFilter PKG_INFO_FILTER = new NameFileFilter(
 133  
             PKG_INFO);
 134  
 
 135  
     /**
 136  
      * The file filter used to determine which files this analyzer supports.
 137  
      */
 138  2
     private static final FileFilter FILTER = FileFilterBuilder.newInstance().addFileFilters(
 139  1
             METADATA_FILTER, PKG_INFO_FILTER).addExtensions(EXTENSIONS).build();
 140  
 
 141  
     /**
 142  
      * Returns the FileFilter
 143  
      *
 144  
      * @return the FileFilter
 145  
      */
 146  
     @Override
 147  
     protected FileFilter getFileFilter() {
 148  15
         return FILTER;
 149  
     }
 150  
 
 151  
     /**
 152  
      * Returns the name of the analyzer.
 153  
      *
 154  
      * @return the name of the analyzer.
 155  
      */
 156  
     @Override
 157  
     public String getName() {
 158  15
         return ANALYZER_NAME;
 159  
     }
 160  
 
 161  
     /**
 162  
      * Returns the phase that the analyzer is intended to run in.
 163  
      *
 164  
      * @return the phase that the analyzer is intended to run in.
 165  
      */
 166  
     @Override
 167  
     public AnalysisPhase getAnalysisPhase() {
 168  4
         return ANALYSIS_PHASE;
 169  
     }
 170  
 
 171  
     /**
 172  
      * Returns the key used in the properties file to reference the analyzer's
 173  
      * enabled property.
 174  
      *
 175  
      * @return the analyzer's enabled property setting key
 176  
      */
 177  
     @Override
 178  
     protected String getAnalyzerEnabledSettingKey() {
 179  14
         return Settings.KEYS.ANALYZER_PYTHON_DISTRIBUTION_ENABLED;
 180  
     }
 181  
 
 182  
     @Override
 183  
     protected void analyzeFileType(Dependency dependency, Engine engine)
 184  
             throws AnalysisException {
 185  6
         final File actualFile = dependency.getActualFile();
 186  6
         if (WHL_FILTER.accept(actualFile)) {
 187  1
             collectMetadataFromArchiveFormat(dependency, DIST_INFO_FILTER,
 188  
                     METADATA_FILTER);
 189  5
         } else if (EGG_OR_ZIP.accept(actualFile)) {
 190  2
             collectMetadataFromArchiveFormat(dependency, EGG_INFO_FILTER,
 191  
                     PKG_INFO_FILTER);
 192  
         } else {
 193  3
             final String name = actualFile.getName();
 194  3
             final boolean metadata = METADATA.equals(name);
 195  3
             if (metadata || PKG_INFO.equals(name)) {
 196  3
                 final File parent = actualFile.getParentFile();
 197  3
                 final String parentName = parent.getName();
 198  3
                 dependency.setDisplayFileName(parentName + "/" + name);
 199  3
                 if (parent.isDirectory()
 200  1
                         && (metadata && parentName.endsWith(".dist-info")
 201  2
                         || parentName.endsWith(".egg-info") || "EGG-INFO"
 202  1
                         .equals(parentName))) {
 203  3
                     collectWheelMetadata(dependency, actualFile);
 204  
                 }
 205  
             }
 206  
         }
 207  6
     }
 208  
 
 209  
     /**
 210  
      * Collects the meta data from an archive.
 211  
      *
 212  
      * @param dependency the archive being scanned
 213  
      * @param folderFilter the filter to apply to the folder
 214  
      * @param metadataFilter the filter to apply to the meta data
 215  
      * @throws AnalysisException thrown when there is a problem analyzing the
 216  
      * dependency
 217  
      */
 218  
     private void collectMetadataFromArchiveFormat(Dependency dependency,
 219  
             FilenameFilter folderFilter, FilenameFilter metadataFilter)
 220  
             throws AnalysisException {
 221  3
         final File temp = getNextTempDirectory();
 222  3
         LOGGER.debug("{} exists? {}", temp, temp.exists());
 223  
         try {
 224  6
             ExtractionUtil.extractFilesUsingFilter(
 225  3
                     new File(dependency.getActualFilePath()), temp,
 226  
                     metadataFilter);
 227  0
         } catch (ExtractionException ex) {
 228  0
             throw new AnalysisException(ex);
 229  3
         }
 230  
 
 231  6
         collectWheelMetadata(
 232  
                 dependency,
 233  3
                 getMatchingFile(getMatchingFile(temp, folderFilter),
 234  
                         metadataFilter));
 235  3
     }
 236  
 
 237  
     /**
 238  
      * Makes sure a usable temporary directory is available.
 239  
      *
 240  
      * @throws InitializationException an AnalyzeException is thrown when the
 241  
      * temp directory cannot be created
 242  
      */
 243  
     @Override
 244  
     protected void initializeFileTypeAnalyzer() throws InitializationException {
 245  
         try {
 246  8
             final File baseDir = Settings.getTempDirectory();
 247  8
             tempFileLocation = File.createTempFile("check", "tmp", baseDir);
 248  8
             if (!tempFileLocation.delete()) {
 249  0
                 setEnabled(false);
 250  0
                 final String msg = String.format(
 251  
                         "Unable to delete temporary file '%s'.",
 252  0
                         tempFileLocation.getAbsolutePath());
 253  0
                 throw new InitializationException(msg);
 254  
             }
 255  8
             if (!tempFileLocation.mkdirs()) {
 256  0
                 setEnabled(false);
 257  0
                 final String msg = String.format(
 258  
                         "Unable to create directory '%s'.",
 259  0
                         tempFileLocation.getAbsolutePath());
 260  0
                 throw new InitializationException(msg);
 261  
             }
 262  0
         } catch (IOException ex) {
 263  0
             setEnabled(false);
 264  0
             throw new InitializationException("Unable to create a temporary file", ex);
 265  8
         }
 266  8
     }
 267  
 
 268  
     /**
 269  
      * Deletes any files extracted from the Wheel during analysis.
 270  
      */
 271  
     @Override
 272  
     public void close() {
 273  10
         if (tempFileLocation != null && tempFileLocation.exists()) {
 274  8
             LOGGER.debug("Attempting to delete temporary files");
 275  8
             final boolean success = FileUtils.delete(tempFileLocation);
 276  8
             if (!success) {
 277  1
                 LOGGER.warn(
 278  
                         "Failed to delete some temporary files, see the log for more details");
 279  
             }
 280  
         }
 281  10
     }
 282  
 
 283  
     /**
 284  
      * Gathers evidence from the METADATA file.
 285  
      *
 286  
      * @param dependency the dependency being analyzed
 287  
      * @param file a reference to the manifest/properties file
 288  
      */
 289  
     private static void collectWheelMetadata(Dependency dependency, File file) {
 290  6
         final InternetHeaders headers = getManifestProperties(file);
 291  6
         addPropertyToEvidence(headers, dependency.getVersionEvidence(),
 292  
                 "Version", Confidence.HIGHEST);
 293  6
         addPropertyToEvidence(headers, dependency.getProductEvidence(), "Name",
 294  
                 Confidence.HIGHEST);
 295  6
         final String url = headers.getHeader("Home-page", null);
 296  6
         final EvidenceCollection vendorEvidence = dependency
 297  6
                 .getVendorEvidence();
 298  6
         if (StringUtils.isNotBlank(url)) {
 299  6
             if (UrlStringUtils.isUrl(url)) {
 300  6
                 vendorEvidence.addEvidence(METADATA, "vendor", url,
 301  
                         Confidence.MEDIUM);
 302  
             }
 303  
         }
 304  6
         addPropertyToEvidence(headers, vendorEvidence, "Author", Confidence.LOW);
 305  6
         final String summary = headers.getHeader("Summary", null);
 306  6
         if (StringUtils.isNotBlank(summary)) {
 307  6
             JarAnalyzer
 308  6
                     .addDescription(dependency, summary, METADATA, "summary");
 309  
         }
 310  6
     }
 311  
 
 312  
     /**
 313  
      * Adds a value to the evidence collection.
 314  
      *
 315  
      * @param headers the properties collection
 316  
      * @param evidence the evidence collection to add the value
 317  
      * @param property the property name
 318  
      * @param confidence the confidence of the evidence
 319  
      */
 320  
     private static void addPropertyToEvidence(InternetHeaders headers,
 321  
             EvidenceCollection evidence, String property, Confidence confidence) {
 322  18
         final String value = headers.getHeader(property, null);
 323  18
         LOGGER.debug("Property: {}, Value: {}", property, value);
 324  18
         if (StringUtils.isNotBlank(value)) {
 325  18
             evidence.addEvidence(METADATA, property, value, confidence);
 326  
         }
 327  18
     }
 328  
 
 329  
     /**
 330  
      * Returns a list of files that match the given filter, this does not
 331  
      * recursively scan the directory.
 332  
      *
 333  
      * @param folder the folder to filter
 334  
      * @param filter the filter to apply to the files in the directory
 335  
      * @return the list of Files in the directory that match the provided filter
 336  
      */
 337  
     private static File getMatchingFile(File folder, FilenameFilter filter) {
 338  6
         File result = null;
 339  6
         final File[] matches = folder.listFiles(filter);
 340  6
         if (null != matches && 1 == matches.length) {
 341  6
             result = matches[0];
 342  
         }
 343  6
         return result;
 344  
     }
 345  
 
 346  
     /**
 347  
      * Reads the manifest entries from the provided file.
 348  
      *
 349  
      * @param manifest the manifest
 350  
      * @return the manifest entries
 351  
      */
 352  
     private static InternetHeaders getManifestProperties(File manifest) {
 353  6
         final InternetHeaders result = new InternetHeaders();
 354  6
         if (null == manifest) {
 355  0
             LOGGER.debug("Manifest file not found.");
 356  
         } else {
 357  
             try {
 358  6
                 result.load(new AutoCloseInputStream(new BufferedInputStream(
 359  
                         new FileInputStream(manifest))));
 360  0
             } catch (MessagingException e) {
 361  0
                 LOGGER.warn(e.getMessage(), e);
 362  0
             } catch (FileNotFoundException e) {
 363  0
                 LOGGER.warn(e.getMessage(), e);
 364  6
             }
 365  
         }
 366  6
         return result;
 367  
     }
 368  
 
 369  
     /**
 370  
      * Retrieves the next temporary destination directory for extracting an
 371  
      * archive.
 372  
      *
 373  
      * @return a directory
 374  
      * @throws AnalysisException thrown if unable to create temporary directory
 375  
      */
 376  
     private File getNextTempDirectory() throws AnalysisException {
 377  
         File directory;
 378  
 
 379  
         // getting an exception for some directories not being able to be
 380  
         // created; might be because the directory already exists?
 381  
         do {
 382  3
             dirCount += 1;
 383  3
             directory = new File(tempFileLocation, String.valueOf(dirCount));
 384  3
         } while (directory.exists());
 385  3
         if (!directory.mkdirs()) {
 386  0
             throw new AnalysisException(String.format(
 387  
                     "Unable to create temp directory '%s'.",
 388  0
                     directory.getAbsolutePath()));
 389  
         }
 390  3
         return directory;
 391  
     }
 392  
 }