Coverage Report - org.owasp.dependencycheck.analyzer.PythonDistributionAnalyzer
 
Classes in this File Line Coverage Branch Coverage Complexity
PythonDistributionAnalyzer
80%
96/119
61%
32/52
3.769
 
 1  
 /*
 2  
  * This file is part of dependency-check-core.
 3  
  *
 4  
  * Licensed under the Apache License, Version 2.0 (the "License");
 5  
  * you may not use this file except in compliance with the License.
 6  
  * You may obtain a copy of the License at
 7  
  *
 8  
  *     http://www.apache.org/licenses/LICENSE-2.0
 9  
  *
 10  
  * Unless required by applicable law or agreed to in writing, software
 11  
  * distributed under the License is distributed on an "AS IS" BASIS,
 12  
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 13  
  * See the License for the specific language governing permissions and
 14  
  * limitations under the License.
 15  
  *
 16  
  * Copyright (c) 2015 Institute for Defense Analyses. All Rights Reserved.
 17  
  */
 18  
 package org.owasp.dependencycheck.analyzer;
 19  
 
 20  
 import java.io.BufferedInputStream;
 21  
 import java.io.File;
 22  
 import java.io.FileFilter;
 23  
 import java.io.FileInputStream;
 24  
 import java.io.FileNotFoundException;
 25  
 import java.io.FilenameFilter;
 26  
 import java.io.IOException;
 27  
 import java.io.InputStream;
 28  
 import org.apache.commons.io.filefilter.NameFileFilter;
 29  
 import org.apache.commons.io.filefilter.SuffixFileFilter;
 30  
 import org.apache.commons.lang3.StringUtils;
 31  
 import org.owasp.dependencycheck.Engine;
 32  
 import org.owasp.dependencycheck.analyzer.exception.AnalysisException;
 33  
 import org.owasp.dependencycheck.dependency.Confidence;
 34  
 import org.owasp.dependencycheck.dependency.Dependency;
 35  
 import org.owasp.dependencycheck.dependency.EvidenceCollection;
 36  
 import org.slf4j.Logger;
 37  
 import org.slf4j.LoggerFactory;
 38  
 
 39  
 import javax.mail.MessagingException;
 40  
 import javax.mail.internet.InternetHeaders;
 41  
 import org.owasp.dependencycheck.exception.InitializationException;
 42  
 import org.owasp.dependencycheck.utils.ExtractionException;
 43  
 import org.owasp.dependencycheck.utils.ExtractionUtil;
 44  
 import org.owasp.dependencycheck.utils.FileFilterBuilder;
 45  
 import org.owasp.dependencycheck.utils.FileUtils;
 46  
 import org.owasp.dependencycheck.utils.Settings;
 47  
 import org.owasp.dependencycheck.utils.UrlStringUtils;
 48  
 
 49  
 /**
 50  
  * Used to analyze a Wheel or egg distribution files, or their contents in
 51  
  * unzipped form, and collect information that can be used to determine the
 52  
  * associated CPE.
 53  
  *
 54  
  * @author Dale Visser
 55  
  */
 56  
 @Experimental
 57  14
 public class PythonDistributionAnalyzer extends AbstractFileTypeAnalyzer {
 58  
 
 59  
     /**
 60  
      * Name of egg metadata files to analyze.
 61  
      */
 62  
     private static final String PKG_INFO = "PKG-INFO";
 63  
 
 64  
     /**
 65  
      * Name of wheel metadata files to analyze.
 66  
      */
 67  
     private static final String METADATA = "METADATA";
 68  
 
 69  
     /**
 70  
      * The logger.
 71  
      */
 72  1
     private static final Logger LOGGER = LoggerFactory
 73  1
             .getLogger(PythonDistributionAnalyzer.class);
 74  
 
 75  
     /**
 76  
      * The count of directories created during analysis. This is used for
 77  
      * creating temporary directories.
 78  
      */
 79  1
     private static int dirCount = 0;
 80  
 
 81  
     /**
 82  
      * The name of the analyzer.
 83  
      */
 84  
     private static final String ANALYZER_NAME = "Python Distribution Analyzer";
 85  
     /**
 86  
      * The phase that this analyzer is intended to run in.
 87  
      */
 88  1
     private static final AnalysisPhase ANALYSIS_PHASE = AnalysisPhase.INFORMATION_COLLECTION;
 89  
 
 90  
     /**
 91  
      * The set of file extensions supported by this analyzer.
 92  
      */
 93  1
     private static final String[] EXTENSIONS = {"whl", "egg", "zip"};
 94  
 
 95  
     /**
 96  
      * Used to match on egg archive candidate extensions.
 97  
      */
 98  1
     private static final FileFilter EGG_OR_ZIP = FileFilterBuilder.newInstance().addExtensions("egg", "zip").build();
 99  
 
 100  
     /**
 101  
      * Used to detect files with a .whl extension.
 102  
      */
 103  1
     private static final FileFilter WHL_FILTER = FileFilterBuilder.newInstance().addExtensions("whl").build();
 104  
 
 105  
     /**
 106  
      * The parent directory for the individual directories per archive.
 107  
      */
 108  
     private File tempFileLocation;
 109  
 
 110  
     /**
 111  
      * Filter that detects *.dist-info files (but doesn't verify they are
 112  
      * directories.
 113  
      */
 114  1
     private static final FilenameFilter DIST_INFO_FILTER = new SuffixFileFilter(
 115  
             ".dist-info");
 116  
 
 117  
     /**
 118  
      * Filter that detects files named "METADATA".
 119  
      */
 120  1
     private static final FilenameFilter EGG_INFO_FILTER = new NameFileFilter(
 121  
             "EGG-INFO");
 122  
 
 123  
     /**
 124  
      * Filter that detects files named "METADATA".
 125  
      */
 126  1
     private static final NameFileFilter METADATA_FILTER = new NameFileFilter(
 127  
             METADATA);
 128  
 
 129  
     /**
 130  
      * Filter that detects files named "PKG-INFO".
 131  
      */
 132  1
     private static final NameFileFilter PKG_INFO_FILTER = new NameFileFilter(
 133  
             PKG_INFO);
 134  
 
 135  
     /**
 136  
      * The file filter used to determine which files this analyzer supports.
 137  
      */
 138  2
     private static final FileFilter FILTER = FileFilterBuilder.newInstance().addFileFilters(
 139  1
             METADATA_FILTER, PKG_INFO_FILTER).addExtensions(EXTENSIONS).build();
 140  
 
 141  
     /**
 142  
      * Returns the FileFilter
 143  
      *
 144  
      * @return the FileFilter
 145  
      */
 146  
     @Override
 147  
     protected FileFilter getFileFilter() {
 148  864
         return FILTER;
 149  
     }
 150  
 
 151  
     /**
 152  
      * Returns the name of the analyzer.
 153  
      *
 154  
      * @return the name of the analyzer.
 155  
      */
 156  
     @Override
 157  
     public String getName() {
 158  15
         return ANALYZER_NAME;
 159  
     }
 160  
 
 161  
     /**
 162  
      * Returns the phase that the analyzer is intended to run in.
 163  
      *
 164  
      * @return the phase that the analyzer is intended to run in.
 165  
      */
 166  
     @Override
 167  
     public AnalysisPhase getAnalysisPhase() {
 168  4
         return ANALYSIS_PHASE;
 169  
     }
 170  
 
 171  
     /**
 172  
      * Returns the key used in the properties file to reference the analyzer's
 173  
      * enabled property.
 174  
      *
 175  
      * @return the analyzer's enabled property setting key
 176  
      */
 177  
     @Override
 178  
     protected String getAnalyzerEnabledSettingKey() {
 179  14
         return Settings.KEYS.ANALYZER_PYTHON_DISTRIBUTION_ENABLED;
 180  
     }
 181  
 
 182  
     @Override
 183  
     protected void analyzeFileType(Dependency dependency, Engine engine)
 184  
             throws AnalysisException {
 185  6
         final File actualFile = dependency.getActualFile();
 186  6
         if (WHL_FILTER.accept(actualFile)) {
 187  1
             collectMetadataFromArchiveFormat(dependency, DIST_INFO_FILTER,
 188  
                     METADATA_FILTER);
 189  5
         } else if (EGG_OR_ZIP.accept(actualFile)) {
 190  2
             collectMetadataFromArchiveFormat(dependency, EGG_INFO_FILTER,
 191  
                     PKG_INFO_FILTER);
 192  
         } else {
 193  3
             final String name = actualFile.getName();
 194  3
             final boolean metadata = METADATA.equals(name);
 195  3
             if (metadata || PKG_INFO.equals(name)) {
 196  3
                 final File parent = actualFile.getParentFile();
 197  3
                 final String parentName = parent.getName();
 198  3
                 dependency.setDisplayFileName(parentName + "/" + name);
 199  3
                 if (parent.isDirectory()
 200  1
                         && (metadata && parentName.endsWith(".dist-info")
 201  2
                         || parentName.endsWith(".egg-info") || "EGG-INFO"
 202  1
                         .equals(parentName))) {
 203  3
                     collectWheelMetadata(dependency, actualFile);
 204  
                 }
 205  
             }
 206  
         }
 207  6
     }
 208  
 
 209  
     /**
 210  
      * Collects the meta data from an archive.
 211  
      *
 212  
      * @param dependency the archive being scanned
 213  
      * @param folderFilter the filter to apply to the folder
 214  
      * @param metadataFilter the filter to apply to the meta data
 215  
      * @throws AnalysisException thrown when there is a problem analyzing the
 216  
      * dependency
 217  
      */
 218  
     private void collectMetadataFromArchiveFormat(Dependency dependency,
 219  
             FilenameFilter folderFilter, FilenameFilter metadataFilter)
 220  
             throws AnalysisException {
 221  3
         final File temp = getNextTempDirectory();
 222  3
         LOGGER.debug("{} exists? {}", temp, temp.exists());
 223  
         try {
 224  6
             ExtractionUtil.extractFilesUsingFilter(
 225  3
                     new File(dependency.getActualFilePath()), temp,
 226  
                     metadataFilter);
 227  0
         } catch (ExtractionException ex) {
 228  0
             throw new AnalysisException(ex);
 229  3
         }
 230  
 
 231  3
         File matchingFile = getMatchingFile(temp, folderFilter);
 232  3
         if (matchingFile != null) {
 233  3
             matchingFile = getMatchingFile(matchingFile, metadataFilter);
 234  3
             if (matchingFile != null) {
 235  3
                 collectWheelMetadata(dependency, matchingFile);
 236  
             }
 237  
         }
 238  3
     }
 239  
 
 240  
     /**
 241  
      * Makes sure a usable temporary directory is available.
 242  
      *
 243  
      * @throws InitializationException an AnalyzeException is thrown when the
 244  
      * temp directory cannot be created
 245  
      */
 246  
     @Override
 247  
     protected void initializeFileTypeAnalyzer() throws InitializationException {
 248  
         try {
 249  8
             final File baseDir = Settings.getTempDirectory();
 250  8
             tempFileLocation = File.createTempFile("check", "tmp", baseDir);
 251  8
             if (!tempFileLocation.delete()) {
 252  0
                 setEnabled(false);
 253  0
                 final String msg = String.format(
 254  
                         "Unable to delete temporary file '%s'.",
 255  0
                         tempFileLocation.getAbsolutePath());
 256  0
                 throw new InitializationException(msg);
 257  
             }
 258  8
             if (!tempFileLocation.mkdirs()) {
 259  0
                 setEnabled(false);
 260  0
                 final String msg = String.format(
 261  
                         "Unable to create directory '%s'.",
 262  0
                         tempFileLocation.getAbsolutePath());
 263  0
                 throw new InitializationException(msg);
 264  
             }
 265  0
         } catch (IOException ex) {
 266  0
             setEnabled(false);
 267  0
             throw new InitializationException("Unable to create a temporary file", ex);
 268  8
         }
 269  8
     }
 270  
 
 271  
     /**
 272  
      * Deletes any files extracted from the Wheel during analysis.
 273  
      */
 274  
     @Override
 275  
     public void close() {
 276  10
         if (tempFileLocation != null && tempFileLocation.exists()) {
 277  8
             LOGGER.debug("Attempting to delete temporary files");
 278  8
             final boolean success = FileUtils.delete(tempFileLocation);
 279  8
             if (!success) {
 280  0
                 LOGGER.warn(
 281  
                         "Failed to delete some temporary files, see the log for more details");
 282  
             }
 283  
         }
 284  10
     }
 285  
 
 286  
     /**
 287  
      * Gathers evidence from the METADATA file.
 288  
      *
 289  
      * @param dependency the dependency being analyzed
 290  
      * @param file a reference to the manifest/properties file
 291  
      */
 292  
     private static void collectWheelMetadata(Dependency dependency, File file) {
 293  6
         final InternetHeaders headers = getManifestProperties(file);
 294  6
         addPropertyToEvidence(headers, dependency.getVersionEvidence(),
 295  
                 "Version", Confidence.HIGHEST);
 296  6
         addPropertyToEvidence(headers, dependency.getProductEvidence(), "Name",
 297  
                 Confidence.HIGHEST);
 298  6
         final String url = headers.getHeader("Home-page", null);
 299  6
         final EvidenceCollection vendorEvidence = dependency
 300  6
                 .getVendorEvidence();
 301  6
         if (StringUtils.isNotBlank(url)) {
 302  6
             if (UrlStringUtils.isUrl(url)) {
 303  6
                 vendorEvidence.addEvidence(METADATA, "vendor", url,
 304  
                         Confidence.MEDIUM);
 305  
             }
 306  
         }
 307  6
         addPropertyToEvidence(headers, vendorEvidence, "Author", Confidence.LOW);
 308  6
         final String summary = headers.getHeader("Summary", null);
 309  6
         if (StringUtils.isNotBlank(summary)) {
 310  6
             JarAnalyzer
 311  6
                     .addDescription(dependency, summary, METADATA, "summary");
 312  
         }
 313  6
     }
 314  
 
 315  
     /**
 316  
      * Adds a value to the evidence collection.
 317  
      *
 318  
      * @param headers the properties collection
 319  
      * @param evidence the evidence collection to add the value
 320  
      * @param property the property name
 321  
      * @param confidence the confidence of the evidence
 322  
      */
 323  
     private static void addPropertyToEvidence(InternetHeaders headers,
 324  
             EvidenceCollection evidence, String property, Confidence confidence) {
 325  18
         final String value = headers.getHeader(property, null);
 326  18
         LOGGER.debug("Property: {}, Value: {}", property, value);
 327  18
         if (StringUtils.isNotBlank(value)) {
 328  18
             evidence.addEvidence(METADATA, property, value, confidence);
 329  
         }
 330  18
     }
 331  
 
 332  
     /**
 333  
      * Returns a list of files that match the given filter, this does not
 334  
      * recursively scan the directory.
 335  
      *
 336  
      * @param folder the folder to filter
 337  
      * @param filter the filter to apply to the files in the directory
 338  
      * @return the list of Files in the directory that match the provided filter
 339  
      */
 340  
     private static File getMatchingFile(File folder, FilenameFilter filter) {
 341  6
         File result = null;
 342  6
         final File[] matches = folder.listFiles(filter);
 343  6
         if (null != matches && 1 == matches.length) {
 344  6
             result = matches[0];
 345  
         }
 346  6
         return result;
 347  
     }
 348  
 
 349  
     /**
 350  
      * Reads the manifest entries from the provided file.
 351  
      *
 352  
      * @param manifest the manifest
 353  
      * @return the manifest entries
 354  
      */
 355  
     private static InternetHeaders getManifestProperties(File manifest) {
 356  6
         final InternetHeaders result = new InternetHeaders();
 357  6
         if (null == manifest) {
 358  0
             LOGGER.debug("Manifest file not found.");
 359  
         } else {
 360  6
             InputStream in = null;
 361  
             try {
 362  6
                 in = new BufferedInputStream(new FileInputStream(manifest));
 363  6
                 result.load(in);
 364  0
             } catch (MessagingException e) {
 365  0
                 LOGGER.warn(e.getMessage(), e);
 366  0
             } catch (FileNotFoundException e) {
 367  0
                 LOGGER.warn(e.getMessage(), e);
 368  
             } finally {
 369  6
                 if (in != null) {
 370  
                     try {
 371  6
                         in.close();
 372  0
                     } catch (IOException ex) {
 373  0
                         LOGGER.debug("failed to close input stream", ex);
 374  6
                     }
 375  
                 }
 376  
             }
 377  
         }
 378  6
         return result;
 379  
     }
 380  
 
 381  
     /**
 382  
      * Retrieves the next temporary destination directory for extracting an
 383  
      * archive.
 384  
      *
 385  
      * @return a directory
 386  
      * @throws AnalysisException thrown if unable to create temporary directory
 387  
      */
 388  
     private File getNextTempDirectory() throws AnalysisException {
 389  
         File directory;
 390  
 
 391  
         // getting an exception for some directories not being able to be
 392  
         // created; might be because the directory already exists?
 393  
         do {
 394  3
             dirCount += 1;
 395  3
             directory = new File(tempFileLocation, String.valueOf(dirCount));
 396  3
         } while (directory.exists());
 397  3
         if (!directory.mkdirs()) {
 398  0
             throw new AnalysisException(String.format(
 399  
                     "Unable to create temp directory '%s'.",
 400  0
                     directory.getAbsolutePath()));
 401  
         }
 402  3
         return directory;
 403  
     }
 404  
 }