Coverage Report - org.owasp.dependencycheck.analyzer.PythonDistributionAnalyzer
 
Classes in this File Line Coverage Branch Coverage Complexity
PythonDistributionAnalyzer
79%
96/121
53%
31/58
4
 
 1  
 /*
 2  
  * This file is part of dependency-check-core.
 3  
  *
 4  
  * Licensed under the Apache License, Version 2.0 (the "License");
 5  
  * you may not use this file except in compliance with the License.
 6  
  * You may obtain a copy of the License at
 7  
  *
 8  
  *     http://www.apache.org/licenses/LICENSE-2.0
 9  
  *
 10  
  * Unless required by applicable law or agreed to in writing, software
 11  
  * distributed under the License is distributed on an "AS IS" BASIS,
 12  
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 13  
  * See the License for the specific language governing permissions and
 14  
  * limitations under the License.
 15  
  *
 16  
  * Copyright (c) 2015 Institute for Defense Analyses. All Rights Reserved.
 17  
  */
 18  
 package org.owasp.dependencycheck.analyzer;
 19  
 
 20  
 import java.io.BufferedInputStream;
 21  
 import java.io.File;
 22  
 import java.io.FileFilter;
 23  
 import java.io.FileInputStream;
 24  
 import java.io.FileNotFoundException;
 25  
 import java.io.FilenameFilter;
 26  
 import java.io.IOException;
 27  
 import java.io.InputStream;
 28  
 import org.apache.commons.io.filefilter.NameFileFilter;
 29  
 import org.apache.commons.io.filefilter.SuffixFileFilter;
 30  
 import org.apache.commons.lang3.StringUtils;
 31  
 import org.owasp.dependencycheck.Engine;
 32  
 import org.owasp.dependencycheck.analyzer.exception.AnalysisException;
 33  
 import org.owasp.dependencycheck.dependency.Confidence;
 34  
 import org.owasp.dependencycheck.dependency.Dependency;
 35  
 import org.owasp.dependencycheck.dependency.EvidenceCollection;
 36  
 import org.slf4j.Logger;
 37  
 import org.slf4j.LoggerFactory;
 38  
 
 39  
 import javax.mail.MessagingException;
 40  
 import javax.mail.internet.InternetHeaders;
 41  
 import org.owasp.dependencycheck.exception.InitializationException;
 42  
 import org.owasp.dependencycheck.utils.ExtractionException;
 43  
 import org.owasp.dependencycheck.utils.ExtractionUtil;
 44  
 import org.owasp.dependencycheck.utils.FileFilterBuilder;
 45  
 import org.owasp.dependencycheck.utils.FileUtils;
 46  
 import org.owasp.dependencycheck.utils.Settings;
 47  
 import org.owasp.dependencycheck.utils.UrlStringUtils;
 48  
 import java.util.concurrent.atomic.AtomicInteger;
 49  
 
 50  
 /**
 51  
  * Used to analyze a Wheel or egg distribution files, or their contents in
 52  
  * unzipped form, and collect information that can be used to determine the
 53  
  * associated CPE.
 54  
  *
 55  
  * @author Dale Visser
 56  
  */
 57  
 @Experimental
 58  16
 public class PythonDistributionAnalyzer extends AbstractFileTypeAnalyzer {
 59  
 
 60  
     /**
 61  
      * Name of egg metadata files to analyze.
 62  
      */
 63  
     private static final String PKG_INFO = "PKG-INFO";
 64  
 
 65  
     /**
 66  
      * Name of wheel metadata files to analyze.
 67  
      */
 68  
     private static final String METADATA = "METADATA";
 69  
 
 70  
     /**
 71  
      * The logger.
 72  
      */
 73  1
     private static final Logger LOGGER = LoggerFactory
 74  1
             .getLogger(PythonDistributionAnalyzer.class);
 75  
 
 76  
     /**
 77  
      * The count of directories created during analysis. This is used for
 78  
      * creating temporary directories.
 79  
      */
 80  1
     private static final AtomicInteger DIR_COUNT = new AtomicInteger(0);
 81  
 
 82  
     /**
 83  
      * The name of the analyzer.
 84  
      */
 85  
     private static final String ANALYZER_NAME = "Python Distribution Analyzer";
 86  
     /**
 87  
      * The phase that this analyzer is intended to run in.
 88  
      */
 89  1
     private static final AnalysisPhase ANALYSIS_PHASE = AnalysisPhase.INFORMATION_COLLECTION;
 90  
 
 91  
     /**
 92  
      * The set of file extensions supported by this analyzer.
 93  
      */
 94  1
     private static final String[] EXTENSIONS = {"whl", "egg", "zip"};
 95  
 
 96  
     /**
 97  
      * Used to match on egg archive candidate extensions.
 98  
      */
 99  1
     private static final FileFilter EGG_OR_ZIP = FileFilterBuilder.newInstance().addExtensions("egg", "zip").build();
 100  
 
 101  
     /**
 102  
      * Used to detect files with a .whl extension.
 103  
      */
 104  1
     private static final FileFilter WHL_FILTER = FileFilterBuilder.newInstance().addExtensions("whl").build();
 105  
 
 106  
     /**
 107  
      * The parent directory for the individual directories per archive.
 108  
      */
 109  
     private File tempFileLocation;
 110  
 
 111  
     /**
 112  
      * Filter that detects *.dist-info files (but doesn't verify they are
 113  
      * directories.
 114  
      */
 115  1
     private static final FilenameFilter DIST_INFO_FILTER = new SuffixFileFilter(
 116  
             ".dist-info");
 117  
 
 118  
     /**
 119  
      * Filter that detects files named "METADATA".
 120  
      */
 121  1
     private static final FilenameFilter EGG_INFO_FILTER = new NameFileFilter(
 122  
             "EGG-INFO");
 123  
 
 124  
     /**
 125  
      * Filter that detects files named "METADATA".
 126  
      */
 127  1
     private static final NameFileFilter METADATA_FILTER = new NameFileFilter(
 128  
             METADATA);
 129  
 
 130  
     /**
 131  
      * Filter that detects files named "PKG-INFO".
 132  
      */
 133  1
     private static final NameFileFilter PKG_INFO_FILTER = new NameFileFilter(
 134  
             PKG_INFO);
 135  
 
 136  
     /**
 137  
      * The file filter used to determine which files this analyzer supports.
 138  
      */
 139  2
     private static final FileFilter FILTER = FileFilterBuilder.newInstance().addFileFilters(
 140  1
             METADATA_FILTER, PKG_INFO_FILTER).addExtensions(EXTENSIONS).build();
 141  
 
 142  
     /**
 143  
      * Returns the FileFilter
 144  
      *
 145  
      * @return the FileFilter
 146  
      */
 147  
     @Override
 148  
     protected FileFilter getFileFilter() {
 149  863
         return FILTER;
 150  
     }
 151  
 
 152  
     /**
 153  
      * Returns the name of the analyzer.
 154  
      *
 155  
      * @return the name of the analyzer.
 156  
      */
 157  
     @Override
 158  
     public String getName() {
 159  17
         return ANALYZER_NAME;
 160  
     }
 161  
 
 162  
     /**
 163  
      * Returns the phase that the analyzer is intended to run in.
 164  
      *
 165  
      * @return the phase that the analyzer is intended to run in.
 166  
      */
 167  
     @Override
 168  
     public AnalysisPhase getAnalysisPhase() {
 169  6
         return ANALYSIS_PHASE;
 170  
     }
 171  
 
 172  
     /**
 173  
      * Returns the key used in the properties file to reference the analyzer's
 174  
      * enabled property.
 175  
      *
 176  
      * @return the analyzer's enabled property setting key
 177  
      */
 178  
     @Override
 179  
     protected String getAnalyzerEnabledSettingKey() {
 180  10
         return Settings.KEYS.ANALYZER_PYTHON_DISTRIBUTION_ENABLED;
 181  
     }
 182  
 
 183  
     @Override
 184  
     protected void analyzeDependency(Dependency dependency, Engine engine)
 185  
             throws AnalysisException {
 186  6
         final File actualFile = dependency.getActualFile();
 187  6
         if (WHL_FILTER.accept(actualFile)) {
 188  1
             collectMetadataFromArchiveFormat(dependency, DIST_INFO_FILTER,
 189  
                     METADATA_FILTER);
 190  5
         } else if (EGG_OR_ZIP.accept(actualFile)) {
 191  2
             collectMetadataFromArchiveFormat(dependency, EGG_INFO_FILTER,
 192  
                     PKG_INFO_FILTER);
 193  
         } else {
 194  3
             final String name = actualFile.getName();
 195  3
             final boolean metadata = METADATA.equals(name);
 196  3
             if (metadata || PKG_INFO.equals(name)) {
 197  3
                 final File parent = actualFile.getParentFile();
 198  3
                 final String parentName = parent.getName();
 199  3
                 dependency.setDisplayFileName(parentName + "/" + name);
 200  3
                 if (parent.isDirectory()
 201  1
                         && (metadata && parentName.endsWith(".dist-info")
 202  2
                         || parentName.endsWith(".egg-info") || "EGG-INFO"
 203  1
                         .equals(parentName))) {
 204  3
                     collectWheelMetadata(dependency, actualFile);
 205  
                 }
 206  
             }
 207  
         }
 208  6
     }
 209  
 
 210  
     /**
 211  
      * Collects the meta data from an archive.
 212  
      *
 213  
      * @param dependency the archive being scanned
 214  
      * @param folderFilter the filter to apply to the folder
 215  
      * @param metadataFilter the filter to apply to the meta data
 216  
      * @throws AnalysisException thrown when there is a problem analyzing the
 217  
      * dependency
 218  
      */
 219  
     private void collectMetadataFromArchiveFormat(Dependency dependency,
 220  
             FilenameFilter folderFilter, FilenameFilter metadataFilter)
 221  
             throws AnalysisException {
 222  3
         final File temp = getNextTempDirectory();
 223  3
         LOGGER.debug("{} exists? {}", temp, temp.exists());
 224  
         try {
 225  6
             ExtractionUtil.extractFilesUsingFilter(
 226  3
                     new File(dependency.getActualFilePath()), temp,
 227  
                     metadataFilter);
 228  0
         } catch (ExtractionException ex) {
 229  0
             throw new AnalysisException(ex);
 230  3
         }
 231  
 
 232  3
         File matchingFile = getMatchingFile(temp, folderFilter);
 233  3
         if (matchingFile != null) {
 234  3
             matchingFile = getMatchingFile(matchingFile, metadataFilter);
 235  3
             if (matchingFile != null) {
 236  3
                 collectWheelMetadata(dependency, matchingFile);
 237  
             }
 238  
         }
 239  3
     }
 240  
 
 241  
     /**
 242  
      * Makes sure a usable temporary directory is available.
 243  
      *
 244  
      * @throws InitializationException an AnalyzeException is thrown when the
 245  
      * temp directory cannot be created
 246  
      */
 247  
     @Override
 248  
     protected void initializeFileTypeAnalyzer() throws InitializationException {
 249  
         try {
 250  8
             final File baseDir = Settings.getTempDirectory();
 251  8
             tempFileLocation = File.createTempFile("check", "tmp", baseDir);
 252  8
             if (!tempFileLocation.delete()) {
 253  0
                 setEnabled(false);
 254  0
                 final String msg = String.format(
 255  
                         "Unable to delete temporary file '%s'.",
 256  0
                         tempFileLocation.getAbsolutePath());
 257  0
                 throw new InitializationException(msg);
 258  
             }
 259  8
             if (!tempFileLocation.mkdirs()) {
 260  0
                 setEnabled(false);
 261  0
                 final String msg = String.format(
 262  
                         "Unable to create directory '%s'.",
 263  0
                         tempFileLocation.getAbsolutePath());
 264  0
                 throw new InitializationException(msg);
 265  
             }
 266  0
         } catch (IOException ex) {
 267  0
             setEnabled(false);
 268  0
             throw new InitializationException("Unable to create a temporary file", ex);
 269  8
         }
 270  8
     }
 271  
 
 272  
     /**
 273  
      * Deletes any files extracted from the Wheel during analysis.
 274  
      */
 275  
     @Override
 276  
     public void closeAnalyzer() {
 277  8
         if (tempFileLocation != null && tempFileLocation.exists()) {
 278  8
             LOGGER.debug("Attempting to delete temporary files");
 279  8
             final boolean success = FileUtils.delete(tempFileLocation);
 280  8
             if (!success && tempFileLocation.exists()) {
 281  0
                 final String[] l = tempFileLocation.list();
 282  0
                 if (l != null && l.length > 0) {
 283  0
                     LOGGER.warn("Failed to delete some temporary files, see the log for more details");
 284  
                 }
 285  
             }
 286  
         }
 287  8
     }
 288  
 
 289  
     /**
 290  
      * Gathers evidence from the METADATA file.
 291  
      *
 292  
      * @param dependency the dependency being analyzed
 293  
      * @param file a reference to the manifest/properties file
 294  
      */
 295  
     private static void collectWheelMetadata(Dependency dependency, File file) {
 296  6
         final InternetHeaders headers = getManifestProperties(file);
 297  6
         addPropertyToEvidence(headers, dependency.getVersionEvidence(),
 298  
                 "Version", Confidence.HIGHEST);
 299  6
         addPropertyToEvidence(headers, dependency.getProductEvidence(), "Name",
 300  
                 Confidence.HIGHEST);
 301  6
         final String url = headers.getHeader("Home-page", null);
 302  6
         final EvidenceCollection vendorEvidence = dependency
 303  6
                 .getVendorEvidence();
 304  6
         if (StringUtils.isNotBlank(url)) {
 305  6
             if (UrlStringUtils.isUrl(url)) {
 306  6
                 vendorEvidence.addEvidence(METADATA, "vendor", url,
 307  
                         Confidence.MEDIUM);
 308  
             }
 309  
         }
 310  6
         addPropertyToEvidence(headers, vendorEvidence, "Author", Confidence.LOW);
 311  6
         final String summary = headers.getHeader("Summary", null);
 312  6
         if (StringUtils.isNotBlank(summary)) {
 313  6
             JarAnalyzer
 314  6
                     .addDescription(dependency, summary, METADATA, "summary");
 315  
         }
 316  6
     }
 317  
 
 318  
     /**
 319  
      * Adds a value to the evidence collection.
 320  
      *
 321  
      * @param headers the properties collection
 322  
      * @param evidence the evidence collection to add the value
 323  
      * @param property the property name
 324  
      * @param confidence the confidence of the evidence
 325  
      */
 326  
     private static void addPropertyToEvidence(InternetHeaders headers,
 327  
             EvidenceCollection evidence, String property, Confidence confidence) {
 328  18
         final String value = headers.getHeader(property, null);
 329  18
         LOGGER.debug("Property: {}, Value: {}", property, value);
 330  18
         if (StringUtils.isNotBlank(value)) {
 331  18
             evidence.addEvidence(METADATA, property, value, confidence);
 332  
         }
 333  18
     }
 334  
 
 335  
     /**
 336  
      * Returns a list of files that match the given filter, this does not
 337  
      * recursively scan the directory.
 338  
      *
 339  
      * @param folder the folder to filter
 340  
      * @param filter the filter to apply to the files in the directory
 341  
      * @return the list of Files in the directory that match the provided filter
 342  
      */
 343  
     private static File getMatchingFile(File folder, FilenameFilter filter) {
 344  6
         File result = null;
 345  6
         final File[] matches = folder.listFiles(filter);
 346  6
         if (null != matches && 1 == matches.length) {
 347  6
             result = matches[0];
 348  
         }
 349  6
         return result;
 350  
     }
 351  
 
 352  
     /**
 353  
      * Reads the manifest entries from the provided file.
 354  
      *
 355  
      * @param manifest the manifest
 356  
      * @return the manifest entries
 357  
      */
 358  
     private static InternetHeaders getManifestProperties(File manifest) {
 359  6
         final InternetHeaders result = new InternetHeaders();
 360  6
         if (null == manifest) {
 361  0
             LOGGER.debug("Manifest file not found.");
 362  
         } else {
 363  6
             InputStream in = null;
 364  
             try {
 365  6
                 in = new BufferedInputStream(new FileInputStream(manifest));
 366  6
                 result.load(in);
 367  0
             } catch (MessagingException e) {
 368  0
                 LOGGER.warn(e.getMessage(), e);
 369  0
             } catch (FileNotFoundException e) {
 370  0
                 LOGGER.warn(e.getMessage(), e);
 371  
             } finally {
 372  6
                 if (in != null) {
 373  
                     try {
 374  6
                         in.close();
 375  0
                     } catch (IOException ex) {
 376  0
                         LOGGER.debug("failed to close input stream", ex);
 377  6
                     }
 378  
                 }
 379  
             }
 380  
         }
 381  6
         return result;
 382  
     }
 383  
 
 384  
     /**
 385  
      * Retrieves the next temporary destination directory for extracting an
 386  
      * archive.
 387  
      *
 388  
      * @return a directory
 389  
      * @throws AnalysisException thrown if unable to create temporary directory
 390  
      */
 391  
     private File getNextTempDirectory() throws AnalysisException {
 392  
         File directory;
 393  
 
 394  
         // getting an exception for some directories not being able to be
 395  
         // created; might be because the directory already exists?
 396  
         do {
 397  3
             final int dirCount = DIR_COUNT.incrementAndGet();
 398  3
             directory = new File(tempFileLocation, String.valueOf(dirCount));
 399  3
         } while (directory.exists());
 400  3
         if (!directory.mkdirs()) {
 401  0
             throw new AnalysisException(String.format(
 402  
                     "Unable to create temp directory '%s'.",
 403  0
                     directory.getAbsolutePath()));
 404  
         }
 405  3
         return directory;
 406  
     }
 407  
 }