| 1 | |
|
| 2 | |
|
| 3 | |
|
| 4 | |
|
| 5 | |
|
| 6 | |
|
| 7 | |
|
| 8 | |
|
| 9 | |
|
| 10 | |
|
| 11 | |
|
| 12 | |
|
| 13 | |
|
| 14 | |
|
| 15 | |
|
| 16 | |
|
| 17 | |
|
| 18 | |
package org.owasp.dependencycheck.analyzer; |
| 19 | |
|
| 20 | |
import java.io.BufferedInputStream; |
| 21 | |
import java.io.File; |
| 22 | |
import java.io.FileFilter; |
| 23 | |
import java.io.FileInputStream; |
| 24 | |
import java.io.FileNotFoundException; |
| 25 | |
import java.io.FilenameFilter; |
| 26 | |
import org.apache.commons.io.filefilter.NameFileFilter; |
| 27 | |
import org.apache.commons.io.filefilter.SuffixFileFilter; |
| 28 | |
import org.apache.commons.io.input.AutoCloseInputStream; |
| 29 | |
import org.apache.commons.lang3.StringUtils; |
| 30 | |
import org.owasp.dependencycheck.Engine; |
| 31 | |
import org.owasp.dependencycheck.analyzer.exception.AnalysisException; |
| 32 | |
import org.owasp.dependencycheck.dependency.Confidence; |
| 33 | |
import org.owasp.dependencycheck.dependency.Dependency; |
| 34 | |
import org.owasp.dependencycheck.dependency.EvidenceCollection; |
| 35 | |
import org.slf4j.Logger; |
| 36 | |
import org.slf4j.LoggerFactory; |
| 37 | |
|
| 38 | |
import javax.mail.MessagingException; |
| 39 | |
import javax.mail.internet.InternetHeaders; |
| 40 | |
import org.owasp.dependencycheck.utils.ExtractionException; |
| 41 | |
import org.owasp.dependencycheck.utils.ExtractionUtil; |
| 42 | |
import org.owasp.dependencycheck.utils.FileFilterBuilder; |
| 43 | |
import org.owasp.dependencycheck.utils.FileUtils; |
| 44 | |
import org.owasp.dependencycheck.utils.Settings; |
| 45 | |
import org.owasp.dependencycheck.utils.UrlStringUtils; |
| 46 | |
|
| 47 | |
|
| 48 | |
|
| 49 | |
|
| 50 | |
|
| 51 | |
|
| 52 | |
|
| 53 | 12 | public class PythonDistributionAnalyzer extends AbstractFileTypeAnalyzer { |
| 54 | |
|
| 55 | |
|
| 56 | |
|
| 57 | |
|
| 58 | |
private static final String PKG_INFO = "PKG-INFO"; |
| 59 | |
|
| 60 | |
|
| 61 | |
|
| 62 | |
|
| 63 | |
private static final String METADATA = "METADATA"; |
| 64 | |
|
| 65 | |
|
| 66 | |
|
| 67 | |
|
| 68 | 1 | private static final Logger LOGGER = LoggerFactory |
| 69 | |
.getLogger(PythonDistributionAnalyzer.class); |
| 70 | |
|
| 71 | |
|
| 72 | |
|
| 73 | |
|
| 74 | 1 | private static int dirCount = 0; |
| 75 | |
|
| 76 | |
|
| 77 | |
|
| 78 | |
|
| 79 | |
private static final String ANALYZER_NAME = "Python Distribution Analyzer"; |
| 80 | |
|
| 81 | |
|
| 82 | |
|
| 83 | 1 | private static final AnalysisPhase ANALYSIS_PHASE = AnalysisPhase.INFORMATION_COLLECTION; |
| 84 | |
|
| 85 | |
|
| 86 | |
|
| 87 | |
|
| 88 | 1 | private static final String[] EXTENSIONS = {"whl", "egg", "zip"}; |
| 89 | |
|
| 90 | |
|
| 91 | |
|
| 92 | |
|
| 93 | 1 | private static final FileFilter EGG_OR_ZIP = FileFilterBuilder.newInstance().addExtensions("egg", "zip").build(); |
| 94 | |
|
| 95 | |
|
| 96 | |
|
| 97 | |
|
| 98 | 1 | private static final FileFilter WHL_FILTER = FileFilterBuilder.newInstance().addExtensions("whl").build(); |
| 99 | |
|
| 100 | |
|
| 101 | |
|
| 102 | |
|
| 103 | |
private File tempFileLocation; |
| 104 | |
|
| 105 | |
|
| 106 | |
|
| 107 | |
|
| 108 | 1 | private static final FilenameFilter DIST_INFO_FILTER = new SuffixFileFilter( |
| 109 | |
".dist-info"); |
| 110 | |
|
| 111 | |
|
| 112 | |
|
| 113 | |
|
| 114 | 1 | private static final FilenameFilter EGG_INFO_FILTER = new NameFileFilter( |
| 115 | |
"EGG-INFO"); |
| 116 | |
|
| 117 | |
|
| 118 | |
|
| 119 | |
|
| 120 | 1 | private static final NameFileFilter METADATA_FILTER = new NameFileFilter( |
| 121 | |
METADATA); |
| 122 | |
|
| 123 | |
|
| 124 | |
|
| 125 | |
|
| 126 | 1 | private static final NameFileFilter PKG_INFO_FILTER = new NameFileFilter( |
| 127 | |
PKG_INFO); |
| 128 | |
|
| 129 | |
|
| 130 | |
|
| 131 | |
|
| 132 | 1 | private static final FileFilter FILTER = FileFilterBuilder.newInstance().addFileFilters( |
| 133 | |
METADATA_FILTER, PKG_INFO_FILTER).addExtensions(EXTENSIONS).build(); |
| 134 | |
|
| 135 | |
|
| 136 | |
|
| 137 | |
|
| 138 | |
|
| 139 | |
|
| 140 | |
@Override |
| 141 | |
protected FileFilter getFileFilter() { |
| 142 | 858 | return FILTER; |
| 143 | |
} |
| 144 | |
|
| 145 | |
|
| 146 | |
|
| 147 | |
|
| 148 | |
|
| 149 | |
|
| 150 | |
@Override |
| 151 | |
public String getName() { |
| 152 | 5 | return ANALYZER_NAME; |
| 153 | |
} |
| 154 | |
|
| 155 | |
|
| 156 | |
|
| 157 | |
|
| 158 | |
|
| 159 | |
|
| 160 | |
@Override |
| 161 | |
public AnalysisPhase getAnalysisPhase() { |
| 162 | 3 | return ANALYSIS_PHASE; |
| 163 | |
} |
| 164 | |
|
| 165 | |
|
| 166 | |
|
| 167 | |
|
| 168 | |
|
| 169 | |
|
| 170 | |
@Override |
| 171 | |
protected String getAnalyzerEnabledSettingKey() { |
| 172 | 12 | return Settings.KEYS.ANALYZER_PYTHON_DISTRIBUTION_ENABLED; |
| 173 | |
} |
| 174 | |
|
| 175 | |
@Override |
| 176 | |
protected void analyzeFileType(Dependency dependency, Engine engine) |
| 177 | |
throws AnalysisException { |
| 178 | 6 | final File actualFile = dependency.getActualFile(); |
| 179 | 6 | if (WHL_FILTER.accept(actualFile)) { |
| 180 | 1 | collectMetadataFromArchiveFormat(dependency, DIST_INFO_FILTER, |
| 181 | |
METADATA_FILTER); |
| 182 | 5 | } else if (EGG_OR_ZIP.accept(actualFile)) { |
| 183 | 2 | collectMetadataFromArchiveFormat(dependency, EGG_INFO_FILTER, |
| 184 | |
PKG_INFO_FILTER); |
| 185 | |
} else { |
| 186 | 3 | final String name = actualFile.getName(); |
| 187 | 3 | final boolean metadata = METADATA.equals(name); |
| 188 | 3 | if (metadata || PKG_INFO.equals(name)) { |
| 189 | 3 | final File parent = actualFile.getParentFile(); |
| 190 | 3 | final String parentName = parent.getName(); |
| 191 | 3 | dependency.setDisplayFileName(parentName + "/" + name); |
| 192 | 3 | if (parent.isDirectory() |
| 193 | |
&& (metadata && parentName.endsWith(".dist-info") |
| 194 | |
|| parentName.endsWith(".egg-info") || "EGG-INFO" |
| 195 | |
.equals(parentName))) { |
| 196 | 3 | collectWheelMetadata(dependency, actualFile); |
| 197 | |
} |
| 198 | |
} |
| 199 | |
} |
| 200 | 6 | } |
| 201 | |
|
| 202 | |
|
| 203 | |
|
| 204 | |
|
| 205 | |
|
| 206 | |
|
| 207 | |
|
| 208 | |
|
| 209 | |
|
| 210 | |
private void collectMetadataFromArchiveFormat(Dependency dependency, |
| 211 | |
FilenameFilter folderFilter, FilenameFilter metadataFilter) |
| 212 | |
throws AnalysisException { |
| 213 | 3 | final File temp = getNextTempDirectory(); |
| 214 | 3 | LOGGER.debug("{} exists? {}", temp, temp.exists()); |
| 215 | |
try { |
| 216 | 3 | ExtractionUtil.extractFilesUsingFilter( |
| 217 | |
new File(dependency.getActualFilePath()), temp, |
| 218 | |
metadataFilter); |
| 219 | 0 | } catch (ExtractionException ex) { |
| 220 | 0 | throw new AnalysisException(ex); |
| 221 | 3 | } |
| 222 | |
|
| 223 | 3 | collectWheelMetadata( |
| 224 | |
dependency, |
| 225 | |
getMatchingFile(getMatchingFile(temp, folderFilter), |
| 226 | |
metadataFilter)); |
| 227 | 3 | } |
| 228 | |
|
| 229 | |
|
| 230 | |
|
| 231 | |
|
| 232 | |
|
| 233 | |
|
| 234 | |
@Override |
| 235 | |
protected void initializeFileTypeAnalyzer() throws Exception { |
| 236 | 8 | final File baseDir = Settings.getTempDirectory(); |
| 237 | 8 | tempFileLocation = File.createTempFile("check", "tmp", baseDir); |
| 238 | 8 | if (!tempFileLocation.delete()) { |
| 239 | 0 | final String msg = String.format( |
| 240 | |
"Unable to delete temporary file '%s'.", |
| 241 | |
tempFileLocation.getAbsolutePath()); |
| 242 | 0 | throw new AnalysisException(msg); |
| 243 | |
} |
| 244 | 8 | if (!tempFileLocation.mkdirs()) { |
| 245 | 0 | final String msg = String.format( |
| 246 | |
"Unable to create directory '%s'.", |
| 247 | |
tempFileLocation.getAbsolutePath()); |
| 248 | 0 | throw new AnalysisException(msg); |
| 249 | |
} |
| 250 | 8 | } |
| 251 | |
|
| 252 | |
|
| 253 | |
|
| 254 | |
|
| 255 | |
@Override |
| 256 | |
public void close() { |
| 257 | 9 | if (tempFileLocation != null && tempFileLocation.exists()) { |
| 258 | 8 | LOGGER.debug("Attempting to delete temporary files"); |
| 259 | 8 | final boolean success = FileUtils.delete(tempFileLocation); |
| 260 | 8 | if (!success) { |
| 261 | 1 | LOGGER.warn( |
| 262 | |
"Failed to delete some temporary files, see the log for more details"); |
| 263 | |
} |
| 264 | |
} |
| 265 | 9 | } |
| 266 | |
|
| 267 | |
|
| 268 | |
|
| 269 | |
|
| 270 | |
|
| 271 | |
|
| 272 | |
|
| 273 | |
private static void collectWheelMetadata(Dependency dependency, File file) { |
| 274 | 6 | final InternetHeaders headers = getManifestProperties(file); |
| 275 | 6 | addPropertyToEvidence(headers, dependency.getVersionEvidence(), |
| 276 | |
"Version", Confidence.HIGHEST); |
| 277 | 6 | addPropertyToEvidence(headers, dependency.getProductEvidence(), "Name", |
| 278 | |
Confidence.HIGHEST); |
| 279 | 6 | final String url = headers.getHeader("Home-page", null); |
| 280 | 6 | final EvidenceCollection vendorEvidence = dependency |
| 281 | |
.getVendorEvidence(); |
| 282 | 6 | if (StringUtils.isNotBlank(url)) { |
| 283 | 6 | if (UrlStringUtils.isUrl(url)) { |
| 284 | 6 | vendorEvidence.addEvidence(METADATA, "vendor", url, |
| 285 | |
Confidence.MEDIUM); |
| 286 | |
} |
| 287 | |
} |
| 288 | 6 | addPropertyToEvidence(headers, vendorEvidence, "Author", Confidence.LOW); |
| 289 | 6 | final String summary = headers.getHeader("Summary", null); |
| 290 | 6 | if (StringUtils.isNotBlank(summary)) { |
| 291 | 6 | JarAnalyzer |
| 292 | |
.addDescription(dependency, summary, METADATA, "summary"); |
| 293 | |
} |
| 294 | 6 | } |
| 295 | |
|
| 296 | |
|
| 297 | |
|
| 298 | |
|
| 299 | |
|
| 300 | |
|
| 301 | |
|
| 302 | |
|
| 303 | |
|
| 304 | |
private static void addPropertyToEvidence(InternetHeaders headers, |
| 305 | |
EvidenceCollection evidence, String property, Confidence confidence) { |
| 306 | 18 | final String value = headers.getHeader(property, null); |
| 307 | 18 | LOGGER.debug("Property: {}, Value: {}", property, value); |
| 308 | 18 | if (StringUtils.isNotBlank(value)) { |
| 309 | 18 | evidence.addEvidence(METADATA, property, value, confidence); |
| 310 | |
} |
| 311 | 18 | } |
| 312 | |
|
| 313 | |
|
| 314 | |
|
| 315 | |
|
| 316 | |
|
| 317 | |
|
| 318 | |
|
| 319 | |
|
| 320 | |
private static File getMatchingFile(File folder, FilenameFilter filter) { |
| 321 | 6 | File result = null; |
| 322 | 6 | final File[] matches = folder.listFiles(filter); |
| 323 | 6 | if (null != matches && 1 == matches.length) { |
| 324 | 6 | result = matches[0]; |
| 325 | |
} |
| 326 | 6 | return result; |
| 327 | |
} |
| 328 | |
|
| 329 | |
|
| 330 | |
|
| 331 | |
|
| 332 | |
|
| 333 | |
|
| 334 | |
|
| 335 | |
private static InternetHeaders getManifestProperties(File manifest) { |
| 336 | 6 | final InternetHeaders result = new InternetHeaders(); |
| 337 | 6 | if (null == manifest) { |
| 338 | 0 | LOGGER.debug("Manifest file not found."); |
| 339 | |
} else { |
| 340 | |
try { |
| 341 | 6 | result.load(new AutoCloseInputStream(new BufferedInputStream( |
| 342 | |
new FileInputStream(manifest)))); |
| 343 | 0 | } catch (MessagingException e) { |
| 344 | 0 | LOGGER.warn(e.getMessage(), e); |
| 345 | 0 | } catch (FileNotFoundException e) { |
| 346 | 0 | LOGGER.warn(e.getMessage(), e); |
| 347 | 6 | } |
| 348 | |
} |
| 349 | 6 | return result; |
| 350 | |
} |
| 351 | |
|
| 352 | |
|
| 353 | |
|
| 354 | |
|
| 355 | |
|
| 356 | |
|
| 357 | |
|
| 358 | |
private File getNextTempDirectory() throws AnalysisException { |
| 359 | |
File directory; |
| 360 | |
|
| 361 | |
|
| 362 | |
|
| 363 | |
do { |
| 364 | 3 | dirCount += 1; |
| 365 | 3 | directory = new File(tempFileLocation, String.valueOf(dirCount)); |
| 366 | 3 | } while (directory.exists()); |
| 367 | 3 | if (!directory.mkdirs()) { |
| 368 | 0 | throw new AnalysisException(String.format( |
| 369 | |
"Unable to create temp directory '%s'.", |
| 370 | |
directory.getAbsolutePath())); |
| 371 | |
} |
| 372 | 3 | return directory; |
| 373 | |
} |
| 374 | |
} |