| 1 | |
|
| 2 | |
|
| 3 | |
|
| 4 | |
|
| 5 | |
|
| 6 | |
|
| 7 | |
|
| 8 | |
|
| 9 | |
|
| 10 | |
|
| 11 | |
|
| 12 | |
|
| 13 | |
|
| 14 | |
|
| 15 | |
|
| 16 | |
|
| 17 | |
|
| 18 | |
package org.owasp.dependencycheck.analyzer; |
| 19 | |
|
| 20 | |
import java.io.BufferedInputStream; |
| 21 | |
import java.io.File; |
| 22 | |
import java.io.FileFilter; |
| 23 | |
import java.io.FileInputStream; |
| 24 | |
import java.io.FileNotFoundException; |
| 25 | |
import java.io.FilenameFilter; |
| 26 | |
import java.io.IOException; |
| 27 | |
import java.io.InputStream; |
| 28 | |
import org.apache.commons.io.filefilter.NameFileFilter; |
| 29 | |
import org.apache.commons.io.filefilter.SuffixFileFilter; |
| 30 | |
import org.apache.commons.lang3.StringUtils; |
| 31 | |
import org.owasp.dependencycheck.Engine; |
| 32 | |
import org.owasp.dependencycheck.analyzer.exception.AnalysisException; |
| 33 | |
import org.owasp.dependencycheck.dependency.Confidence; |
| 34 | |
import org.owasp.dependencycheck.dependency.Dependency; |
| 35 | |
import org.owasp.dependencycheck.dependency.EvidenceCollection; |
| 36 | |
import org.slf4j.Logger; |
| 37 | |
import org.slf4j.LoggerFactory; |
| 38 | |
|
| 39 | |
import javax.mail.MessagingException; |
| 40 | |
import javax.mail.internet.InternetHeaders; |
| 41 | |
import org.owasp.dependencycheck.exception.InitializationException; |
| 42 | |
import org.owasp.dependencycheck.utils.ExtractionException; |
| 43 | |
import org.owasp.dependencycheck.utils.ExtractionUtil; |
| 44 | |
import org.owasp.dependencycheck.utils.FileFilterBuilder; |
| 45 | |
import org.owasp.dependencycheck.utils.FileUtils; |
| 46 | |
import org.owasp.dependencycheck.utils.Settings; |
| 47 | |
import org.owasp.dependencycheck.utils.UrlStringUtils; |
| 48 | |
|
| 49 | |
|
| 50 | |
|
| 51 | |
|
| 52 | |
|
| 53 | |
|
| 54 | |
|
| 55 | |
|
| 56 | |
@Experimental |
| 57 | 14 | public class PythonDistributionAnalyzer extends AbstractFileTypeAnalyzer { |
| 58 | |
|
| 59 | |
|
| 60 | |
|
| 61 | |
|
| 62 | |
private static final String PKG_INFO = "PKG-INFO"; |
| 63 | |
|
| 64 | |
|
| 65 | |
|
| 66 | |
|
| 67 | |
private static final String METADATA = "METADATA"; |
| 68 | |
|
| 69 | |
|
| 70 | |
|
| 71 | |
|
| 72 | 1 | private static final Logger LOGGER = LoggerFactory |
| 73 | 1 | .getLogger(PythonDistributionAnalyzer.class); |
| 74 | |
|
| 75 | |
|
| 76 | |
|
| 77 | |
|
| 78 | |
|
| 79 | 1 | private static int dirCount = 0; |
| 80 | |
|
| 81 | |
|
| 82 | |
|
| 83 | |
|
| 84 | |
private static final String ANALYZER_NAME = "Python Distribution Analyzer"; |
| 85 | |
|
| 86 | |
|
| 87 | |
|
| 88 | 1 | private static final AnalysisPhase ANALYSIS_PHASE = AnalysisPhase.INFORMATION_COLLECTION; |
| 89 | |
|
| 90 | |
|
| 91 | |
|
| 92 | |
|
| 93 | 1 | private static final String[] EXTENSIONS = {"whl", "egg", "zip"}; |
| 94 | |
|
| 95 | |
|
| 96 | |
|
| 97 | |
|
| 98 | 1 | private static final FileFilter EGG_OR_ZIP = FileFilterBuilder.newInstance().addExtensions("egg", "zip").build(); |
| 99 | |
|
| 100 | |
|
| 101 | |
|
| 102 | |
|
| 103 | 1 | private static final FileFilter WHL_FILTER = FileFilterBuilder.newInstance().addExtensions("whl").build(); |
| 104 | |
|
| 105 | |
|
| 106 | |
|
| 107 | |
|
| 108 | |
private File tempFileLocation; |
| 109 | |
|
| 110 | |
|
| 111 | |
|
| 112 | |
|
| 113 | |
|
| 114 | 1 | private static final FilenameFilter DIST_INFO_FILTER = new SuffixFileFilter( |
| 115 | |
".dist-info"); |
| 116 | |
|
| 117 | |
|
| 118 | |
|
| 119 | |
|
| 120 | 1 | private static final FilenameFilter EGG_INFO_FILTER = new NameFileFilter( |
| 121 | |
"EGG-INFO"); |
| 122 | |
|
| 123 | |
|
| 124 | |
|
| 125 | |
|
| 126 | 1 | private static final NameFileFilter METADATA_FILTER = new NameFileFilter( |
| 127 | |
METADATA); |
| 128 | |
|
| 129 | |
|
| 130 | |
|
| 131 | |
|
| 132 | 1 | private static final NameFileFilter PKG_INFO_FILTER = new NameFileFilter( |
| 133 | |
PKG_INFO); |
| 134 | |
|
| 135 | |
|
| 136 | |
|
| 137 | |
|
| 138 | 2 | private static final FileFilter FILTER = FileFilterBuilder.newInstance().addFileFilters( |
| 139 | 1 | METADATA_FILTER, PKG_INFO_FILTER).addExtensions(EXTENSIONS).build(); |
| 140 | |
|
| 141 | |
|
| 142 | |
|
| 143 | |
|
| 144 | |
|
| 145 | |
|
| 146 | |
@Override |
| 147 | |
protected FileFilter getFileFilter() { |
| 148 | 864 | return FILTER; |
| 149 | |
} |
| 150 | |
|
| 151 | |
|
| 152 | |
|
| 153 | |
|
| 154 | |
|
| 155 | |
|
| 156 | |
@Override |
| 157 | |
public String getName() { |
| 158 | 15 | return ANALYZER_NAME; |
| 159 | |
} |
| 160 | |
|
| 161 | |
|
| 162 | |
|
| 163 | |
|
| 164 | |
|
| 165 | |
|
| 166 | |
@Override |
| 167 | |
public AnalysisPhase getAnalysisPhase() { |
| 168 | 4 | return ANALYSIS_PHASE; |
| 169 | |
} |
| 170 | |
|
| 171 | |
|
| 172 | |
|
| 173 | |
|
| 174 | |
|
| 175 | |
|
| 176 | |
|
| 177 | |
@Override |
| 178 | |
protected String getAnalyzerEnabledSettingKey() { |
| 179 | 14 | return Settings.KEYS.ANALYZER_PYTHON_DISTRIBUTION_ENABLED; |
| 180 | |
} |
| 181 | |
|
| 182 | |
@Override |
| 183 | |
protected void analyzeFileType(Dependency dependency, Engine engine) |
| 184 | |
throws AnalysisException { |
| 185 | 6 | final File actualFile = dependency.getActualFile(); |
| 186 | 6 | if (WHL_FILTER.accept(actualFile)) { |
| 187 | 1 | collectMetadataFromArchiveFormat(dependency, DIST_INFO_FILTER, |
| 188 | |
METADATA_FILTER); |
| 189 | 5 | } else if (EGG_OR_ZIP.accept(actualFile)) { |
| 190 | 2 | collectMetadataFromArchiveFormat(dependency, EGG_INFO_FILTER, |
| 191 | |
PKG_INFO_FILTER); |
| 192 | |
} else { |
| 193 | 3 | final String name = actualFile.getName(); |
| 194 | 3 | final boolean metadata = METADATA.equals(name); |
| 195 | 3 | if (metadata || PKG_INFO.equals(name)) { |
| 196 | 3 | final File parent = actualFile.getParentFile(); |
| 197 | 3 | final String parentName = parent.getName(); |
| 198 | 3 | dependency.setDisplayFileName(parentName + "/" + name); |
| 199 | 3 | if (parent.isDirectory() |
| 200 | 1 | && (metadata && parentName.endsWith(".dist-info") |
| 201 | 2 | || parentName.endsWith(".egg-info") || "EGG-INFO" |
| 202 | 1 | .equals(parentName))) { |
| 203 | 3 | collectWheelMetadata(dependency, actualFile); |
| 204 | |
} |
| 205 | |
} |
| 206 | |
} |
| 207 | 6 | } |
| 208 | |
|
| 209 | |
|
| 210 | |
|
| 211 | |
|
| 212 | |
|
| 213 | |
|
| 214 | |
|
| 215 | |
|
| 216 | |
|
| 217 | |
|
| 218 | |
private void collectMetadataFromArchiveFormat(Dependency dependency, |
| 219 | |
FilenameFilter folderFilter, FilenameFilter metadataFilter) |
| 220 | |
throws AnalysisException { |
| 221 | 3 | final File temp = getNextTempDirectory(); |
| 222 | 3 | LOGGER.debug("{} exists? {}", temp, temp.exists()); |
| 223 | |
try { |
| 224 | 6 | ExtractionUtil.extractFilesUsingFilter( |
| 225 | 3 | new File(dependency.getActualFilePath()), temp, |
| 226 | |
metadataFilter); |
| 227 | 0 | } catch (ExtractionException ex) { |
| 228 | 0 | throw new AnalysisException(ex); |
| 229 | 3 | } |
| 230 | |
|
| 231 | 3 | File matchingFile = getMatchingFile(temp, folderFilter); |
| 232 | 3 | if (matchingFile != null) { |
| 233 | 3 | matchingFile = getMatchingFile(matchingFile, metadataFilter); |
| 234 | 3 | if (matchingFile != null) { |
| 235 | 3 | collectWheelMetadata(dependency, matchingFile); |
| 236 | |
} |
| 237 | |
} |
| 238 | 3 | } |
| 239 | |
|
| 240 | |
|
| 241 | |
|
| 242 | |
|
| 243 | |
|
| 244 | |
|
| 245 | |
|
| 246 | |
@Override |
| 247 | |
protected void initializeFileTypeAnalyzer() throws InitializationException { |
| 248 | |
try { |
| 249 | 8 | final File baseDir = Settings.getTempDirectory(); |
| 250 | 8 | tempFileLocation = File.createTempFile("check", "tmp", baseDir); |
| 251 | 8 | if (!tempFileLocation.delete()) { |
| 252 | 0 | setEnabled(false); |
| 253 | 0 | final String msg = String.format( |
| 254 | |
"Unable to delete temporary file '%s'.", |
| 255 | 0 | tempFileLocation.getAbsolutePath()); |
| 256 | 0 | throw new InitializationException(msg); |
| 257 | |
} |
| 258 | 8 | if (!tempFileLocation.mkdirs()) { |
| 259 | 0 | setEnabled(false); |
| 260 | 0 | final String msg = String.format( |
| 261 | |
"Unable to create directory '%s'.", |
| 262 | 0 | tempFileLocation.getAbsolutePath()); |
| 263 | 0 | throw new InitializationException(msg); |
| 264 | |
} |
| 265 | 0 | } catch (IOException ex) { |
| 266 | 0 | setEnabled(false); |
| 267 | 0 | throw new InitializationException("Unable to create a temporary file", ex); |
| 268 | 8 | } |
| 269 | 8 | } |
| 270 | |
|
| 271 | |
|
| 272 | |
|
| 273 | |
|
| 274 | |
@Override |
| 275 | |
public void close() { |
| 276 | 10 | if (tempFileLocation != null && tempFileLocation.exists()) { |
| 277 | 8 | LOGGER.debug("Attempting to delete temporary files"); |
| 278 | 8 | final boolean success = FileUtils.delete(tempFileLocation); |
| 279 | 8 | if (!success) { |
| 280 | 0 | LOGGER.warn( |
| 281 | |
"Failed to delete some temporary files, see the log for more details"); |
| 282 | |
} |
| 283 | |
} |
| 284 | 10 | } |
| 285 | |
|
| 286 | |
|
| 287 | |
|
| 288 | |
|
| 289 | |
|
| 290 | |
|
| 291 | |
|
| 292 | |
private static void collectWheelMetadata(Dependency dependency, File file) { |
| 293 | 6 | final InternetHeaders headers = getManifestProperties(file); |
| 294 | 6 | addPropertyToEvidence(headers, dependency.getVersionEvidence(), |
| 295 | |
"Version", Confidence.HIGHEST); |
| 296 | 6 | addPropertyToEvidence(headers, dependency.getProductEvidence(), "Name", |
| 297 | |
Confidence.HIGHEST); |
| 298 | 6 | final String url = headers.getHeader("Home-page", null); |
| 299 | 6 | final EvidenceCollection vendorEvidence = dependency |
| 300 | 6 | .getVendorEvidence(); |
| 301 | 6 | if (StringUtils.isNotBlank(url)) { |
| 302 | 6 | if (UrlStringUtils.isUrl(url)) { |
| 303 | 6 | vendorEvidence.addEvidence(METADATA, "vendor", url, |
| 304 | |
Confidence.MEDIUM); |
| 305 | |
} |
| 306 | |
} |
| 307 | 6 | addPropertyToEvidence(headers, vendorEvidence, "Author", Confidence.LOW); |
| 308 | 6 | final String summary = headers.getHeader("Summary", null); |
| 309 | 6 | if (StringUtils.isNotBlank(summary)) { |
| 310 | 6 | JarAnalyzer |
| 311 | 6 | .addDescription(dependency, summary, METADATA, "summary"); |
| 312 | |
} |
| 313 | 6 | } |
| 314 | |
|
| 315 | |
|
| 316 | |
|
| 317 | |
|
| 318 | |
|
| 319 | |
|
| 320 | |
|
| 321 | |
|
| 322 | |
|
| 323 | |
private static void addPropertyToEvidence(InternetHeaders headers, |
| 324 | |
EvidenceCollection evidence, String property, Confidence confidence) { |
| 325 | 18 | final String value = headers.getHeader(property, null); |
| 326 | 18 | LOGGER.debug("Property: {}, Value: {}", property, value); |
| 327 | 18 | if (StringUtils.isNotBlank(value)) { |
| 328 | 18 | evidence.addEvidence(METADATA, property, value, confidence); |
| 329 | |
} |
| 330 | 18 | } |
| 331 | |
|
| 332 | |
|
| 333 | |
|
| 334 | |
|
| 335 | |
|
| 336 | |
|
| 337 | |
|
| 338 | |
|
| 339 | |
|
| 340 | |
private static File getMatchingFile(File folder, FilenameFilter filter) { |
| 341 | 6 | File result = null; |
| 342 | 6 | final File[] matches = folder.listFiles(filter); |
| 343 | 6 | if (null != matches && 1 == matches.length) { |
| 344 | 6 | result = matches[0]; |
| 345 | |
} |
| 346 | 6 | return result; |
| 347 | |
} |
| 348 | |
|
| 349 | |
|
| 350 | |
|
| 351 | |
|
| 352 | |
|
| 353 | |
|
| 354 | |
|
| 355 | |
private static InternetHeaders getManifestProperties(File manifest) { |
| 356 | 6 | final InternetHeaders result = new InternetHeaders(); |
| 357 | 6 | if (null == manifest) { |
| 358 | 0 | LOGGER.debug("Manifest file not found."); |
| 359 | |
} else { |
| 360 | 6 | InputStream in = null; |
| 361 | |
try { |
| 362 | 6 | in = new BufferedInputStream(new FileInputStream(manifest)); |
| 363 | 6 | result.load(in); |
| 364 | 0 | } catch (MessagingException e) { |
| 365 | 0 | LOGGER.warn(e.getMessage(), e); |
| 366 | 0 | } catch (FileNotFoundException e) { |
| 367 | 0 | LOGGER.warn(e.getMessage(), e); |
| 368 | |
} finally { |
| 369 | 6 | if (in != null) { |
| 370 | |
try { |
| 371 | 6 | in.close(); |
| 372 | 0 | } catch (IOException ex) { |
| 373 | 0 | LOGGER.debug("failed to close input stream", ex); |
| 374 | 6 | } |
| 375 | |
} |
| 376 | |
} |
| 377 | |
} |
| 378 | 6 | return result; |
| 379 | |
} |
| 380 | |
|
| 381 | |
|
| 382 | |
|
| 383 | |
|
| 384 | |
|
| 385 | |
|
| 386 | |
|
| 387 | |
|
| 388 | |
private File getNextTempDirectory() throws AnalysisException { |
| 389 | |
File directory; |
| 390 | |
|
| 391 | |
|
| 392 | |
|
| 393 | |
do { |
| 394 | 3 | dirCount += 1; |
| 395 | 3 | directory = new File(tempFileLocation, String.valueOf(dirCount)); |
| 396 | 3 | } while (directory.exists()); |
| 397 | 3 | if (!directory.mkdirs()) { |
| 398 | 0 | throw new AnalysisException(String.format( |
| 399 | |
"Unable to create temp directory '%s'.", |
| 400 | 0 | directory.getAbsolutePath())); |
| 401 | |
} |
| 402 | 3 | return directory; |
| 403 | |
} |
| 404 | |
} |