1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.owasp.dependencycheck.analyzer;
19
20 import java.io.File;
21 import java.io.FileFilter;
22 import java.io.FileOutputStream;
23 import java.io.IOException;
24 import java.io.InputStream;
25 import java.io.InputStreamReader;
26 import java.io.OutputStream;
27 import java.io.Reader;
28 import java.util.ArrayList;
29 import java.util.Collections;
30 import java.util.Enumeration;
31 import java.util.HashMap;
32 import java.util.List;
33 import java.util.Map;
34 import java.util.Map.Entry;
35 import java.util.Properties;
36 import java.util.Set;
37 import java.util.StringTokenizer;
38 import java.util.jar.Attributes;
39 import java.util.jar.JarEntry;
40 import java.util.jar.JarFile;
41 import java.util.jar.Manifest;
42 import java.util.regex.Pattern;
43 import java.util.zip.ZipEntry;
44 import org.apache.commons.compress.utils.IOUtils;
45 import org.apache.commons.io.FilenameUtils;
46 import org.jsoup.Jsoup;
47 import org.owasp.dependencycheck.Engine;
48 import org.owasp.dependencycheck.analyzer.exception.AnalysisException;
49 import org.owasp.dependencycheck.dependency.Confidence;
50 import org.owasp.dependencycheck.dependency.Dependency;
51 import org.owasp.dependencycheck.dependency.EvidenceCollection;
52 import org.owasp.dependencycheck.exception.InitializationException;
53 import org.owasp.dependencycheck.utils.FileFilterBuilder;
54 import org.owasp.dependencycheck.xml.pom.License;
55 import org.owasp.dependencycheck.xml.pom.PomUtils;
56 import org.owasp.dependencycheck.xml.pom.Model;
57 import org.owasp.dependencycheck.utils.FileUtils;
58 import org.owasp.dependencycheck.utils.Settings;
59 import org.slf4j.Logger;
60 import org.slf4j.LoggerFactory;
61
62
63
64
65
66
67
68 public class JarAnalyzer extends AbstractFileTypeAnalyzer {
69
70
71
72
73
74 private static final Logger LOGGER = LoggerFactory.getLogger(JarAnalyzer.class);
75
76
77
78
79 private static int dirCount = 0;
80
81
82
83 private static final String NEWLINE = System.getProperty("line.separator");
84
85
86
87
88 private static final Set<String> IGNORE_VALUES = newHashSet(
89 "Sun Java System Application Server");
90
91
92
93 private static final Set<String> IGNORE_KEYS = newHashSet(
94 "built-by",
95 "created-by",
96 "builtby",
97 "createdby",
98 "build-jdk",
99 "buildjdk",
100 "ant-version",
101 "antversion",
102 "dynamicimportpackage",
103 "dynamicimport-package",
104 "dynamic-importpackage",
105 "dynamic-import-package",
106 "import-package",
107 "ignore-package",
108 "export-package",
109 "importpackage",
110 "ignorepackage",
111 "exportpackage",
112 "sealed",
113 "manifest-version",
114 "archiver-version",
115 "manifestversion",
116 "archiverversion",
117 "classpath",
118 "class-path",
119 "tool",
120 "bundle-manifestversion",
121 "bundlemanifestversion",
122 "bundle-vendor",
123 "include-resource",
124 "embed-dependency",
125 "ipojo-components",
126 "ipojo-extension",
127 "eclipse-sourcereferences");
128
129
130
131
132 @SuppressWarnings("deprecation")
133 private static final String IMPLEMENTATION_VENDOR_ID = Attributes.Name.IMPLEMENTATION_VENDOR_ID
134 .toString();
135
136
137
138 private static final String BUNDLE_VERSION = "Bundle-Version";
139
140
141
142 private static final String BUNDLE_DESCRIPTION = "Bundle-Description";
143
144
145
146 private static final String BUNDLE_NAME = "Bundle-Name";
147
148
149
150 private static final Pattern HTML_DETECTION_PATTERN = Pattern.compile("\\<[a-z]+.*/?\\>", Pattern.CASE_INSENSITIVE);
151
152
153
154
155
156 public JarAnalyzer() {
157 }
158
159
160
161
162
163 private static final String ANALYZER_NAME = "Jar Analyzer";
164
165
166
167 private static final AnalysisPhase ANALYSIS_PHASE = AnalysisPhase.INFORMATION_COLLECTION;
168
169
170
171 private static final String[] EXTENSIONS = {"jar", "war"};
172
173
174
175
176 private static final FileFilter FILTER = FileFilterBuilder.newInstance().addExtensions(EXTENSIONS).build();
177
178
179
180
181
182
183 @Override
184 protected FileFilter getFileFilter() {
185 return FILTER;
186 }
187
188
189
190
191
192
193 @Override
194 public String getName() {
195 return ANALYZER_NAME;
196 }
197
198
199
200
201
202
203 @Override
204 public AnalysisPhase getAnalysisPhase() {
205 return ANALYSIS_PHASE;
206 }
207
208
209
210
211
212
213
214
215 @Override
216 protected String getAnalyzerEnabledSettingKey() {
217 return Settings.KEYS.ANALYZER_JAR_ENABLED;
218 }
219
220
221
222
223
224
225
226
227
228
229 @Override
230 public void analyzeFileType(Dependency dependency, Engine engine) throws AnalysisException {
231 try {
232 final List<ClassNameInformation> classNames = collectClassNames(dependency);
233 final String fileName = dependency.getFileName().toLowerCase();
234 if (classNames.isEmpty()
235 && (fileName.endsWith("-sources.jar")
236 || fileName.endsWith("-javadoc.jar")
237 || fileName.endsWith("-src.jar")
238 || fileName.endsWith("-doc.jar"))) {
239 engine.getDependencies().remove(dependency);
240 }
241 final boolean hasManifest = parseManifest(dependency, classNames);
242 final boolean hasPOM = analyzePOM(dependency, classNames, engine);
243 final boolean addPackagesAsEvidence = !(hasManifest && hasPOM);
244 analyzePackageNames(classNames, dependency, addPackagesAsEvidence);
245 } catch (IOException ex) {
246 throw new AnalysisException("Exception occurred reading the JAR file.", ex);
247 }
248 }
249
250
251
252
253
254
255
256
257
258
259
260
261
262 protected boolean analyzePOM(Dependency dependency, List<ClassNameInformation> classes, Engine engine) throws AnalysisException {
263 boolean foundSomething = false;
264 final JarFile jar;
265 try {
266 jar = new JarFile(dependency.getActualFilePath());
267 } catch (IOException ex) {
268 LOGGER.warn("Unable to read JarFile '{}'.", dependency.getActualFilePath());
269 LOGGER.trace("", ex);
270 return false;
271 }
272 List<String> pomEntries;
273 try {
274 pomEntries = retrievePomListing(jar);
275 } catch (IOException ex) {
276 LOGGER.warn("Unable to read Jar file entries in '{}'.", dependency.getActualFilePath());
277 LOGGER.trace("", ex);
278 return false;
279 }
280 File externalPom = null;
281 if (pomEntries.isEmpty()) {
282 final String pomPath = FilenameUtils.removeExtension(dependency.getActualFilePath()) + ".pom";
283 externalPom = new File(pomPath);
284 if (externalPom.isFile()) {
285 pomEntries.add(pomPath);
286 } else {
287 return false;
288 }
289 }
290 for (String path : pomEntries) {
291 LOGGER.debug("Reading pom entry: {}", path);
292 Properties pomProperties = null;
293 try {
294 if (externalPom == null) {
295 pomProperties = retrievePomProperties(path, jar);
296 }
297 } catch (IOException ex) {
298 LOGGER.trace("ignore this, failed reading a non-existent pom.properties", ex);
299 }
300 Model pom = null;
301 try {
302 if (pomEntries.size() > 1) {
303
304 final Dependency newDependency = new Dependency();
305 pom = extractPom(path, jar, newDependency);
306
307 final String displayPath = String.format("%s%s%s",
308 dependency.getFilePath(),
309 File.separator,
310 path);
311 final String displayName = String.format("%s%s%s",
312 dependency.getFileName(),
313 File.separator,
314 path);
315
316 newDependency.setFileName(displayName);
317 newDependency.setFilePath(displayPath);
318 pom.processProperties(pomProperties);
319 setPomEvidence(newDependency, pom, null);
320 engine.getDependencies().add(newDependency);
321 Collections.sort(engine.getDependencies());
322 } else {
323 if (externalPom == null) {
324 pom = PomUtils.readPom(path, jar);
325 } else {
326 pom = PomUtils.readPom(externalPom);
327 }
328 if (pom != null) {
329 pom.processProperties(pomProperties);
330 foundSomething |= setPomEvidence(dependency, pom, classes);
331 }
332 }
333 } catch (AnalysisException ex) {
334 LOGGER.warn("An error occurred while analyzing '{}'.", dependency.getActualFilePath());
335 LOGGER.trace("", ex);
336 }
337 }
338 return foundSomething;
339 }
340
341
342
343
344
345
346
347
348
349
350
351 private Properties retrievePomProperties(String path, final JarFile jar) throws IOException {
352 Properties pomProperties = null;
353 final String propPath = path.substring(0, path.length() - 7) + "pom.properies";
354 final ZipEntry propEntry = jar.getEntry(propPath);
355 if (propEntry != null) {
356 Reader reader = null;
357 try {
358 reader = new InputStreamReader(jar.getInputStream(propEntry), "UTF-8");
359 pomProperties = new Properties();
360 pomProperties.load(reader);
361 LOGGER.debug("Read pom.properties: {}", propPath);
362 } finally {
363 if (reader != null) {
364 try {
365 reader.close();
366 } catch (IOException ex) {
367 LOGGER.trace("close error", ex);
368 }
369 }
370 }
371 }
372 return pomProperties;
373 }
374
375
376
377
378
379
380
381
382
383 private List<String> retrievePomListing(final JarFile jar) throws IOException {
384 final List<String> pomEntries = new ArrayList<String>();
385 final Enumeration<JarEntry> entries = jar.entries();
386 while (entries.hasMoreElements()) {
387 final JarEntry entry = entries.nextElement();
388 final String entryName = (new File(entry.getName())).getName().toLowerCase();
389 if (!entry.isDirectory() && "pom.xml".equals(entryName)) {
390 LOGGER.trace("POM Entry found: {}", entry.getName());
391 pomEntries.add(entry.getName());
392 }
393 }
394 return pomEntries;
395 }
396
397
398
399
400
401
402
403
404
405
406
407 private Model extractPom(String path, JarFile jar, Dependency dependency) throws AnalysisException {
408 InputStream input = null;
409 FileOutputStream fos = null;
410 final File tmpDir = getNextTempDirectory();
411 final File file = new File(tmpDir, "pom.xml");
412 try {
413 final ZipEntry entry = jar.getEntry(path);
414 if (entry == null) {
415 throw new AnalysisException(String.format("Pom (%s)does not exist in %s", path, jar.getName()));
416 }
417 input = jar.getInputStream(entry);
418 fos = new FileOutputStream(file);
419 IOUtils.copy(input, fos);
420 dependency.setActualFilePath(file.getAbsolutePath());
421 } catch (IOException ex) {
422 LOGGER.warn("An error occurred reading '{}' from '{}'.", path, dependency.getFilePath());
423 LOGGER.error("", ex);
424 } finally {
425 closeStream(fos);
426 closeStream(input);
427 }
428 return PomUtils.readPom(file);
429 }
430
431
432
433
434
435
436 private void closeStream(InputStream stream) {
437 if (stream != null) {
438 try {
439 stream.close();
440 } catch (IOException ex) {
441 LOGGER.trace("", ex);
442 }
443 }
444 }
445
446
447
448
449
450
451 private void closeStream(OutputStream stream) {
452 if (stream != null) {
453 try {
454 stream.close();
455 } catch (IOException ex) {
456 LOGGER.trace("", ex);
457 }
458 }
459 }
460
461
462
463
464
465
466
467
468
469
470
471 public static boolean setPomEvidence(Dependency dependency, Model pom, List<ClassNameInformation> classes) {
472 boolean foundSomething = false;
473 boolean addAsIdentifier = true;
474 if (pom == null) {
475 return foundSomething;
476 }
477 String groupid = pom.getGroupId();
478 String parentGroupId = pom.getParentGroupId();
479 String artifactid = pom.getArtifactId();
480 String parentArtifactId = pom.getParentArtifactId();
481 String version = pom.getVersion();
482 String parentVersion = pom.getParentVersion();
483
484 if ("org.sonatype.oss".equals(parentGroupId) && "oss-parent".equals(parentArtifactId)) {
485 parentGroupId = null;
486 parentArtifactId = null;
487 parentVersion = null;
488 }
489
490 if ((groupid == null || groupid.isEmpty()) && parentGroupId != null && !parentGroupId.isEmpty()) {
491 groupid = parentGroupId;
492 }
493
494 final String originalGroupID = groupid;
495 if (groupid != null && (groupid.startsWith("org.") || groupid.startsWith("com."))) {
496 groupid = groupid.substring(4);
497 }
498
499 if ((artifactid == null || artifactid.isEmpty()) && parentArtifactId != null && !parentArtifactId.isEmpty()) {
500 artifactid = parentArtifactId;
501 }
502
503 final String originalArtifactID = artifactid;
504 if (artifactid != null && (artifactid.startsWith("org.") || artifactid.startsWith("com."))) {
505 artifactid = artifactid.substring(4);
506 }
507
508 if ((version == null || version.isEmpty()) && parentVersion != null && !parentVersion.isEmpty()) {
509 version = parentVersion;
510 }
511
512 if (groupid != null && !groupid.isEmpty()) {
513 foundSomething = true;
514 dependency.getVendorEvidence().addEvidence("pom", "groupid", groupid, Confidence.HIGHEST);
515 dependency.getProductEvidence().addEvidence("pom", "groupid", groupid, Confidence.LOW);
516 addMatchingValues(classes, groupid, dependency.getVendorEvidence());
517 addMatchingValues(classes, groupid, dependency.getProductEvidence());
518 if (parentGroupId != null && !parentGroupId.isEmpty() && !parentGroupId.equals(groupid)) {
519 dependency.getVendorEvidence().addEvidence("pom", "parent-groupid", parentGroupId, Confidence.MEDIUM);
520 dependency.getProductEvidence().addEvidence("pom", "parent-groupid", parentGroupId, Confidence.LOW);
521 addMatchingValues(classes, parentGroupId, dependency.getVendorEvidence());
522 addMatchingValues(classes, parentGroupId, dependency.getProductEvidence());
523 }
524 } else {
525 addAsIdentifier = false;
526 }
527
528 if (artifactid != null && !artifactid.isEmpty()) {
529 foundSomething = true;
530 dependency.getProductEvidence().addEvidence("pom", "artifactid", artifactid, Confidence.HIGHEST);
531 dependency.getVendorEvidence().addEvidence("pom", "artifactid", artifactid, Confidence.LOW);
532 addMatchingValues(classes, artifactid, dependency.getVendorEvidence());
533 addMatchingValues(classes, artifactid, dependency.getProductEvidence());
534 if (parentArtifactId != null && !parentArtifactId.isEmpty() && !parentArtifactId.equals(artifactid)) {
535 dependency.getProductEvidence().addEvidence("pom", "parent-artifactid", parentArtifactId, Confidence.MEDIUM);
536 dependency.getVendorEvidence().addEvidence("pom", "parent-artifactid", parentArtifactId, Confidence.LOW);
537 addMatchingValues(classes, parentArtifactId, dependency.getVendorEvidence());
538 addMatchingValues(classes, parentArtifactId, dependency.getProductEvidence());
539 }
540 } else {
541 addAsIdentifier = false;
542 }
543
544 if (version != null && !version.isEmpty()) {
545 foundSomething = true;
546 dependency.getVersionEvidence().addEvidence("pom", "version", version, Confidence.HIGHEST);
547 if (parentVersion != null && !parentVersion.isEmpty() && !parentVersion.equals(version)) {
548 dependency.getVersionEvidence().addEvidence("pom", "parent-version", version, Confidence.LOW);
549 }
550 } else {
551 addAsIdentifier = false;
552 }
553
554 if (addAsIdentifier) {
555 dependency.addIdentifier("maven", String.format("%s:%s:%s", originalGroupID, originalArtifactID, version), null, Confidence.HIGH);
556 }
557
558
559 final String org = pom.getOrganization();
560 if (org != null && !org.isEmpty()) {
561 dependency.getVendorEvidence().addEvidence("pom", "organization name", org, Confidence.HIGH);
562 dependency.getProductEvidence().addEvidence("pom", "organization name", org, Confidence.LOW);
563 addMatchingValues(classes, org, dependency.getVendorEvidence());
564 addMatchingValues(classes, org, dependency.getProductEvidence());
565 }
566
567 final String pomName = pom.getName();
568 if (pomName
569 != null && !pomName.isEmpty()) {
570 foundSomething = true;
571 dependency.getProductEvidence().addEvidence("pom", "name", pomName, Confidence.HIGH);
572 dependency.getVendorEvidence().addEvidence("pom", "name", pomName, Confidence.HIGH);
573 addMatchingValues(classes, pomName, dependency.getVendorEvidence());
574 addMatchingValues(classes, pomName, dependency.getProductEvidence());
575 }
576
577
578 final String description = pom.getDescription();
579 if (description != null && !description.isEmpty() && !description.startsWith("POM was created by")) {
580 foundSomething = true;
581 final String trimmedDescription = addDescription(dependency, description, "pom", "description");
582 addMatchingValues(classes, trimmedDescription, dependency.getVendorEvidence());
583 addMatchingValues(classes, trimmedDescription, dependency.getProductEvidence());
584 }
585
586 final String projectURL = pom.getProjectURL();
587 if (projectURL != null && !projectURL.trim().isEmpty()) {
588 dependency.getVendorEvidence().addEvidence("pom", "url", projectURL, Confidence.HIGHEST);
589 }
590
591 extractLicense(pom, dependency);
592 return foundSomething;
593 }
594
595
596
597
598
599
600
601
602
603
604
605
606 protected void analyzePackageNames(List<ClassNameInformation> classNames,
607 Dependency dependency, boolean addPackagesAsEvidence) {
608 final Map<String, Integer> vendorIdentifiers = new HashMap<String, Integer>();
609 final Map<String, Integer> productIdentifiers = new HashMap<String, Integer>();
610 analyzeFullyQualifiedClassNames(classNames, vendorIdentifiers, productIdentifiers);
611
612 final int classCount = classNames.size();
613 final EvidenceCollection vendor = dependency.getVendorEvidence();
614 final EvidenceCollection product = dependency.getProductEvidence();
615
616 for (Map.Entry<String, Integer> entry : vendorIdentifiers.entrySet()) {
617 final float ratio = entry.getValue() / (float) classCount;
618 if (ratio > 0.5) {
619
620 vendor.addWeighting(entry.getKey());
621 if (addPackagesAsEvidence && entry.getKey().length() > 1) {
622 vendor.addEvidence("jar", "package name", entry.getKey(), Confidence.LOW);
623 }
624 }
625 }
626 for (Map.Entry<String, Integer> entry : productIdentifiers.entrySet()) {
627 final float ratio = entry.getValue() / (float) classCount;
628 if (ratio > 0.5) {
629 product.addWeighting(entry.getKey());
630 if (addPackagesAsEvidence && entry.getKey().length() > 1) {
631 product.addEvidence("jar", "package name", entry.getKey(), Confidence.LOW);
632 }
633 }
634 }
635 }
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653 protected boolean parseManifest(Dependency dependency, List<ClassNameInformation> classInformation) throws IOException {
654 boolean foundSomething = false;
655 JarFile jar = null;
656 try {
657 jar = new JarFile(dependency.getActualFilePath());
658 final Manifest manifest = jar.getManifest();
659 if (manifest == null) {
660 if (!dependency.getFileName().toLowerCase().endsWith("-sources.jar")
661 && !dependency.getFileName().toLowerCase().endsWith("-javadoc.jar")
662 && !dependency.getFileName().toLowerCase().endsWith("-src.jar")
663 && !dependency.getFileName().toLowerCase().endsWith("-doc.jar")) {
664 LOGGER.debug("Jar file '{}' does not contain a manifest.",
665 dependency.getFileName());
666 }
667 return false;
668 }
669 final EvidenceCollection vendorEvidence = dependency.getVendorEvidence();
670 final EvidenceCollection productEvidence = dependency.getProductEvidence();
671 final EvidenceCollection versionEvidence = dependency.getVersionEvidence();
672 String source = "Manifest";
673 String specificationVersion = null;
674 boolean hasImplementationVersion = false;
675 Attributes atts = manifest.getMainAttributes();
676 for (Entry<Object, Object> entry : atts.entrySet()) {
677 String key = entry.getKey().toString();
678 String value = atts.getValue(key);
679 if (HTML_DETECTION_PATTERN.matcher(value).find()) {
680 value = Jsoup.parse(value).text();
681 }
682 if (IGNORE_VALUES.contains(value)) {
683 continue;
684 } else if (key.equalsIgnoreCase(Attributes.Name.IMPLEMENTATION_TITLE.toString())) {
685 foundSomething = true;
686 productEvidence.addEvidence(source, key, value, Confidence.HIGH);
687 addMatchingValues(classInformation, value, productEvidence);
688 } else if (key.equalsIgnoreCase(Attributes.Name.IMPLEMENTATION_VERSION.toString())) {
689 hasImplementationVersion = true;
690 foundSomething = true;
691 versionEvidence.addEvidence(source, key, value, Confidence.HIGH);
692 } else if ("specification-version".equalsIgnoreCase(key)) {
693 specificationVersion = value;
694 } else if (key.equalsIgnoreCase(Attributes.Name.IMPLEMENTATION_VENDOR.toString())) {
695 foundSomething = true;
696 vendorEvidence.addEvidence(source, key, value, Confidence.HIGH);
697 addMatchingValues(classInformation, value, vendorEvidence);
698 } else if (key.equalsIgnoreCase(IMPLEMENTATION_VENDOR_ID)) {
699 foundSomething = true;
700 vendorEvidence.addEvidence(source, key, value, Confidence.MEDIUM);
701 addMatchingValues(classInformation, value, vendorEvidence);
702 } else if (key.equalsIgnoreCase(BUNDLE_DESCRIPTION)) {
703 foundSomething = true;
704 addDescription(dependency, value, "manifest", key);
705 addMatchingValues(classInformation, value, productEvidence);
706 } else if (key.equalsIgnoreCase(BUNDLE_NAME)) {
707 foundSomething = true;
708 productEvidence.addEvidence(source, key, value, Confidence.MEDIUM);
709 addMatchingValues(classInformation, value, productEvidence);
710
711
712 } else if (key.equalsIgnoreCase(BUNDLE_VERSION)) {
713 foundSomething = true;
714 versionEvidence.addEvidence(source, key, value, Confidence.HIGH);
715 } else if (key.equalsIgnoreCase(Attributes.Name.MAIN_CLASS.toString())) {
716 continue;
717
718 } else {
719 key = key.toLowerCase();
720 if (!IGNORE_KEYS.contains(key)
721 && !key.endsWith("jdk")
722 && !key.contains("lastmodified")
723 && !key.endsWith("package")
724 && !key.endsWith("classpath")
725 && !key.endsWith("class-path")
726 && !key.endsWith("-scm")
727 && !key.startsWith("scm-")
728 && !value.trim().startsWith("scm:")
729 && !isImportPackage(key, value)
730 && !isPackage(key, value)) {
731 foundSomething = true;
732 if (key.contains("version")) {
733 if (!key.contains("specification")) {
734 versionEvidence.addEvidence(source, key, value, Confidence.MEDIUM);
735 }
736 } else if ("build-id".equals(key)) {
737 int pos = value.indexOf('(');
738 if (pos >= 0) {
739 value = value.substring(0, pos - 1);
740 }
741 pos = value.indexOf('[');
742 if (pos >= 0) {
743 value = value.substring(0, pos - 1);
744 }
745 versionEvidence.addEvidence(source, key, value, Confidence.MEDIUM);
746 } else if (key.contains("title")) {
747 productEvidence.addEvidence(source, key, value, Confidence.MEDIUM);
748 addMatchingValues(classInformation, value, productEvidence);
749 } else if (key.contains("vendor")) {
750 if (key.contains("specification")) {
751 vendorEvidence.addEvidence(source, key, value, Confidence.LOW);
752 } else {
753 vendorEvidence.addEvidence(source, key, value, Confidence.MEDIUM);
754 addMatchingValues(classInformation, value, vendorEvidence);
755 }
756 } else if (key.contains("name")) {
757 productEvidence.addEvidence(source, key, value, Confidence.MEDIUM);
758 vendorEvidence.addEvidence(source, key, value, Confidence.MEDIUM);
759 addMatchingValues(classInformation, value, vendorEvidence);
760 addMatchingValues(classInformation, value, productEvidence);
761 } else if (key.contains("license")) {
762 addLicense(dependency, value);
763 } else if (key.contains("description")) {
764 addDescription(dependency, value, "manifest", key);
765 } else {
766 productEvidence.addEvidence(source, key, value, Confidence.LOW);
767 vendorEvidence.addEvidence(source, key, value, Confidence.LOW);
768 addMatchingValues(classInformation, value, vendorEvidence);
769 addMatchingValues(classInformation, value, productEvidence);
770 if (value.matches(".*\\d.*")) {
771 final StringTokenizer tokenizer = new StringTokenizer(value, " ");
772 while (tokenizer.hasMoreElements()) {
773 final String s = tokenizer.nextToken();
774 if (s.matches("^[0-9.]+$")) {
775 versionEvidence.addEvidence(source, key, s, Confidence.LOW);
776 }
777 }
778 }
779 }
780 }
781 }
782 }
783 for (Map.Entry<String, Attributes> item : manifest.getEntries().entrySet()) {
784 final String name = item.getKey();
785 source = "manifest: " + name;
786 atts = item.getValue();
787 for (Entry<Object, Object> entry : atts.entrySet()) {
788 final String key = entry.getKey().toString();
789 final String value = atts.getValue(key);
790 if (key.equalsIgnoreCase(Attributes.Name.IMPLEMENTATION_TITLE.toString())) {
791 foundSomething = true;
792 productEvidence.addEvidence(source, key, value, Confidence.MEDIUM);
793 addMatchingValues(classInformation, value, productEvidence);
794 } else if (key.equalsIgnoreCase(Attributes.Name.IMPLEMENTATION_VERSION.toString())) {
795 foundSomething = true;
796 versionEvidence.addEvidence(source, key, value, Confidence.MEDIUM);
797 } else if (key.equalsIgnoreCase(Attributes.Name.IMPLEMENTATION_VENDOR.toString())) {
798 foundSomething = true;
799 vendorEvidence.addEvidence(source, key, value, Confidence.MEDIUM);
800 addMatchingValues(classInformation, value, vendorEvidence);
801 } else if (key.equalsIgnoreCase(Attributes.Name.SPECIFICATION_TITLE.toString())) {
802 foundSomething = true;
803 productEvidence.addEvidence(source, key, value, Confidence.MEDIUM);
804 addMatchingValues(classInformation, value, productEvidence);
805 }
806 }
807 }
808 if (specificationVersion != null && !hasImplementationVersion) {
809 foundSomething = true;
810 versionEvidence.addEvidence(source, "specification-version", specificationVersion, Confidence.HIGH);
811 }
812 } finally {
813 if (jar != null) {
814 jar.close();
815 }
816 }
817 return foundSomething;
818 }
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834 public static String addDescription(Dependency dependency, String description, String source, String key) {
835 if (dependency.getDescription() == null) {
836 dependency.setDescription(description);
837 }
838 String desc;
839 if (HTML_DETECTION_PATTERN.matcher(description).find()) {
840 desc = Jsoup.parse(description).text();
841 } else {
842 desc = description;
843 }
844 dependency.setDescription(desc);
845 if (desc.length() > 100) {
846 desc = desc.replaceAll("\\s\\s+", " ");
847 final int posSuchAs = desc.toLowerCase().indexOf("such as ", 100);
848 final int posLike = desc.toLowerCase().indexOf("like ", 100);
849 final int posWillUse = desc.toLowerCase().indexOf("will use ", 100);
850 final int posUses = desc.toLowerCase().indexOf(" uses ", 100);
851 int pos = -1;
852 pos = Math.max(pos, posSuchAs);
853 if (pos >= 0 && posLike >= 0) {
854 pos = Math.min(pos, posLike);
855 } else {
856 pos = Math.max(pos, posLike);
857 }
858 if (pos >= 0 && posWillUse >= 0) {
859 pos = Math.min(pos, posWillUse);
860 } else {
861 pos = Math.max(pos, posWillUse);
862 }
863 if (pos >= 0 && posUses >= 0) {
864 pos = Math.min(pos, posUses);
865 } else {
866 pos = Math.max(pos, posUses);
867 }
868
869 if (pos > 0) {
870 desc = desc.substring(0, pos) + "...";
871 }
872 dependency.getProductEvidence().addEvidence(source, key, desc, Confidence.LOW);
873 dependency.getVendorEvidence().addEvidence(source, key, desc, Confidence.LOW);
874 } else {
875 dependency.getProductEvidence().addEvidence(source, key, desc, Confidence.MEDIUM);
876 dependency.getVendorEvidence().addEvidence(source, key, desc, Confidence.MEDIUM);
877 }
878 return desc;
879 }
880
881
882
883
884
885
886
887 private void addLicense(Dependency d, String license) {
888 if (d.getLicense() == null) {
889 d.setLicense(license);
890 } else if (!d.getLicense().contains(license)) {
891 d.setLicense(d.getLicense() + NEWLINE + license);
892 }
893 }
894
895
896
897
898 private File tempFileLocation = null;
899
900
901
902
903
904
905
906 @Override
907 public void initializeFileTypeAnalyzer() throws InitializationException {
908 try {
909 final File baseDir = Settings.getTempDirectory();
910 tempFileLocation = File.createTempFile("check", "tmp", baseDir);
911 if (!tempFileLocation.delete()) {
912 final String msg = String.format("Unable to delete temporary file '%s'.", tempFileLocation.getAbsolutePath());
913 setEnabled(false);
914 throw new InitializationException(msg);
915 }
916 if (!tempFileLocation.mkdirs()) {
917 final String msg = String.format("Unable to create directory '%s'.", tempFileLocation.getAbsolutePath());
918 setEnabled(false);
919 throw new InitializationException(msg);
920 }
921 } catch (IOException ex) {
922 setEnabled(false);
923 throw new InitializationException("Unable to create a temporary file", ex);
924 }
925 }
926
927
928
929
930 @Override
931 public void close() {
932 if (tempFileLocation != null && tempFileLocation.exists()) {
933 LOGGER.debug("Attempting to delete temporary files");
934 final boolean success = FileUtils.delete(tempFileLocation);
935 if (!success) {
936 LOGGER.warn("Failed to delete some temporary files, see the log for more details");
937 }
938 }
939 }
940
941
942
943
944
945
946
947
948
949
950 private boolean isImportPackage(String key, String value) {
951 final Pattern packageRx = Pattern.compile("^([a-zA-Z0-9_#\\$\\*\\.]+\\s*[,;]\\s*)+([a-zA-Z0-9_#\\$\\*\\.]+\\s*)?$");
952 final boolean matches = packageRx.matcher(value).matches();
953 return matches && (key.contains("import") || key.contains("include") || value.length() > 10);
954 }
955
956
957
958
959
960
961
962
963
964 private List<ClassNameInformation> collectClassNames(Dependency dependency) {
965 final List<ClassNameInformation> classNames = new ArrayList<ClassNameInformation>();
966 JarFile jar = null;
967 try {
968 jar = new JarFile(dependency.getActualFilePath());
969 final Enumeration<JarEntry> entries = jar.entries();
970 while (entries.hasMoreElements()) {
971 final JarEntry entry = entries.nextElement();
972 final String name = entry.getName().toLowerCase();
973
974 if (name.endsWith(".class") && !name.matches("^javax?\\..*$")) {
975 final ClassNameInformation className = new ClassNameInformation(name.substring(0, name.length() - 6));
976 classNames.add(className);
977 }
978 }
979 } catch (IOException ex) {
980 LOGGER.warn("Unable to open jar file '{}'.", dependency.getFileName());
981 LOGGER.debug("", ex);
982 } finally {
983 if (jar != null) {
984 try {
985 jar.close();
986 } catch (IOException ex) {
987 LOGGER.trace("", ex);
988 }
989 }
990 }
991 return classNames;
992 }
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006 private void analyzeFullyQualifiedClassNames(List<ClassNameInformation> classNames,
1007 Map<String, Integer> vendor, Map<String, Integer> product) {
1008 for (ClassNameInformation entry : classNames) {
1009 final List<String> list = entry.getPackageStructure();
1010 addEntry(vendor, list.get(0));
1011
1012 if (list.size() == 2) {
1013 addEntry(product, list.get(1));
1014 }
1015 if (list.size() == 3) {
1016 addEntry(vendor, list.get(1));
1017 addEntry(product, list.get(1));
1018 addEntry(product, list.get(2));
1019 }
1020 if (list.size() >= 4) {
1021 addEntry(vendor, list.get(1));
1022 addEntry(vendor, list.get(2));
1023 addEntry(product, list.get(1));
1024 addEntry(product, list.get(2));
1025 addEntry(product, list.get(3));
1026 }
1027 }
1028 }
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038 private void addEntry(Map<String, Integer> collection, String key) {
1039 if (collection.containsKey(key)) {
1040 collection.put(key, collection.get(key) + 1);
1041 } else {
1042 collection.put(key, 1);
1043 }
1044 }
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056 private static void addMatchingValues(List<ClassNameInformation> classes, String value, EvidenceCollection evidence) {
1057 if (value == null || value.isEmpty() || classes == null || classes.isEmpty()) {
1058 return;
1059 }
1060 final String text = value.toLowerCase();
1061 for (ClassNameInformation cni : classes) {
1062 for (String key : cni.getPackageStructure()) {
1063 final Pattern p = Pattern.compile("\b" + key + "\b");
1064 if (p.matcher(text).find()) {
1065
1066 evidence.addEvidence("jar", "package name", key, Confidence.HIGHEST);
1067 }
1068 }
1069 }
1070 }
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080 private boolean isPackage(String key, String value) {
1081
1082 return !key.matches(".*(version|title|vendor|name|license|description).*")
1083 && value.matches("^([a-zA-Z_][a-zA-Z0-9_\\$]*(\\.[a-zA-Z_][a-zA-Z0-9_\\$]*)*)?$");
1084
1085 }
1086
1087
1088
1089
1090
1091
1092
1093
1094 public static void extractLicense(Model pom, Dependency dependency) {
1095
1096 if (pom.getLicenses() != null) {
1097 String license = null;
1098 for (License lic : pom.getLicenses()) {
1099 String tmp = null;
1100 if (lic.getName() != null) {
1101 tmp = lic.getName();
1102 }
1103 if (lic.getUrl() != null) {
1104 if (tmp == null) {
1105 tmp = lic.getUrl();
1106 } else {
1107 tmp += ": " + lic.getUrl();
1108 }
1109 }
1110 if (tmp == null) {
1111 continue;
1112 }
1113 if (HTML_DETECTION_PATTERN.matcher(tmp).find()) {
1114 tmp = Jsoup.parse(tmp).text();
1115 }
1116 if (license == null) {
1117 license = tmp;
1118 } else {
1119 license += "\n" + tmp;
1120 }
1121 }
1122 if (license != null) {
1123 dependency.setLicense(license);
1124
1125 }
1126 }
1127 }
1128
1129
1130
1131
1132 protected static class ClassNameInformation {
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156 ClassNameInformation(String className) {
1157 name = className;
1158 if (name.contains("/")) {
1159 final String[] tmp = className.toLowerCase().split("/");
1160 int start = 0;
1161 int end = 3;
1162 if ("com".equals(tmp[0]) || "org".equals(tmp[0])) {
1163 start = 1;
1164 end = 4;
1165 }
1166 if (tmp.length <= end) {
1167 end = tmp.length - 1;
1168 }
1169 for (int i = start; i <= end; i++) {
1170 packageStructure.add(tmp[i]);
1171 }
1172 } else {
1173 packageStructure.add(name);
1174 }
1175 }
1176
1177
1178
1179 private String name;
1180
1181
1182
1183
1184
1185
1186 public String getName() {
1187 return name;
1188 }
1189
1190
1191
1192
1193
1194
1195 public void setName(String name) {
1196 this.name = name;
1197 }
1198
1199
1200
1201
1202 private final ArrayList<String> packageStructure = new ArrayList<String>();
1203
1204
1205
1206
1207
1208
1209 public ArrayList<String> getPackageStructure() {
1210 return packageStructure;
1211 }
1212 }
1213
1214
1215
1216
1217
1218
1219
1220 private File getNextTempDirectory() throws AnalysisException {
1221 dirCount += 1;
1222 final File directory = new File(tempFileLocation, String.valueOf(dirCount));
1223
1224 if (directory.exists()) {
1225 return getNextTempDirectory();
1226 }
1227 if (!directory.mkdirs()) {
1228 final String msg = String.format("Unable to create temp directory '%s'.", directory.getAbsolutePath());
1229 throw new AnalysisException(msg);
1230 }
1231 return directory;
1232 }
1233 }