1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.owasp.dependencycheck.analyzer;
19
20 import java.io.IOException;
21 import java.io.UnsupportedEncodingException;
22 import java.net.URLEncoder;
23 import java.util.ArrayList;
24 import java.util.Collections;
25 import java.util.List;
26 import java.util.Set;
27 import java.util.StringTokenizer;
28 import java.util.concurrent.TimeUnit;
29 import org.apache.commons.lang3.builder.CompareToBuilder;
30 import org.apache.lucene.document.Document;
31 import org.apache.lucene.index.CorruptIndexException;
32 import org.apache.lucene.queryparser.classic.ParseException;
33 import org.apache.lucene.search.ScoreDoc;
34 import org.apache.lucene.search.TopDocs;
35 import org.owasp.dependencycheck.Engine;
36 import org.owasp.dependencycheck.analyzer.exception.AnalysisException;
37 import org.owasp.dependencycheck.data.cpe.CpeMemoryIndex;
38 import org.owasp.dependencycheck.data.cpe.Fields;
39 import org.owasp.dependencycheck.data.cpe.IndexEntry;
40 import org.owasp.dependencycheck.data.cpe.IndexException;
41 import org.owasp.dependencycheck.data.lucene.LuceneUtils;
42 import org.owasp.dependencycheck.data.nvdcve.CveDB;
43 import org.owasp.dependencycheck.data.nvdcve.DatabaseException;
44 import org.owasp.dependencycheck.dependency.Confidence;
45 import org.owasp.dependencycheck.dependency.Dependency;
46 import org.owasp.dependencycheck.dependency.Evidence;
47 import org.owasp.dependencycheck.dependency.EvidenceCollection;
48 import org.owasp.dependencycheck.dependency.Identifier;
49 import org.owasp.dependencycheck.dependency.VulnerableSoftware;
50 import org.owasp.dependencycheck.exception.InitializationException;
51 import org.owasp.dependencycheck.utils.DependencyVersion;
52 import org.owasp.dependencycheck.utils.DependencyVersionUtil;
53 import org.slf4j.Logger;
54 import org.slf4j.LoggerFactory;
55
56
57
58
59
60
61
62
63 public class CPEAnalyzer extends AbstractAnalyzer {
64
65
66
67
68 private static final Logger LOGGER = LoggerFactory.getLogger(CPEAnalyzer.class);
69
70
71
72 static final int MAX_QUERY_RESULTS = 25;
73
74
75
76 static final String WEIGHTING_BOOST = "^5";
77
78
79
80
81 static final String CLEANSE_CHARACTER_RX = "[^A-Za-z0-9 ._-]";
82
83
84
85
86 static final String CLEANSE_NONALPHA_RX = "[^A-Za-z]*";
87
88
89
90
91 static final int STRING_BUILDER_BUFFER = 20;
92
93
94
95 private CpeMemoryIndex cpe;
96
97
98
99 private CveDB cve;
100
101
102
103
104 public static final String NVD_SEARCH_URL = "https://web.nvd.nist.gov/view/vuln/search-results?adv_search=true&cves=on&cpe_version=%s";
105
106
107
108
109
110
111 @Override
112 public String getName() {
113 return "CPE Analyzer";
114 }
115
116
117
118
119
120
121 @Override
122 public AnalysisPhase getAnalysisPhase() {
123 return AnalysisPhase.IDENTIFIER_ANALYSIS;
124 }
125
126
127
128
129
130
131
132 @Override
133 public void initialize() throws InitializationException {
134 try {
135 this.open();
136 } catch (IOException ex) {
137 LOGGER.debug("Exception initializing the Lucene Index", ex);
138 throw new InitializationException("An exception occurred initializing the Lucene Index", ex);
139 } catch (DatabaseException ex) {
140 LOGGER.debug("Exception accessing the database", ex);
141 throw new InitializationException("An exception occurred accessing the database", ex);
142 }
143 }
144
145
146
147
148
149
150
151
152
153 public void open() throws IOException, DatabaseException {
154 if (!isOpen()) {
155 cve = new CveDB();
156 cve.open();
157 cpe = CpeMemoryIndex.getInstance();
158 try {
159 final long creationStart = System.currentTimeMillis();
160 cpe.open(cve);
161 final long creationSeconds = TimeUnit.MILLISECONDS.toSeconds(System.currentTimeMillis() - creationStart);
162 LOGGER.info("Created CPE Index ({} seconds)", creationSeconds);
163 } catch (IndexException ex) {
164 LOGGER.debug("IndexException", ex);
165 throw new DatabaseException(ex);
166 }
167 }
168 }
169
170
171
172
173 @Override
174 public void close() {
175 if (cpe != null) {
176 cpe.close();
177 cpe = null;
178 }
179 if (cve != null) {
180 cve.close();
181 cve = null;
182 }
183 }
184
185 public boolean isOpen() {
186 return cpe != null && cpe.isOpen();
187 }
188
189
190
191
192
193
194
195
196
197
198
199 protected void determineCPE(Dependency dependency) throws CorruptIndexException, IOException, ParseException {
200
201 String vendors = "";
202 String products = "";
203 for (Confidence confidence : Confidence.values()) {
204 if (dependency.getVendorEvidence().contains(confidence)) {
205 vendors = addEvidenceWithoutDuplicateTerms(vendors, dependency.getVendorEvidence(), confidence);
206 LOGGER.debug("vendor search: {}", vendors);
207 }
208 if (dependency.getProductEvidence().contains(confidence)) {
209 products = addEvidenceWithoutDuplicateTerms(products, dependency.getProductEvidence(), confidence);
210 LOGGER.debug("product search: {}", products);
211 }
212 if (!vendors.isEmpty() && !products.isEmpty()) {
213 final List<IndexEntry> entries = searchCPE(vendors, products, dependency.getVendorEvidence().getWeighting(),
214 dependency.getProductEvidence().getWeighting());
215 if (entries == null) {
216 continue;
217 }
218 boolean identifierAdded = false;
219 for (IndexEntry e : entries) {
220 LOGGER.debug("Verifying entry: {}", e);
221 if (verifyEntry(e, dependency)) {
222 final String vendor = e.getVendor();
223 final String product = e.getProduct();
224 LOGGER.debug("identified vendor/product: {}/{}", vendor, product);
225 identifierAdded |= determineIdentifiers(dependency, vendor, product, confidence);
226 }
227 }
228 if (identifierAdded) {
229 break;
230 }
231 }
232 }
233 }
234
235
236
237
238
239
240
241
242
243
244
245
246 private String addEvidenceWithoutDuplicateTerms(final String text, final EvidenceCollection ec, Confidence confidenceFilter) {
247 final String txt = (text == null) ? "" : text;
248 final StringBuilder sb = new StringBuilder(txt.length() + (20 * ec.size()));
249 sb.append(' ').append(txt).append(' ');
250 for (Evidence e : ec.iterator(confidenceFilter)) {
251 String value = e.getValue();
252
253
254
255 if (value.startsWith("http://")) {
256 value = value.substring(7).replaceAll("\\.", " ");
257 }
258 if (value.startsWith("https://")) {
259 value = value.substring(8).replaceAll("\\.", " ");
260 }
261 if (sb.indexOf(" " + value + " ") < 0) {
262 sb.append(value).append(' ');
263 }
264 }
265 return sb.toString().trim();
266 }
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285 protected List<IndexEntry> searchCPE(String vendor, String product,
286 Set<String> vendorWeightings, Set<String> productWeightings) {
287
288 final List<IndexEntry> ret = new ArrayList<IndexEntry>(MAX_QUERY_RESULTS);
289
290 final String searchString = buildSearch(vendor, product, vendorWeightings, productWeightings);
291 if (searchString == null) {
292 return ret;
293 }
294 try {
295 final TopDocs docs = cpe.search(searchString, MAX_QUERY_RESULTS);
296 for (ScoreDoc d : docs.scoreDocs) {
297 if (d.score >= 0.08) {
298 final Document doc = cpe.getDocument(d.doc);
299 final IndexEntry entry = new IndexEntry();
300 entry.setVendor(doc.get(Fields.VENDOR));
301 entry.setProduct(doc.get(Fields.PRODUCT));
302 entry.setSearchScore(d.score);
303 if (!ret.contains(entry)) {
304 ret.add(entry);
305 }
306 }
307 }
308 return ret;
309 } catch (ParseException ex) {
310 LOGGER.warn("An error occurred querying the CPE data. See the log for more details.");
311 LOGGER.info("Unable to parse: {}", searchString, ex);
312 } catch (IOException ex) {
313 LOGGER.warn("An error occurred reading CPE data. See the log for more details.");
314 LOGGER.info("IO Error with search string: {}", searchString, ex);
315 }
316 return null;
317 }
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337 protected String buildSearch(String vendor, String product,
338 Set<String> vendorWeighting, Set<String> productWeightings) {
339 final String v = vendor;
340 final String p = product;
341 final StringBuilder sb = new StringBuilder(v.length() + p.length()
342 + Fields.PRODUCT.length() + Fields.VENDOR.length() + STRING_BUILDER_BUFFER);
343
344 if (!appendWeightedSearch(sb, Fields.PRODUCT, p, productWeightings)) {
345 return null;
346 }
347 sb.append(" AND ");
348 if (!appendWeightedSearch(sb, Fields.VENDOR, v, vendorWeighting)) {
349 return null;
350 }
351 return sb.toString();
352 }
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368 private boolean appendWeightedSearch(StringBuilder sb, String field, String searchText, Set<String> weightedText) {
369 sb.append(' ').append(field).append(":( ");
370
371 final String cleanText = cleanseText(searchText);
372
373 if (cleanText.isEmpty()) {
374 return false;
375 }
376
377 if (weightedText == null || weightedText.isEmpty()) {
378 LuceneUtils.appendEscapedLuceneQuery(sb, cleanText);
379 } else {
380 final StringTokenizer tokens = new StringTokenizer(cleanText);
381 while (tokens.hasMoreElements()) {
382 final String word = tokens.nextToken();
383 StringBuilder temp = null;
384 for (String weighted : weightedText) {
385 final String weightedStr = cleanseText(weighted);
386 if (equalsIgnoreCaseAndNonAlpha(word, weightedStr)) {
387 temp = new StringBuilder(word.length() + 2);
388 LuceneUtils.appendEscapedLuceneQuery(temp, word);
389 temp.append(WEIGHTING_BOOST);
390 if (!word.equalsIgnoreCase(weightedStr)) {
391 temp.append(' ');
392 LuceneUtils.appendEscapedLuceneQuery(temp, weightedStr);
393 temp.append(WEIGHTING_BOOST);
394 }
395 break;
396 }
397 }
398 sb.append(' ');
399 if (temp == null) {
400 LuceneUtils.appendEscapedLuceneQuery(sb, word);
401 } else {
402 sb.append(temp);
403 }
404 }
405 }
406 sb.append(" ) ");
407 return true;
408 }
409
410
411
412
413
414
415
416
417 private String cleanseText(String text) {
418 return text.replaceAll(CLEANSE_CHARACTER_RX, " ");
419 }
420
421
422
423
424
425
426
427
428
429 private boolean equalsIgnoreCaseAndNonAlpha(String l, String r) {
430 if (l == null || r == null) {
431 return false;
432 }
433
434 final String left = l.replaceAll(CLEANSE_NONALPHA_RX, "");
435 final String right = r.replaceAll(CLEANSE_NONALPHA_RX, "");
436 return left.equalsIgnoreCase(right);
437 }
438
439
440
441
442
443
444
445
446
447
448 private boolean verifyEntry(final IndexEntry entry, final Dependency dependency) {
449 boolean isValid = false;
450
451
452
453 if (collectionContainsString(dependency.getProductEvidence(), entry.getProduct())
454 && collectionContainsString(dependency.getVendorEvidence(), entry.getVendor())) {
455
456 isValid = true;
457 }
458 return isValid;
459 }
460
461
462
463
464
465
466
467
468 private boolean collectionContainsString(EvidenceCollection ec, String text) {
469
470 if (text == null) {
471 return false;
472 }
473 final String[] words = text.split("[\\s_-]");
474 final List<String> list = new ArrayList<String>();
475 String tempWord = null;
476 for (String word : words) {
477
478
479
480
481 if (tempWord != null) {
482 list.add(tempWord + word);
483 tempWord = null;
484 } else if (word.length() <= 2) {
485 tempWord = word;
486 } else {
487 list.add(word);
488 }
489 }
490 if (tempWord != null) {
491 if (!list.isEmpty()) {
492 final String tmp = list.get(list.size() - 1) + tempWord;
493 list.add(tmp);
494 } else {
495 list.add(tempWord);
496 }
497 }
498 if (list.isEmpty()) {
499 return false;
500 }
501 boolean contains = true;
502 for (String word : list) {
503 contains &= ec.containsUsedString(word);
504 }
505 return contains;
506 }
507
508
509
510
511
512
513
514
515
516
517 @Override
518 public synchronized void analyze(Dependency dependency, Engine engine) throws AnalysisException {
519 try {
520 determineCPE(dependency);
521 } catch (CorruptIndexException ex) {
522 throw new AnalysisException("CPE Index is corrupt.", ex);
523 } catch (IOException ex) {
524 throw new AnalysisException("Failure opening the CPE Index.", ex);
525 } catch (ParseException ex) {
526 throw new AnalysisException("Unable to parse the generated Lucene query for this dependency.", ex);
527 }
528 }
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546 protected boolean determineIdentifiers(Dependency dependency, String vendor, String product,
547 Confidence currentConfidence) throws UnsupportedEncodingException {
548 final Set<VulnerableSoftware> cpes = cve.getCPEs(vendor, product);
549 DependencyVersion bestGuess = new DependencyVersion("-");
550 Confidence bestGuessConf = null;
551 boolean hasBroadMatch = false;
552 final List<IdentifierMatch> collected = new ArrayList<IdentifierMatch>();
553
554
555
556
557 for (Confidence conf : Confidence.values()) {
558 for (Evidence evidence : dependency.getVersionEvidence().iterator(conf)) {
559 final DependencyVersion evVer = DependencyVersionUtil.parseVersion(evidence.getValue());
560 if (evVer == null) {
561 continue;
562 }
563 for (VulnerableSoftware vs : cpes) {
564 DependencyVersion dbVer;
565 if (vs.getUpdate() != null && !vs.getUpdate().isEmpty()) {
566 dbVer = DependencyVersionUtil.parseVersion(vs.getVersion() + '.' + vs.getUpdate());
567 } else {
568 dbVer = DependencyVersionUtil.parseVersion(vs.getVersion());
569 }
570 if (dbVer == null) {
571 hasBroadMatch = true;
572 final String url = String.format(NVD_SEARCH_URL, URLEncoder.encode(vs.getName(), "UTF-8"));
573 final IdentifierMatch match = new IdentifierMatch("cpe", vs.getName(), url, IdentifierConfidence.BROAD_MATCH, conf);
574 collected.add(match);
575 } else if (evVer.equals(dbVer)) {
576 final String url = String.format(NVD_SEARCH_URL, URLEncoder.encode(vs.getName(), "UTF-8"));
577 final IdentifierMatch match = new IdentifierMatch("cpe", vs.getName(), url, IdentifierConfidence.EXACT_MATCH, conf);
578 collected.add(match);
579
580
581 } else if (evVer.getVersionParts().size() <= dbVer.getVersionParts().size()
582 && evVer.matchesAtLeastThreeLevels(dbVer)) {
583 if (bestGuessConf == null || bestGuessConf.compareTo(conf) > 0) {
584 if (bestGuess.getVersionParts().size() < dbVer.getVersionParts().size()) {
585 bestGuess = dbVer;
586 bestGuessConf = conf;
587 }
588 }
589 }
590 }
591 if (bestGuessConf == null || bestGuessConf.compareTo(conf) > 0) {
592 if (bestGuess.getVersionParts().size() < evVer.getVersionParts().size()) {
593 bestGuess = evVer;
594 bestGuessConf = conf;
595 }
596 }
597 }
598 }
599 final String cpeName = String.format("cpe:/a:%s:%s:%s", vendor, product, bestGuess.toString());
600 String url = null;
601 if (hasBroadMatch) {
602 final String cpeUrlName = String.format("cpe:/a:%s:%s", vendor, product);
603 url = String.format(NVD_SEARCH_URL, URLEncoder.encode(cpeUrlName, "UTF-8"));
604 }
605 if (bestGuessConf == null) {
606 bestGuessConf = Confidence.LOW;
607 }
608 final IdentifierMatch match = new IdentifierMatch("cpe", cpeName, url, IdentifierConfidence.BEST_GUESS, bestGuessConf);
609 collected.add(match);
610
611 Collections.sort(collected);
612 final IdentifierConfidence bestIdentifierQuality = collected.get(0).getConfidence();
613 final Confidence bestEvidenceQuality = collected.get(0).getEvidenceConfidence();
614 boolean identifierAdded = false;
615 for (IdentifierMatch m : collected) {
616 if (bestIdentifierQuality.equals(m.getConfidence())
617 && bestEvidenceQuality.equals(m.getEvidenceConfidence())) {
618 final Identifier i = m.getIdentifier();
619 if (bestIdentifierQuality == IdentifierConfidence.BEST_GUESS) {
620 i.setConfidence(Confidence.LOW);
621 } else {
622 i.setConfidence(bestEvidenceQuality);
623 }
624 dependency.addIdentifier(i);
625 identifierAdded = true;
626 }
627 }
628 return identifierAdded;
629 }
630
631
632
633
634 private enum IdentifierConfidence {
635
636
637
638
639 EXACT_MATCH,
640
641
642
643 BEST_GUESS,
644
645
646
647
648
649 BROAD_MATCH
650 }
651
652
653
654
655
656 private static class IdentifierMatch implements Comparable<IdentifierMatch> {
657
658
659
660
661
662
663
664
665
666
667
668
669 IdentifierMatch(String type, String value, String url, IdentifierConfidence identifierConfidence, Confidence evidenceConfidence) {
670 this.identifier = new Identifier(type, value, url);
671 this.confidence = identifierConfidence;
672 this.evidenceConfidence = evidenceConfidence;
673 }
674
675
676
677
678 private Confidence evidenceConfidence;
679
680
681
682
683
684
685 public Confidence getEvidenceConfidence() {
686 return evidenceConfidence;
687 }
688
689
690
691
692
693
694 public void setEvidenceConfidence(Confidence evidenceConfidence) {
695 this.evidenceConfidence = evidenceConfidence;
696 }
697
698
699
700 private IdentifierConfidence confidence;
701
702
703
704
705
706
707 public IdentifierConfidence getConfidence() {
708 return confidence;
709 }
710
711
712
713
714
715
716 public void setConfidence(IdentifierConfidence confidence) {
717 this.confidence = confidence;
718 }
719
720
721
722 private Identifier identifier;
723
724
725
726
727
728
729 public Identifier getIdentifier() {
730 return identifier;
731 }
732
733
734
735
736
737
738 public void setIdentifier(Identifier identifier) {
739 this.identifier = identifier;
740 }
741
742
743
744
745
746
747
748
749 @Override
750 public String toString() {
751 return "IdentifierMatch{" + "evidenceConfidence=" + evidenceConfidence
752 + ", confidence=" + confidence + ", identifier=" + identifier + '}';
753 }
754
755
756
757
758
759
760 @Override
761 public int hashCode() {
762 int hash = 5;
763 hash = 97 * hash + (this.evidenceConfidence != null ? this.evidenceConfidence.hashCode() : 0);
764 hash = 97 * hash + (this.confidence != null ? this.confidence.hashCode() : 0);
765 hash = 97 * hash + (this.identifier != null ? this.identifier.hashCode() : 0);
766 return hash;
767 }
768
769
770
771
772
773
774
775 @Override
776 public boolean equals(Object obj) {
777 if (obj == null) {
778 return false;
779 }
780 if (getClass() != obj.getClass()) {
781 return false;
782 }
783 final IdentifierMatch other = (IdentifierMatch) obj;
784 if (this.evidenceConfidence != other.evidenceConfidence) {
785 return false;
786 }
787 if (this.confidence != other.confidence) {
788 return false;
789 }
790 if (this.identifier != other.identifier && (this.identifier == null || !this.identifier.equals(other.identifier))) {
791 return false;
792 }
793 return true;
794 }
795
796
797
798
799
800
801
802
803
804 @Override
805 public int compareTo(IdentifierMatch o) {
806 return new CompareToBuilder()
807 .append(confidence, o.confidence)
808 .append(evidenceConfidence, o.evidenceConfidence)
809 .append(identifier, o.identifier)
810 .toComparison();
811
812
813
814
815
816
817
818
819
820
821 }
822 }
823 }