1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.owasp.dependencycheck.analyzer;
19
20 import java.io.IOException;
21 import java.io.UnsupportedEncodingException;
22 import java.net.URLEncoder;
23 import java.util.ArrayList;
24 import java.util.Collections;
25 import java.util.List;
26 import java.util.Set;
27 import java.util.StringTokenizer;
28 import org.apache.commons.lang3.builder.CompareToBuilder;
29 import org.apache.lucene.document.Document;
30 import org.apache.lucene.index.CorruptIndexException;
31 import org.apache.lucene.queryparser.classic.ParseException;
32 import org.apache.lucene.search.ScoreDoc;
33 import org.apache.lucene.search.TopDocs;
34 import org.owasp.dependencycheck.Engine;
35 import org.owasp.dependencycheck.analyzer.exception.AnalysisException;
36 import org.owasp.dependencycheck.data.cpe.CpeMemoryIndex;
37 import org.owasp.dependencycheck.data.cpe.Fields;
38 import org.owasp.dependencycheck.data.cpe.IndexEntry;
39 import org.owasp.dependencycheck.data.cpe.IndexException;
40 import org.owasp.dependencycheck.data.lucene.LuceneUtils;
41 import org.owasp.dependencycheck.data.nvdcve.CveDB;
42 import org.owasp.dependencycheck.data.nvdcve.DatabaseException;
43 import org.owasp.dependencycheck.dependency.Confidence;
44 import org.owasp.dependencycheck.dependency.Dependency;
45 import org.owasp.dependencycheck.dependency.Evidence;
46 import org.owasp.dependencycheck.dependency.EvidenceCollection;
47 import org.owasp.dependencycheck.dependency.Identifier;
48 import org.owasp.dependencycheck.dependency.VulnerableSoftware;
49 import org.owasp.dependencycheck.exception.InitializationException;
50 import org.owasp.dependencycheck.utils.DependencyVersion;
51 import org.owasp.dependencycheck.utils.DependencyVersionUtil;
52 import org.slf4j.Logger;
53 import org.slf4j.LoggerFactory;
54
55
56
57
58
59
60
61
62 public class CPEAnalyzer implements Analyzer {
63
64
65
66
67 private static final Logger LOGGER = LoggerFactory.getLogger(CPEAnalyzer.class);
68
69
70
71 static final int MAX_QUERY_RESULTS = 25;
72
73
74
75 static final String WEIGHTING_BOOST = "^5";
76
77
78
79
80 static final String CLEANSE_CHARACTER_RX = "[^A-Za-z0-9 ._-]";
81
82
83
84
85 static final String CLEANSE_NONALPHA_RX = "[^A-Za-z]*";
86
87
88
89
90 static final int STRING_BUILDER_BUFFER = 20;
91
92
93
94 private CpeMemoryIndex cpe;
95
96
97
98 private CveDB cve;
99
100
101
102
103 public static final String NVD_SEARCH_URL = "https://web.nvd.nist.gov/view/vuln/search-results?adv_search=true&cves=on&cpe_version=%s";
104
105
106
107
108
109
110 @Override
111 public String getName() {
112 return "CPE Analyzer";
113 }
114
115
116
117
118
119
120 @Override
121 public AnalysisPhase getAnalysisPhase() {
122 return AnalysisPhase.IDENTIFIER_ANALYSIS;
123 }
124
125
126
127
128
129
130
131 @Override
132 public void initialize() throws InitializationException {
133 try {
134 this.open();
135 } catch (IOException ex) {
136 LOGGER.debug("Exception initializing the Lucene Index", ex);
137 throw new InitializationException("An exception occurred initializing the Lucene Index", ex);
138 } catch (DatabaseException ex) {
139 LOGGER.debug("Exception accessing the database", ex);
140 throw new InitializationException("An exception occurred accessing the database", ex);
141 }
142 }
143
144
145
146
147
148
149
150
151
152 public void open() throws IOException, DatabaseException {
153 if (!isOpen()) {
154 cve = new CveDB();
155 cve.open();
156 cpe = CpeMemoryIndex.getInstance();
157 try {
158 LOGGER.info("Creating the CPE Index");
159 final long creationStart = System.currentTimeMillis();
160 cpe.open(cve);
161 LOGGER.info("CPE Index Created ({} ms)", System.currentTimeMillis() - creationStart);
162 } catch (IndexException ex) {
163 LOGGER.debug("IndexException", ex);
164 throw new DatabaseException(ex);
165 }
166 }
167 }
168
169
170
171
172 @Override
173 public void close() {
174 if (cpe != null) {
175 cpe.close();
176 cpe = null;
177 }
178 if (cve != null) {
179 cve.close();
180 cve = null;
181 }
182 }
183
184 public boolean isOpen() {
185 return cpe != null && cpe.isOpen();
186 }
187
188
189
190
191
192
193
194
195
196
197
198 protected void determineCPE(Dependency dependency) throws CorruptIndexException, IOException, ParseException {
199
200 String vendors = "";
201 String products = "";
202 for (Confidence confidence : Confidence.values()) {
203 if (dependency.getVendorEvidence().contains(confidence)) {
204 vendors = addEvidenceWithoutDuplicateTerms(vendors, dependency.getVendorEvidence(), confidence);
205 LOGGER.debug("vendor search: {}", vendors);
206 }
207 if (dependency.getProductEvidence().contains(confidence)) {
208 products = addEvidenceWithoutDuplicateTerms(products, dependency.getProductEvidence(), confidence);
209 LOGGER.debug("product search: {}", products);
210 }
211 if (!vendors.isEmpty() && !products.isEmpty()) {
212 final List<IndexEntry> entries = searchCPE(vendors, products, dependency.getVendorEvidence().getWeighting(),
213 dependency.getProductEvidence().getWeighting());
214 if (entries == null) {
215 continue;
216 }
217 boolean identifierAdded = false;
218 for (IndexEntry e : entries) {
219 LOGGER.debug("Verifying entry: {}", e);
220 if (verifyEntry(e, dependency)) {
221 final String vendor = e.getVendor();
222 final String product = e.getProduct();
223 LOGGER.debug("identified vendor/product: {}/{}", vendor, product);
224 identifierAdded |= determineIdentifiers(dependency, vendor, product, confidence);
225 }
226 }
227 if (identifierAdded) {
228 break;
229 }
230 }
231 }
232 }
233
234
235
236
237
238
239
240
241
242
243
244
245 private String addEvidenceWithoutDuplicateTerms(final String text, final EvidenceCollection ec, Confidence confidenceFilter) {
246 final String txt = (text == null) ? "" : text;
247 final StringBuilder sb = new StringBuilder(txt.length() + (20 * ec.size()));
248 sb.append(' ').append(txt).append(' ');
249 for (Evidence e : ec.iterator(confidenceFilter)) {
250 String value = e.getValue();
251
252
253
254 if (value.startsWith("http://")) {
255 value = value.substring(7).replaceAll("\\.", " ");
256 }
257 if (value.startsWith("https://")) {
258 value = value.substring(8).replaceAll("\\.", " ");
259 }
260 if (sb.indexOf(" " + value + " ") < 0) {
261 sb.append(value).append(' ');
262 }
263 }
264 return sb.toString().trim();
265 }
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284 protected List<IndexEntry> searchCPE(String vendor, String product,
285 Set<String> vendorWeightings, Set<String> productWeightings) {
286
287 final List<IndexEntry> ret = new ArrayList<IndexEntry>(MAX_QUERY_RESULTS);
288
289 final String searchString = buildSearch(vendor, product, vendorWeightings, productWeightings);
290 if (searchString == null) {
291 return ret;
292 }
293 try {
294 final TopDocs docs = cpe.search(searchString, MAX_QUERY_RESULTS);
295 for (ScoreDoc d : docs.scoreDocs) {
296 if (d.score >= 0.08) {
297 final Document doc = cpe.getDocument(d.doc);
298 final IndexEntry entry = new IndexEntry();
299 entry.setVendor(doc.get(Fields.VENDOR));
300 entry.setProduct(doc.get(Fields.PRODUCT));
301 entry.setSearchScore(d.score);
302 if (!ret.contains(entry)) {
303 ret.add(entry);
304 }
305 }
306 }
307 return ret;
308 } catch (ParseException ex) {
309 LOGGER.warn("An error occurred querying the CPE data. See the log for more details.");
310 LOGGER.info("Unable to parse: {}", searchString, ex);
311 } catch (IOException ex) {
312 LOGGER.warn("An error occurred reading CPE data. See the log for more details.");
313 LOGGER.info("IO Error with search string: {}", searchString, ex);
314 }
315 return null;
316 }
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336 protected String buildSearch(String vendor, String product,
337 Set<String> vendorWeighting, Set<String> productWeightings) {
338 final String v = vendor;
339 final String p = product;
340 final StringBuilder sb = new StringBuilder(v.length() + p.length()
341 + Fields.PRODUCT.length() + Fields.VENDOR.length() + STRING_BUILDER_BUFFER);
342
343 if (!appendWeightedSearch(sb, Fields.PRODUCT, p, productWeightings)) {
344 return null;
345 }
346 sb.append(" AND ");
347 if (!appendWeightedSearch(sb, Fields.VENDOR, v, vendorWeighting)) {
348 return null;
349 }
350 return sb.toString();
351 }
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367 private boolean appendWeightedSearch(StringBuilder sb, String field, String searchText, Set<String> weightedText) {
368 sb.append(' ').append(field).append(":( ");
369
370 final String cleanText = cleanseText(searchText);
371
372 if (cleanText.isEmpty()) {
373 return false;
374 }
375
376 if (weightedText == null || weightedText.isEmpty()) {
377 LuceneUtils.appendEscapedLuceneQuery(sb, cleanText);
378 } else {
379 final StringTokenizer tokens = new StringTokenizer(cleanText);
380 while (tokens.hasMoreElements()) {
381 final String word = tokens.nextToken();
382 StringBuilder temp = null;
383 for (String weighted : weightedText) {
384 final String weightedStr = cleanseText(weighted);
385 if (equalsIgnoreCaseAndNonAlpha(word, weightedStr)) {
386 temp = new StringBuilder(word.length() + 2);
387 LuceneUtils.appendEscapedLuceneQuery(temp, word);
388 temp.append(WEIGHTING_BOOST);
389 if (!word.equalsIgnoreCase(weightedStr)) {
390 temp.append(' ');
391 LuceneUtils.appendEscapedLuceneQuery(temp, weightedStr);
392 temp.append(WEIGHTING_BOOST);
393 }
394 break;
395 }
396 }
397 sb.append(' ');
398 if (temp == null) {
399 LuceneUtils.appendEscapedLuceneQuery(sb, word);
400 } else {
401 sb.append(temp);
402 }
403 }
404 }
405 sb.append(" ) ");
406 return true;
407 }
408
409
410
411
412
413
414
415
416 private String cleanseText(String text) {
417 return text.replaceAll(CLEANSE_CHARACTER_RX, " ");
418 }
419
420
421
422
423
424
425
426
427
428 private boolean equalsIgnoreCaseAndNonAlpha(String l, String r) {
429 if (l == null || r == null) {
430 return false;
431 }
432
433 final String left = l.replaceAll(CLEANSE_NONALPHA_RX, "");
434 final String right = r.replaceAll(CLEANSE_NONALPHA_RX, "");
435 return left.equalsIgnoreCase(right);
436 }
437
438
439
440
441
442
443
444
445
446
447 private boolean verifyEntry(final IndexEntry entry, final Dependency dependency) {
448 boolean isValid = false;
449
450
451
452 if (collectionContainsString(dependency.getProductEvidence(), entry.getProduct())
453 && collectionContainsString(dependency.getVendorEvidence(), entry.getVendor())) {
454
455 isValid = true;
456 }
457 return isValid;
458 }
459
460
461
462
463
464
465
466
467 private boolean collectionContainsString(EvidenceCollection ec, String text) {
468
469 if (text == null) {
470 return false;
471 }
472 final String[] words = text.split("[\\s_-]");
473 final List<String> list = new ArrayList<String>();
474 String tempWord = null;
475 for (String word : words) {
476
477
478
479
480 if (tempWord != null) {
481 list.add(tempWord + word);
482 tempWord = null;
483 } else if (word.length() <= 2) {
484 tempWord = word;
485 } else {
486 list.add(word);
487 }
488 }
489 if (tempWord != null) {
490 if (!list.isEmpty()) {
491 final String tmp = list.get(list.size() - 1) + tempWord;
492 list.add(tmp);
493 } else {
494 list.add(tempWord);
495 }
496 }
497 if (list.isEmpty()) {
498 return false;
499 }
500 boolean contains = true;
501 for (String word : list) {
502 contains &= ec.containsUsedString(word);
503 }
504 return contains;
505 }
506
507
508
509
510
511
512
513
514
515
516 @Override
517 public synchronized void analyze(Dependency dependency, Engine engine) throws AnalysisException {
518 try {
519 determineCPE(dependency);
520 } catch (CorruptIndexException ex) {
521 throw new AnalysisException("CPE Index is corrupt.", ex);
522 } catch (IOException ex) {
523 throw new AnalysisException("Failure opening the CPE Index.", ex);
524 } catch (ParseException ex) {
525 throw new AnalysisException("Unable to parse the generated Lucene query for this dependency.", ex);
526 }
527 }
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545 protected boolean determineIdentifiers(Dependency dependency, String vendor, String product,
546 Confidence currentConfidence) throws UnsupportedEncodingException {
547 final Set<VulnerableSoftware> cpes = cve.getCPEs(vendor, product);
548 DependencyVersion bestGuess = new DependencyVersion("-");
549 Confidence bestGuessConf = null;
550 boolean hasBroadMatch = false;
551 final List<IdentifierMatch> collected = new ArrayList<IdentifierMatch>();
552
553
554
555
556 for (Confidence conf : Confidence.values()) {
557 for (Evidence evidence : dependency.getVersionEvidence().iterator(conf)) {
558 final DependencyVersion evVer = DependencyVersionUtil.parseVersion(evidence.getValue());
559 if (evVer == null) {
560 continue;
561 }
562 for (VulnerableSoftware vs : cpes) {
563 DependencyVersion dbVer;
564 if (vs.getUpdate() != null && !vs.getUpdate().isEmpty()) {
565 dbVer = DependencyVersionUtil.parseVersion(vs.getVersion() + '.' + vs.getUpdate());
566 } else {
567 dbVer = DependencyVersionUtil.parseVersion(vs.getVersion());
568 }
569 if (dbVer == null) {
570 hasBroadMatch = true;
571 final String url = String.format(NVD_SEARCH_URL, URLEncoder.encode(vs.getName(), "UTF-8"));
572 final IdentifierMatch match = new IdentifierMatch("cpe", vs.getName(), url, IdentifierConfidence.BROAD_MATCH, conf);
573 collected.add(match);
574 } else if (evVer.equals(dbVer)) {
575 final String url = String.format(NVD_SEARCH_URL, URLEncoder.encode(vs.getName(), "UTF-8"));
576 final IdentifierMatch match = new IdentifierMatch("cpe", vs.getName(), url, IdentifierConfidence.EXACT_MATCH, conf);
577 collected.add(match);
578
579
580 } else if (evVer.getVersionParts().size() <= dbVer.getVersionParts().size()
581 && evVer.matchesAtLeastThreeLevels(dbVer)) {
582 if (bestGuessConf == null || bestGuessConf.compareTo(conf) > 0) {
583 if (bestGuess.getVersionParts().size() < dbVer.getVersionParts().size()) {
584 bestGuess = dbVer;
585 bestGuessConf = conf;
586 }
587 }
588 }
589 }
590 if (bestGuessConf == null || bestGuessConf.compareTo(conf) > 0) {
591 if (bestGuess.getVersionParts().size() < evVer.getVersionParts().size()) {
592 bestGuess = evVer;
593 bestGuessConf = conf;
594 }
595 }
596 }
597 }
598 final String cpeName = String.format("cpe:/a:%s:%s:%s", vendor, product, bestGuess.toString());
599 String url = null;
600 if (hasBroadMatch) {
601 final String cpeUrlName = String.format("cpe:/a:%s:%s", vendor, product);
602 url = String.format(NVD_SEARCH_URL, URLEncoder.encode(cpeUrlName, "UTF-8"));
603 }
604 if (bestGuessConf == null) {
605 bestGuessConf = Confidence.LOW;
606 }
607 final IdentifierMatch match = new IdentifierMatch("cpe", cpeName, url, IdentifierConfidence.BEST_GUESS, bestGuessConf);
608 collected.add(match);
609
610 Collections.sort(collected);
611 final IdentifierConfidence bestIdentifierQuality = collected.get(0).getConfidence();
612 final Confidence bestEvidenceQuality = collected.get(0).getEvidenceConfidence();
613 boolean identifierAdded = false;
614 for (IdentifierMatch m : collected) {
615 if (bestIdentifierQuality.equals(m.getConfidence())
616 && bestEvidenceQuality.equals(m.getEvidenceConfidence())) {
617 final Identifier i = m.getIdentifier();
618 if (bestIdentifierQuality == IdentifierConfidence.BEST_GUESS) {
619 i.setConfidence(Confidence.LOW);
620 } else {
621 i.setConfidence(bestEvidenceQuality);
622 }
623 dependency.addIdentifier(i);
624 identifierAdded = true;
625 }
626 }
627 return identifierAdded;
628 }
629
630
631
632
633 private enum IdentifierConfidence {
634
635
636
637
638 EXACT_MATCH,
639
640
641
642 BEST_GUESS,
643
644
645
646
647
648 BROAD_MATCH
649 }
650
651
652
653
654
655 private static class IdentifierMatch implements Comparable<IdentifierMatch> {
656
657
658
659
660
661
662
663
664
665
666
667
668 IdentifierMatch(String type, String value, String url, IdentifierConfidence identifierConfidence, Confidence evidenceConfidence) {
669 this.identifier = new Identifier(type, value, url);
670 this.confidence = identifierConfidence;
671 this.evidenceConfidence = evidenceConfidence;
672 }
673
674
675
676
677 private Confidence evidenceConfidence;
678
679
680
681
682
683
684 public Confidence getEvidenceConfidence() {
685 return evidenceConfidence;
686 }
687
688
689
690
691
692
693 public void setEvidenceConfidence(Confidence evidenceConfidence) {
694 this.evidenceConfidence = evidenceConfidence;
695 }
696
697
698
699 private IdentifierConfidence confidence;
700
701
702
703
704
705
706 public IdentifierConfidence getConfidence() {
707 return confidence;
708 }
709
710
711
712
713
714
715 public void setConfidence(IdentifierConfidence confidence) {
716 this.confidence = confidence;
717 }
718
719
720
721 private Identifier identifier;
722
723
724
725
726
727
728 public Identifier getIdentifier() {
729 return identifier;
730 }
731
732
733
734
735
736
737 public void setIdentifier(Identifier identifier) {
738 this.identifier = identifier;
739 }
740
741
742
743
744
745
746
747
748 @Override
749 public String toString() {
750 return "IdentifierMatch{" + "evidenceConfidence=" + evidenceConfidence
751 + ", confidence=" + confidence + ", identifier=" + identifier + '}';
752 }
753
754
755
756
757
758
759 @Override
760 public int hashCode() {
761 int hash = 5;
762 hash = 97 * hash + (this.evidenceConfidence != null ? this.evidenceConfidence.hashCode() : 0);
763 hash = 97 * hash + (this.confidence != null ? this.confidence.hashCode() : 0);
764 hash = 97 * hash + (this.identifier != null ? this.identifier.hashCode() : 0);
765 return hash;
766 }
767
768
769
770
771
772
773
774 @Override
775 public boolean equals(Object obj) {
776 if (obj == null) {
777 return false;
778 }
779 if (getClass() != obj.getClass()) {
780 return false;
781 }
782 final IdentifierMatch other = (IdentifierMatch) obj;
783 if (this.evidenceConfidence != other.evidenceConfidence) {
784 return false;
785 }
786 if (this.confidence != other.confidence) {
787 return false;
788 }
789 if (this.identifier != other.identifier && (this.identifier == null || !this.identifier.equals(other.identifier))) {
790 return false;
791 }
792 return true;
793 }
794
795
796
797
798
799
800
801
802
803 @Override
804 public int compareTo(IdentifierMatch o) {
805 return new CompareToBuilder()
806 .append(confidence, o.confidence)
807 .append(evidenceConfidence, o.evidenceConfidence)
808 .append(identifier, o.identifier)
809 .toComparison();
810
811
812
813
814
815
816
817
818
819
820 }
821 }
822 }