1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.owasp.dependencycheck.analyzer;
19
20 import java.io.IOException;
21 import java.io.UnsupportedEncodingException;
22 import java.net.URLEncoder;
23 import java.util.ArrayList;
24 import java.util.Collections;
25 import java.util.List;
26 import java.util.Set;
27 import java.util.StringTokenizer;
28 import org.apache.lucene.document.Document;
29 import org.apache.lucene.index.CorruptIndexException;
30 import org.apache.lucene.queryparser.classic.ParseException;
31 import org.apache.lucene.search.ScoreDoc;
32 import org.apache.lucene.search.TopDocs;
33 import org.owasp.dependencycheck.Engine;
34 import org.owasp.dependencycheck.analyzer.exception.AnalysisException;
35 import org.owasp.dependencycheck.data.cpe.CpeMemoryIndex;
36 import org.owasp.dependencycheck.data.cpe.Fields;
37 import org.owasp.dependencycheck.data.cpe.IndexEntry;
38 import org.owasp.dependencycheck.data.cpe.IndexException;
39 import org.owasp.dependencycheck.data.lucene.LuceneUtils;
40 import org.owasp.dependencycheck.data.nvdcve.CveDB;
41 import org.owasp.dependencycheck.data.nvdcve.DatabaseException;
42 import org.owasp.dependencycheck.dependency.Confidence;
43 import org.owasp.dependencycheck.dependency.Dependency;
44 import org.owasp.dependencycheck.dependency.Evidence;
45 import org.owasp.dependencycheck.dependency.EvidenceCollection;
46 import org.owasp.dependencycheck.dependency.Identifier;
47 import org.owasp.dependencycheck.dependency.VulnerableSoftware;
48 import org.owasp.dependencycheck.utils.DependencyVersion;
49 import org.owasp.dependencycheck.utils.DependencyVersionUtil;
50 import org.slf4j.Logger;
51 import org.slf4j.LoggerFactory;
52
53
54
55
56
57
58
59
60 public class CPEAnalyzer implements Analyzer {
61
62
63
64
65 private static final Logger LOGGER = LoggerFactory.getLogger(CPEAnalyzer.class);
66
67
68
69 static final int MAX_QUERY_RESULTS = 25;
70
71
72
73 static final String WEIGHTING_BOOST = "^5";
74
75
76
77
78 static final String CLEANSE_CHARACTER_RX = "[^A-Za-z0-9 ._-]";
79
80
81
82
83 static final String CLEANSE_NONALPHA_RX = "[^A-Za-z]*";
84
85
86
87
88 static final int STRING_BUILDER_BUFFER = 20;
89
90
91
92 private CpeMemoryIndex cpe;
93
94
95
96 private CveDB cve;
97
98
99
100
101 public static final String NVD_SEARCH_URL = "https://web.nvd.nist.gov/view/vuln/search-results?adv_search=true&cves=on&cpe_version=%s";
102
103
104
105
106
107
108 @Override
109 public String getName() {
110 return "CPE Analyzer";
111 }
112
113
114
115
116
117
118 @Override
119 public AnalysisPhase getAnalysisPhase() {
120 return AnalysisPhase.IDENTIFIER_ANALYSIS;
121 }
122
123
124
125
126
127
128 @Override
129 public void initialize() throws Exception {
130 this.open();
131 }
132
133
134
135
136
137
138
139
140
141 public void open() throws IOException, DatabaseException {
142 if (!isOpen()) {
143 cve = new CveDB();
144 cve.open();
145 cpe = CpeMemoryIndex.getInstance();
146 try {
147 LOGGER.info("Creating the CPE Index");
148 final long creationStart = System.currentTimeMillis();
149 cpe.open(cve);
150 LOGGER.info("CPE Index Created ({} ms)", System.currentTimeMillis() - creationStart);
151 } catch (IndexException ex) {
152 LOGGER.debug("IndexException", ex);
153 throw new DatabaseException(ex);
154 }
155 }
156 }
157
158
159
160
161 @Override
162 public void close() {
163 if (cpe != null) {
164 cpe.close();
165 cpe = null;
166 }
167 if (cve != null) {
168 cve.close();
169 cve = null;
170 }
171 }
172
173 public boolean isOpen() {
174 return cpe != null && cpe.isOpen();
175 }
176
177
178
179
180
181
182
183
184
185
186
187 protected void determineCPE(Dependency dependency) throws CorruptIndexException, IOException, ParseException {
188
189 String vendors = "";
190 String products = "";
191 for (Confidence confidence : Confidence.values()) {
192 if (dependency.getVendorEvidence().contains(confidence)) {
193 vendors = addEvidenceWithoutDuplicateTerms(vendors, dependency.getVendorEvidence(), confidence);
194 LOGGER.debug("vendor search: {}", vendors);
195 }
196 if (dependency.getProductEvidence().contains(confidence)) {
197 products = addEvidenceWithoutDuplicateTerms(products, dependency.getProductEvidence(), confidence);
198 LOGGER.debug("product search: {}", products);
199 }
200 if (!vendors.isEmpty() && !products.isEmpty()) {
201 final List<IndexEntry> entries = searchCPE(vendors, products, dependency.getVendorEvidence().getWeighting(),
202 dependency.getProductEvidence().getWeighting());
203 if (entries == null) {
204 continue;
205 }
206 boolean identifierAdded = false;
207 for (IndexEntry e : entries) {
208 LOGGER.debug("Verifying entry: {}", e);
209 if (verifyEntry(e, dependency)) {
210 final String vendor = e.getVendor();
211 final String product = e.getProduct();
212 LOGGER.debug("identified vendor/product: {}/{}", vendor, product);
213 identifierAdded |= determineIdentifiers(dependency, vendor, product, confidence);
214 }
215 }
216 if (identifierAdded) {
217 break;
218 }
219 }
220 }
221 }
222
223
224
225
226
227
228
229
230
231
232
233
234 private String addEvidenceWithoutDuplicateTerms(final String text, final EvidenceCollection ec, Confidence confidenceFilter) {
235 final String txt = (text == null) ? "" : text;
236 final StringBuilder sb = new StringBuilder(txt.length() + (20 * ec.size()));
237 sb.append(' ').append(txt).append(' ');
238 for (Evidence e : ec.iterator(confidenceFilter)) {
239 String value = e.getValue();
240
241
242
243 if (value.startsWith("http://")) {
244 value = value.substring(7).replaceAll("\\.", " ");
245 }
246 if (value.startsWith("https://")) {
247 value = value.substring(8).replaceAll("\\.", " ");
248 }
249 if (sb.indexOf(" " + value + " ") < 0) {
250 sb.append(value).append(' ');
251 }
252 }
253 return sb.toString().trim();
254 }
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273 protected List<IndexEntry> searchCPE(String vendor, String product,
274 Set<String> vendorWeightings, Set<String> productWeightings) {
275
276 final List<IndexEntry> ret = new ArrayList<IndexEntry>(MAX_QUERY_RESULTS);
277
278 final String searchString = buildSearch(vendor, product, vendorWeightings, productWeightings);
279 if (searchString == null) {
280 return ret;
281 }
282 try {
283 final TopDocs docs = cpe.search(searchString, MAX_QUERY_RESULTS);
284 for (ScoreDoc d : docs.scoreDocs) {
285 if (d.score >= 0.08) {
286 final Document doc = cpe.getDocument(d.doc);
287 final IndexEntry entry = new IndexEntry();
288 entry.setVendor(doc.get(Fields.VENDOR));
289 entry.setProduct(doc.get(Fields.PRODUCT));
290 entry.setSearchScore(d.score);
291 if (!ret.contains(entry)) {
292 ret.add(entry);
293 }
294 }
295 }
296 return ret;
297 } catch (ParseException ex) {
298 LOGGER.warn("An error occurred querying the CPE data. See the log for more details.");
299 LOGGER.info("Unable to parse: {}", searchString, ex);
300 } catch (IOException ex) {
301 LOGGER.warn("An error occurred reading CPE data. See the log for more details.");
302 LOGGER.info("IO Error with search string: {}", searchString, ex);
303 }
304 return null;
305 }
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325 protected String buildSearch(String vendor, String product,
326 Set<String> vendorWeighting, Set<String> productWeightings) {
327 final String v = vendor;
328 final String p = product;
329 final StringBuilder sb = new StringBuilder(v.length() + p.length()
330 + Fields.PRODUCT.length() + Fields.VENDOR.length() + STRING_BUILDER_BUFFER);
331
332 if (!appendWeightedSearch(sb, Fields.PRODUCT, p, productWeightings)) {
333 return null;
334 }
335 sb.append(" AND ");
336 if (!appendWeightedSearch(sb, Fields.VENDOR, v, vendorWeighting)) {
337 return null;
338 }
339 return sb.toString();
340 }
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356 private boolean appendWeightedSearch(StringBuilder sb, String field, String searchText, Set<String> weightedText) {
357 sb.append(' ').append(field).append(":( ");
358
359 final String cleanText = cleanseText(searchText);
360
361 if (cleanText.isEmpty()) {
362 return false;
363 }
364
365 if (weightedText == null || weightedText.isEmpty()) {
366 LuceneUtils.appendEscapedLuceneQuery(sb, cleanText);
367 } else {
368 final StringTokenizer tokens = new StringTokenizer(cleanText);
369 while (tokens.hasMoreElements()) {
370 final String word = tokens.nextToken();
371 StringBuilder temp = null;
372 for (String weighted : weightedText) {
373 final String weightedStr = cleanseText(weighted);
374 if (equalsIgnoreCaseAndNonAlpha(word, weightedStr)) {
375 temp = new StringBuilder(word.length() + 2);
376 LuceneUtils.appendEscapedLuceneQuery(temp, word);
377 temp.append(WEIGHTING_BOOST);
378 if (!word.equalsIgnoreCase(weightedStr)) {
379 temp.append(' ');
380 LuceneUtils.appendEscapedLuceneQuery(temp, weightedStr);
381 temp.append(WEIGHTING_BOOST);
382 }
383 break;
384 }
385 }
386 sb.append(' ');
387 if (temp == null) {
388 LuceneUtils.appendEscapedLuceneQuery(sb, word);
389 } else {
390 sb.append(temp);
391 }
392 }
393 }
394 sb.append(" ) ");
395 return true;
396 }
397
398
399
400
401
402
403
404
405 private String cleanseText(String text) {
406 return text.replaceAll(CLEANSE_CHARACTER_RX, " ");
407 }
408
409
410
411
412
413
414
415
416
417 private boolean equalsIgnoreCaseAndNonAlpha(String l, String r) {
418 if (l == null || r == null) {
419 return false;
420 }
421
422 final String left = l.replaceAll(CLEANSE_NONALPHA_RX, "");
423 final String right = r.replaceAll(CLEANSE_NONALPHA_RX, "");
424 return left.equalsIgnoreCase(right);
425 }
426
427
428
429
430
431
432
433
434
435
436 private boolean verifyEntry(final IndexEntry entry, final Dependency dependency) {
437 boolean isValid = false;
438
439
440
441 if (collectionContainsString(dependency.getProductEvidence(), entry.getProduct())
442 && collectionContainsString(dependency.getVendorEvidence(), entry.getVendor())) {
443
444 isValid = true;
445 }
446 return isValid;
447 }
448
449
450
451
452
453
454
455
456 private boolean collectionContainsString(EvidenceCollection ec, String text) {
457
458 if (text == null) {
459 return false;
460 }
461 final String[] words = text.split("[\\s_-]");
462 final List<String> list = new ArrayList<String>();
463 String tempWord = null;
464 for (String word : words) {
465
466
467
468
469 if (tempWord != null) {
470 list.add(tempWord + word);
471 tempWord = null;
472 } else if (word.length() <= 2) {
473 tempWord = word;
474 } else {
475 list.add(word);
476 }
477 }
478 if (tempWord != null) {
479 if (!list.isEmpty()) {
480 final String tmp = list.get(list.size() - 1) + tempWord;
481 list.add(tmp);
482 } else {
483 list.add(tempWord);
484 }
485 }
486 if (list.isEmpty()) {
487 return false;
488 }
489 boolean contains = true;
490 for (String word : list) {
491 contains &= ec.containsUsedString(word);
492 }
493 return contains;
494 }
495
496
497
498
499
500
501
502
503
504
505 @Override
506 public synchronized void analyze(Dependency dependency, Engine engine) throws AnalysisException {
507 try {
508 determineCPE(dependency);
509 } catch (CorruptIndexException ex) {
510 throw new AnalysisException("CPE Index is corrupt.", ex);
511 } catch (IOException ex) {
512 throw new AnalysisException("Failure opening the CPE Index.", ex);
513 } catch (ParseException ex) {
514 throw new AnalysisException("Unable to parse the generated Lucene query for this dependency.", ex);
515 }
516 }
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534 protected boolean determineIdentifiers(Dependency dependency, String vendor, String product,
535 Confidence currentConfidence) throws UnsupportedEncodingException {
536 final Set<VulnerableSoftware> cpes = cve.getCPEs(vendor, product);
537 DependencyVersion bestGuess = new DependencyVersion("-");
538 Confidence bestGuessConf = null;
539 boolean hasBroadMatch = false;
540 final List<IdentifierMatch> collected = new ArrayList<IdentifierMatch>();
541
542
543
544
545 for (Confidence conf : Confidence.values()) {
546 for (Evidence evidence : dependency.getVersionEvidence().iterator(conf)) {
547 final DependencyVersion evVer = DependencyVersionUtil.parseVersion(evidence.getValue());
548 if (evVer == null) {
549 continue;
550 }
551 for (VulnerableSoftware vs : cpes) {
552 DependencyVersion dbVer;
553 if (vs.getUpdate() != null && !vs.getUpdate().isEmpty()) {
554 dbVer = DependencyVersionUtil.parseVersion(vs.getVersion() + '.' + vs.getUpdate());
555 } else {
556 dbVer = DependencyVersionUtil.parseVersion(vs.getVersion());
557 }
558 if (dbVer == null) {
559 hasBroadMatch = true;
560 final String url = String.format(NVD_SEARCH_URL, URLEncoder.encode(vs.getName(), "UTF-8"));
561 final IdentifierMatch match = new IdentifierMatch("cpe", vs.getName(), url, IdentifierConfidence.BROAD_MATCH, conf);
562 collected.add(match);
563 } else if (evVer.equals(dbVer)) {
564 final String url = String.format(NVD_SEARCH_URL, URLEncoder.encode(vs.getName(), "UTF-8"));
565 final IdentifierMatch match = new IdentifierMatch("cpe", vs.getName(), url, IdentifierConfidence.EXACT_MATCH, conf);
566 collected.add(match);
567 } else
568 if (evVer.getVersionParts().size() <= dbVer.getVersionParts().size()
569 && evVer.matchesAtLeastThreeLevels(dbVer)) {
570 if (bestGuessConf == null || bestGuessConf.compareTo(conf) > 0) {
571 if (bestGuess.getVersionParts().size() < dbVer.getVersionParts().size()) {
572 bestGuess = dbVer;
573 bestGuessConf = conf;
574 }
575 }
576 }
577 }
578 if (bestGuessConf == null || bestGuessConf.compareTo(conf) > 0) {
579 if (bestGuess.getVersionParts().size() < evVer.getVersionParts().size()) {
580 bestGuess = evVer;
581 bestGuessConf = conf;
582 }
583 }
584 }
585 }
586 final String cpeName = String.format("cpe:/a:%s:%s:%s", vendor, product, bestGuess.toString());
587 String url = null;
588 if (hasBroadMatch) {
589 final String cpeUrlName = String.format("cpe:/a:%s:%s", vendor, product);
590 url = String.format(NVD_SEARCH_URL, URLEncoder.encode(cpeUrlName, "UTF-8"));
591 }
592 if (bestGuessConf == null) {
593 bestGuessConf = Confidence.LOW;
594 }
595 final IdentifierMatch match = new IdentifierMatch("cpe", cpeName, url, IdentifierConfidence.BEST_GUESS, bestGuessConf);
596 collected.add(match);
597
598 Collections.sort(collected);
599 final IdentifierConfidence bestIdentifierQuality = collected.get(0).getConfidence();
600 final Confidence bestEvidenceQuality = collected.get(0).getEvidenceConfidence();
601 boolean identifierAdded = false;
602 for (IdentifierMatch m : collected) {
603 if (bestIdentifierQuality.equals(m.getConfidence())
604 && bestEvidenceQuality.equals(m.getEvidenceConfidence())) {
605 final Identifier i = m.getIdentifier();
606 if (bestIdentifierQuality == IdentifierConfidence.BEST_GUESS) {
607 i.setConfidence(Confidence.LOW);
608 } else {
609 i.setConfidence(bestEvidenceQuality);
610 }
611 dependency.addIdentifier(i);
612 identifierAdded = true;
613 }
614 }
615 return identifierAdded;
616 }
617
618
619
620
621 private enum IdentifierConfidence {
622
623
624
625
626 EXACT_MATCH,
627
628
629
630 BEST_GUESS,
631
632
633
634
635
636 BROAD_MATCH
637 }
638
639
640
641
642
643 private static class IdentifierMatch implements Comparable<IdentifierMatch> {
644
645
646
647
648
649
650
651
652
653
654
655
656 IdentifierMatch(String type, String value, String url, IdentifierConfidence identifierConfidence, Confidence evidenceConfidence) {
657 this.identifier = new Identifier(type, value, url);
658 this.confidence = identifierConfidence;
659 this.evidenceConfidence = evidenceConfidence;
660 }
661
662
663
664
665 private Confidence evidenceConfidence;
666
667
668
669
670
671
672 public Confidence getEvidenceConfidence() {
673 return evidenceConfidence;
674 }
675
676
677
678
679
680
681 public void setEvidenceConfidence(Confidence evidenceConfidence) {
682 this.evidenceConfidence = evidenceConfidence;
683 }
684
685
686
687 private IdentifierConfidence confidence;
688
689
690
691
692
693
694 public IdentifierConfidence getConfidence() {
695 return confidence;
696 }
697
698
699
700
701
702
703 public void setConfidence(IdentifierConfidence confidence) {
704 this.confidence = confidence;
705 }
706
707
708
709 private Identifier identifier;
710
711
712
713
714
715
716 public Identifier getIdentifier() {
717 return identifier;
718 }
719
720
721
722
723
724
725 public void setIdentifier(Identifier identifier) {
726 this.identifier = identifier;
727 }
728
729
730
731
732
733
734
735
736 @Override
737 public String toString() {
738 return "IdentifierMatch{" + "evidenceConfidence=" + evidenceConfidence
739 + ", confidence=" + confidence + ", identifier=" + identifier + '}';
740 }
741
742
743
744
745
746
747 @Override
748 public int hashCode() {
749 int hash = 5;
750 hash = 97 * hash + (this.evidenceConfidence != null ? this.evidenceConfidence.hashCode() : 0);
751 hash = 97 * hash + (this.confidence != null ? this.confidence.hashCode() : 0);
752 hash = 97 * hash + (this.identifier != null ? this.identifier.hashCode() : 0);
753 return hash;
754 }
755
756
757
758
759
760
761
762 @Override
763 public boolean equals(Object obj) {
764 if (obj == null) {
765 return false;
766 }
767 if (getClass() != obj.getClass()) {
768 return false;
769 }
770 final IdentifierMatch other = (IdentifierMatch) obj;
771 if (this.evidenceConfidence != other.evidenceConfidence) {
772 return false;
773 }
774 if (this.confidence != other.confidence) {
775 return false;
776 }
777 if (this.identifier != other.identifier && (this.identifier == null || !this.identifier.equals(other.identifier))) {
778 return false;
779 }
780 return true;
781 }
782
783
784
785
786
787
788
789
790
791 @Override
792 public int compareTo(IdentifierMatch o) {
793 int conf = this.confidence.compareTo(o.confidence);
794 if (conf == 0) {
795 conf = this.evidenceConfidence.compareTo(o.evidenceConfidence);
796 if (conf == 0) {
797 conf = identifier.compareTo(o.identifier);
798 }
799 }
800 return conf;
801 }
802 }
803 }