1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.owasp.dependencycheck.analyzer;
19
20 import java.io.IOException;
21 import java.io.UnsupportedEncodingException;
22 import java.net.URLEncoder;
23 import java.util.ArrayList;
24 import java.util.Collections;
25 import java.util.List;
26 import java.util.Set;
27 import java.util.StringTokenizer;
28 import org.apache.lucene.document.Document;
29 import org.apache.lucene.index.CorruptIndexException;
30 import org.apache.lucene.queryparser.classic.ParseException;
31 import org.apache.lucene.search.ScoreDoc;
32 import org.apache.lucene.search.TopDocs;
33 import org.owasp.dependencycheck.Engine;
34 import org.owasp.dependencycheck.analyzer.exception.AnalysisException;
35 import org.owasp.dependencycheck.data.cpe.CpeMemoryIndex;
36 import org.owasp.dependencycheck.data.cpe.Fields;
37 import org.owasp.dependencycheck.data.cpe.IndexEntry;
38 import org.owasp.dependencycheck.data.cpe.IndexException;
39 import org.owasp.dependencycheck.data.lucene.LuceneUtils;
40 import org.owasp.dependencycheck.data.nvdcve.CveDB;
41 import org.owasp.dependencycheck.data.nvdcve.DatabaseException;
42 import org.owasp.dependencycheck.dependency.Confidence;
43 import org.owasp.dependencycheck.dependency.Dependency;
44 import org.owasp.dependencycheck.dependency.Evidence;
45 import org.owasp.dependencycheck.dependency.EvidenceCollection;
46 import org.owasp.dependencycheck.dependency.Identifier;
47 import org.owasp.dependencycheck.dependency.VulnerableSoftware;
48 import org.owasp.dependencycheck.exception.InitializationException;
49 import org.owasp.dependencycheck.utils.DependencyVersion;
50 import org.owasp.dependencycheck.utils.DependencyVersionUtil;
51 import org.slf4j.Logger;
52 import org.slf4j.LoggerFactory;
53
54
55
56
57
58
59
60
61 public class CPEAnalyzer implements Analyzer {
62
63
64
65
66 private static final Logger LOGGER = LoggerFactory.getLogger(CPEAnalyzer.class);
67
68
69
70 static final int MAX_QUERY_RESULTS = 25;
71
72
73
74 static final String WEIGHTING_BOOST = "^5";
75
76
77
78
79 static final String CLEANSE_CHARACTER_RX = "[^A-Za-z0-9 ._-]";
80
81
82
83
84 static final String CLEANSE_NONALPHA_RX = "[^A-Za-z]*";
85
86
87
88
89 static final int STRING_BUILDER_BUFFER = 20;
90
91
92
93 private CpeMemoryIndex cpe;
94
95
96
97 private CveDB cve;
98
99
100
101
102 public static final String NVD_SEARCH_URL = "https://web.nvd.nist.gov/view/vuln/search-results?adv_search=true&cves=on&cpe_version=%s";
103
104
105
106
107
108
109 @Override
110 public String getName() {
111 return "CPE Analyzer";
112 }
113
114
115
116
117
118
119 @Override
120 public AnalysisPhase getAnalysisPhase() {
121 return AnalysisPhase.IDENTIFIER_ANALYSIS;
122 }
123
124
125
126
127
128
129
130 @Override
131 public void initialize() throws InitializationException {
132 try {
133 this.open();
134 } catch (IOException ex) {
135 LOGGER.debug("Exception initializing the Lucene Index", ex);
136 throw new InitializationException("An exception occurred initializing the Lucene Index", ex);
137 } catch (DatabaseException ex) {
138 LOGGER.debug("Exception accessing the database", ex);
139 throw new InitializationException("An exception occurred accessing the database", ex);
140 }
141 }
142
143
144
145
146
147
148
149
150
151 public void open() throws IOException, DatabaseException {
152 if (!isOpen()) {
153 cve = new CveDB();
154 cve.open();
155 cpe = CpeMemoryIndex.getInstance();
156 try {
157 LOGGER.info("Creating the CPE Index");
158 final long creationStart = System.currentTimeMillis();
159 cpe.open(cve);
160 LOGGER.info("CPE Index Created ({} ms)", System.currentTimeMillis() - creationStart);
161 } catch (IndexException ex) {
162 LOGGER.debug("IndexException", ex);
163 throw new DatabaseException(ex);
164 }
165 }
166 }
167
168
169
170
171 @Override
172 public void close() {
173 if (cpe != null) {
174 cpe.close();
175 cpe = null;
176 }
177 if (cve != null) {
178 cve.close();
179 cve = null;
180 }
181 }
182
183 public boolean isOpen() {
184 return cpe != null && cpe.isOpen();
185 }
186
187
188
189
190
191
192
193
194
195
196
197 protected void determineCPE(Dependency dependency) throws CorruptIndexException, IOException, ParseException {
198
199 String vendors = "";
200 String products = "";
201 for (Confidence confidence : Confidence.values()) {
202 if (dependency.getVendorEvidence().contains(confidence)) {
203 vendors = addEvidenceWithoutDuplicateTerms(vendors, dependency.getVendorEvidence(), confidence);
204 LOGGER.debug("vendor search: {}", vendors);
205 }
206 if (dependency.getProductEvidence().contains(confidence)) {
207 products = addEvidenceWithoutDuplicateTerms(products, dependency.getProductEvidence(), confidence);
208 LOGGER.debug("product search: {}", products);
209 }
210 if (!vendors.isEmpty() && !products.isEmpty()) {
211 final List<IndexEntry> entries = searchCPE(vendors, products, dependency.getVendorEvidence().getWeighting(),
212 dependency.getProductEvidence().getWeighting());
213 if (entries == null) {
214 continue;
215 }
216 boolean identifierAdded = false;
217 for (IndexEntry e : entries) {
218 LOGGER.debug("Verifying entry: {}", e);
219 if (verifyEntry(e, dependency)) {
220 final String vendor = e.getVendor();
221 final String product = e.getProduct();
222 LOGGER.debug("identified vendor/product: {}/{}", vendor, product);
223 identifierAdded |= determineIdentifiers(dependency, vendor, product, confidence);
224 }
225 }
226 if (identifierAdded) {
227 break;
228 }
229 }
230 }
231 }
232
233
234
235
236
237
238
239
240
241
242
243
244 private String addEvidenceWithoutDuplicateTerms(final String text, final EvidenceCollection ec, Confidence confidenceFilter) {
245 final String txt = (text == null) ? "" : text;
246 final StringBuilder sb = new StringBuilder(txt.length() + (20 * ec.size()));
247 sb.append(' ').append(txt).append(' ');
248 for (Evidence e : ec.iterator(confidenceFilter)) {
249 String value = e.getValue();
250
251
252
253 if (value.startsWith("http://")) {
254 value = value.substring(7).replaceAll("\\.", " ");
255 }
256 if (value.startsWith("https://")) {
257 value = value.substring(8).replaceAll("\\.", " ");
258 }
259 if (sb.indexOf(" " + value + " ") < 0) {
260 sb.append(value).append(' ');
261 }
262 }
263 return sb.toString().trim();
264 }
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283 protected List<IndexEntry> searchCPE(String vendor, String product,
284 Set<String> vendorWeightings, Set<String> productWeightings) {
285
286 final List<IndexEntry> ret = new ArrayList<IndexEntry>(MAX_QUERY_RESULTS);
287
288 final String searchString = buildSearch(vendor, product, vendorWeightings, productWeightings);
289 if (searchString == null) {
290 return ret;
291 }
292 try {
293 final TopDocs docs = cpe.search(searchString, MAX_QUERY_RESULTS);
294 for (ScoreDoc d : docs.scoreDocs) {
295 if (d.score >= 0.08) {
296 final Document doc = cpe.getDocument(d.doc);
297 final IndexEntry entry = new IndexEntry();
298 entry.setVendor(doc.get(Fields.VENDOR));
299 entry.setProduct(doc.get(Fields.PRODUCT));
300 entry.setSearchScore(d.score);
301 if (!ret.contains(entry)) {
302 ret.add(entry);
303 }
304 }
305 }
306 return ret;
307 } catch (ParseException ex) {
308 LOGGER.warn("An error occurred querying the CPE data. See the log for more details.");
309 LOGGER.info("Unable to parse: {}", searchString, ex);
310 } catch (IOException ex) {
311 LOGGER.warn("An error occurred reading CPE data. See the log for more details.");
312 LOGGER.info("IO Error with search string: {}", searchString, ex);
313 }
314 return null;
315 }
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335 protected String buildSearch(String vendor, String product,
336 Set<String> vendorWeighting, Set<String> productWeightings) {
337 final String v = vendor;
338 final String p = product;
339 final StringBuilder sb = new StringBuilder(v.length() + p.length()
340 + Fields.PRODUCT.length() + Fields.VENDOR.length() + STRING_BUILDER_BUFFER);
341
342 if (!appendWeightedSearch(sb, Fields.PRODUCT, p, productWeightings)) {
343 return null;
344 }
345 sb.append(" AND ");
346 if (!appendWeightedSearch(sb, Fields.VENDOR, v, vendorWeighting)) {
347 return null;
348 }
349 return sb.toString();
350 }
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366 private boolean appendWeightedSearch(StringBuilder sb, String field, String searchText, Set<String> weightedText) {
367 sb.append(' ').append(field).append(":( ");
368
369 final String cleanText = cleanseText(searchText);
370
371 if (cleanText.isEmpty()) {
372 return false;
373 }
374
375 if (weightedText == null || weightedText.isEmpty()) {
376 LuceneUtils.appendEscapedLuceneQuery(sb, cleanText);
377 } else {
378 final StringTokenizer tokens = new StringTokenizer(cleanText);
379 while (tokens.hasMoreElements()) {
380 final String word = tokens.nextToken();
381 StringBuilder temp = null;
382 for (String weighted : weightedText) {
383 final String weightedStr = cleanseText(weighted);
384 if (equalsIgnoreCaseAndNonAlpha(word, weightedStr)) {
385 temp = new StringBuilder(word.length() + 2);
386 LuceneUtils.appendEscapedLuceneQuery(temp, word);
387 temp.append(WEIGHTING_BOOST);
388 if (!word.equalsIgnoreCase(weightedStr)) {
389 temp.append(' ');
390 LuceneUtils.appendEscapedLuceneQuery(temp, weightedStr);
391 temp.append(WEIGHTING_BOOST);
392 }
393 break;
394 }
395 }
396 sb.append(' ');
397 if (temp == null) {
398 LuceneUtils.appendEscapedLuceneQuery(sb, word);
399 } else {
400 sb.append(temp);
401 }
402 }
403 }
404 sb.append(" ) ");
405 return true;
406 }
407
408
409
410
411
412
413
414
415 private String cleanseText(String text) {
416 return text.replaceAll(CLEANSE_CHARACTER_RX, " ");
417 }
418
419
420
421
422
423
424
425
426
427 private boolean equalsIgnoreCaseAndNonAlpha(String l, String r) {
428 if (l == null || r == null) {
429 return false;
430 }
431
432 final String left = l.replaceAll(CLEANSE_NONALPHA_RX, "");
433 final String right = r.replaceAll(CLEANSE_NONALPHA_RX, "");
434 return left.equalsIgnoreCase(right);
435 }
436
437
438
439
440
441
442
443
444
445
446 private boolean verifyEntry(final IndexEntry entry, final Dependency dependency) {
447 boolean isValid = false;
448
449
450
451 if (collectionContainsString(dependency.getProductEvidence(), entry.getProduct())
452 && collectionContainsString(dependency.getVendorEvidence(), entry.getVendor())) {
453
454 isValid = true;
455 }
456 return isValid;
457 }
458
459
460
461
462
463
464
465
466 private boolean collectionContainsString(EvidenceCollection ec, String text) {
467
468 if (text == null) {
469 return false;
470 }
471 final String[] words = text.split("[\\s_-]");
472 final List<String> list = new ArrayList<String>();
473 String tempWord = null;
474 for (String word : words) {
475
476
477
478
479 if (tempWord != null) {
480 list.add(tempWord + word);
481 tempWord = null;
482 } else if (word.length() <= 2) {
483 tempWord = word;
484 } else {
485 list.add(word);
486 }
487 }
488 if (tempWord != null) {
489 if (!list.isEmpty()) {
490 final String tmp = list.get(list.size() - 1) + tempWord;
491 list.add(tmp);
492 } else {
493 list.add(tempWord);
494 }
495 }
496 if (list.isEmpty()) {
497 return false;
498 }
499 boolean contains = true;
500 for (String word : list) {
501 contains &= ec.containsUsedString(word);
502 }
503 return contains;
504 }
505
506
507
508
509
510
511
512
513
514
515 @Override
516 public synchronized void analyze(Dependency dependency, Engine engine) throws AnalysisException {
517 try {
518 determineCPE(dependency);
519 } catch (CorruptIndexException ex) {
520 throw new AnalysisException("CPE Index is corrupt.", ex);
521 } catch (IOException ex) {
522 throw new AnalysisException("Failure opening the CPE Index.", ex);
523 } catch (ParseException ex) {
524 throw new AnalysisException("Unable to parse the generated Lucene query for this dependency.", ex);
525 }
526 }
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544 protected boolean determineIdentifiers(Dependency dependency, String vendor, String product,
545 Confidence currentConfidence) throws UnsupportedEncodingException {
546 final Set<VulnerableSoftware> cpes = cve.getCPEs(vendor, product);
547 DependencyVersion bestGuess = new DependencyVersion("-");
548 Confidence bestGuessConf = null;
549 boolean hasBroadMatch = false;
550 final List<IdentifierMatch> collected = new ArrayList<IdentifierMatch>();
551
552
553
554
555 for (Confidence conf : Confidence.values()) {
556 for (Evidence evidence : dependency.getVersionEvidence().iterator(conf)) {
557 final DependencyVersion evVer = DependencyVersionUtil.parseVersion(evidence.getValue());
558 if (evVer == null) {
559 continue;
560 }
561 for (VulnerableSoftware vs : cpes) {
562 DependencyVersion dbVer;
563 if (vs.getUpdate() != null && !vs.getUpdate().isEmpty()) {
564 dbVer = DependencyVersionUtil.parseVersion(vs.getVersion() + '.' + vs.getUpdate());
565 } else {
566 dbVer = DependencyVersionUtil.parseVersion(vs.getVersion());
567 }
568 if (dbVer == null) {
569 hasBroadMatch = true;
570 final String url = String.format(NVD_SEARCH_URL, URLEncoder.encode(vs.getName(), "UTF-8"));
571 final IdentifierMatch match = new IdentifierMatch("cpe", vs.getName(), url, IdentifierConfidence.BROAD_MATCH, conf);
572 collected.add(match);
573 } else if (evVer.equals(dbVer)) {
574 final String url = String.format(NVD_SEARCH_URL, URLEncoder.encode(vs.getName(), "UTF-8"));
575 final IdentifierMatch match = new IdentifierMatch("cpe", vs.getName(), url, IdentifierConfidence.EXACT_MATCH, conf);
576 collected.add(match);
577
578
579 } else if (evVer.getVersionParts().size() <= dbVer.getVersionParts().size()
580 && evVer.matchesAtLeastThreeLevels(dbVer)) {
581 if (bestGuessConf == null || bestGuessConf.compareTo(conf) > 0) {
582 if (bestGuess.getVersionParts().size() < dbVer.getVersionParts().size()) {
583 bestGuess = dbVer;
584 bestGuessConf = conf;
585 }
586 }
587 }
588 }
589 if (bestGuessConf == null || bestGuessConf.compareTo(conf) > 0) {
590 if (bestGuess.getVersionParts().size() < evVer.getVersionParts().size()) {
591 bestGuess = evVer;
592 bestGuessConf = conf;
593 }
594 }
595 }
596 }
597 final String cpeName = String.format("cpe:/a:%s:%s:%s", vendor, product, bestGuess.toString());
598 String url = null;
599 if (hasBroadMatch) {
600 final String cpeUrlName = String.format("cpe:/a:%s:%s", vendor, product);
601 url = String.format(NVD_SEARCH_URL, URLEncoder.encode(cpeUrlName, "UTF-8"));
602 }
603 if (bestGuessConf == null) {
604 bestGuessConf = Confidence.LOW;
605 }
606 final IdentifierMatch match = new IdentifierMatch("cpe", cpeName, url, IdentifierConfidence.BEST_GUESS, bestGuessConf);
607 collected.add(match);
608
609 Collections.sort(collected);
610 final IdentifierConfidence bestIdentifierQuality = collected.get(0).getConfidence();
611 final Confidence bestEvidenceQuality = collected.get(0).getEvidenceConfidence();
612 boolean identifierAdded = false;
613 for (IdentifierMatch m : collected) {
614 if (bestIdentifierQuality.equals(m.getConfidence())
615 && bestEvidenceQuality.equals(m.getEvidenceConfidence())) {
616 final Identifier i = m.getIdentifier();
617 if (bestIdentifierQuality == IdentifierConfidence.BEST_GUESS) {
618 i.setConfidence(Confidence.LOW);
619 } else {
620 i.setConfidence(bestEvidenceQuality);
621 }
622 dependency.addIdentifier(i);
623 identifierAdded = true;
624 }
625 }
626 return identifierAdded;
627 }
628
629
630
631
632 private enum IdentifierConfidence {
633
634
635
636
637 EXACT_MATCH,
638
639
640
641 BEST_GUESS,
642
643
644
645
646
647 BROAD_MATCH
648 }
649
650
651
652
653
654 private static class IdentifierMatch implements Comparable<IdentifierMatch> {
655
656
657
658
659
660
661
662
663
664
665
666
667 IdentifierMatch(String type, String value, String url, IdentifierConfidence identifierConfidence, Confidence evidenceConfidence) {
668 this.identifier = new Identifier(type, value, url);
669 this.confidence = identifierConfidence;
670 this.evidenceConfidence = evidenceConfidence;
671 }
672
673
674
675
676 private Confidence evidenceConfidence;
677
678
679
680
681
682
683 public Confidence getEvidenceConfidence() {
684 return evidenceConfidence;
685 }
686
687
688
689
690
691
692 public void setEvidenceConfidence(Confidence evidenceConfidence) {
693 this.evidenceConfidence = evidenceConfidence;
694 }
695
696
697
698 private IdentifierConfidence confidence;
699
700
701
702
703
704
705 public IdentifierConfidence getConfidence() {
706 return confidence;
707 }
708
709
710
711
712
713
714 public void setConfidence(IdentifierConfidence confidence) {
715 this.confidence = confidence;
716 }
717
718
719
720 private Identifier identifier;
721
722
723
724
725
726
727 public Identifier getIdentifier() {
728 return identifier;
729 }
730
731
732
733
734
735
736 public void setIdentifier(Identifier identifier) {
737 this.identifier = identifier;
738 }
739
740
741
742
743
744
745
746
747 @Override
748 public String toString() {
749 return "IdentifierMatch{" + "evidenceConfidence=" + evidenceConfidence
750 + ", confidence=" + confidence + ", identifier=" + identifier + '}';
751 }
752
753
754
755
756
757
758 @Override
759 public int hashCode() {
760 int hash = 5;
761 hash = 97 * hash + (this.evidenceConfidence != null ? this.evidenceConfidence.hashCode() : 0);
762 hash = 97 * hash + (this.confidence != null ? this.confidence.hashCode() : 0);
763 hash = 97 * hash + (this.identifier != null ? this.identifier.hashCode() : 0);
764 return hash;
765 }
766
767
768
769
770
771
772
773 @Override
774 public boolean equals(Object obj) {
775 if (obj == null) {
776 return false;
777 }
778 if (getClass() != obj.getClass()) {
779 return false;
780 }
781 final IdentifierMatch other = (IdentifierMatch) obj;
782 if (this.evidenceConfidence != other.evidenceConfidence) {
783 return false;
784 }
785 if (this.confidence != other.confidence) {
786 return false;
787 }
788 if (this.identifier != other.identifier && (this.identifier == null || !this.identifier.equals(other.identifier))) {
789 return false;
790 }
791 return true;
792 }
793
794
795
796
797
798
799
800
801
802 @Override
803 public int compareTo(IdentifierMatch o) {
804 int conf = this.confidence.compareTo(o.confidence);
805 if (conf == 0) {
806 conf = this.evidenceConfidence.compareTo(o.evidenceConfidence);
807 if (conf == 0) {
808 conf = identifier.compareTo(o.identifier);
809 }
810 }
811 return conf;
812 }
813 }
814 }