1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.owasp.dependencycheck.analyzer;
19
20 import java.io.IOException;
21 import java.io.UnsupportedEncodingException;
22 import java.net.URLEncoder;
23 import java.util.ArrayList;
24 import java.util.Collections;
25 import java.util.List;
26 import java.util.Set;
27 import java.util.StringTokenizer;
28 import org.apache.lucene.document.Document;
29 import org.apache.lucene.index.CorruptIndexException;
30 import org.apache.lucene.queryparser.classic.ParseException;
31 import org.apache.lucene.search.ScoreDoc;
32 import org.apache.lucene.search.TopDocs;
33 import org.owasp.dependencycheck.Engine;
34 import org.owasp.dependencycheck.analyzer.exception.AnalysisException;
35 import org.owasp.dependencycheck.data.cpe.CpeMemoryIndex;
36 import org.owasp.dependencycheck.data.cpe.Fields;
37 import org.owasp.dependencycheck.data.cpe.IndexEntry;
38 import org.owasp.dependencycheck.data.cpe.IndexException;
39 import org.owasp.dependencycheck.data.lucene.LuceneUtils;
40 import org.owasp.dependencycheck.data.nvdcve.CveDB;
41 import org.owasp.dependencycheck.data.nvdcve.DatabaseException;
42 import org.owasp.dependencycheck.dependency.Confidence;
43 import org.owasp.dependencycheck.dependency.Dependency;
44 import org.owasp.dependencycheck.dependency.Evidence;
45 import org.owasp.dependencycheck.dependency.EvidenceCollection;
46 import org.owasp.dependencycheck.dependency.Identifier;
47 import org.owasp.dependencycheck.dependency.VulnerableSoftware;
48 import org.owasp.dependencycheck.utils.DependencyVersion;
49 import org.owasp.dependencycheck.utils.DependencyVersionUtil;
50 import org.slf4j.Logger;
51 import org.slf4j.LoggerFactory;
52
53
54
55
56
57
58
59 public class CPEAnalyzer implements Analyzer {
60
61
62
63
64 private static final Logger LOGGER = LoggerFactory.getLogger(CPEAnalyzer.class);
65
66
67
68 static final int MAX_QUERY_RESULTS = 25;
69
70
71
72 static final String WEIGHTING_BOOST = "^5";
73
74
75
76 static final String CLEANSE_CHARACTER_RX = "[^A-Za-z0-9 ._-]";
77
78
79
80 static final String CLEANSE_NONALPHA_RX = "[^A-Za-z]*";
81
82
83
84 static final int STRING_BUILDER_BUFFER = 20;
85
86
87
88 private CpeMemoryIndex cpe;
89
90
91
92 private CveDB cve;
93
94
95
96
97 public static final String NVD_SEARCH_URL = "https://web.nvd.nist.gov/view/vuln/search-results?adv_search=true&cves=on&cpe_version=%s";
98
99
100
101
102
103
104 @Override
105 public String getName() {
106 return "CPE Analyzer";
107 }
108
109
110
111
112
113
114 @Override
115 public AnalysisPhase getAnalysisPhase() {
116 return AnalysisPhase.IDENTIFIER_ANALYSIS;
117 }
118
119
120
121
122
123
124 @Override
125 public void initialize() throws Exception {
126 this.open();
127 }
128
129
130
131
132
133
134
135
136 public void open() throws IOException, DatabaseException {
137 cve = new CveDB();
138 cve.open();
139 cpe = CpeMemoryIndex.getInstance();
140 try {
141 LOGGER.info("Creating the CPE Index");
142 final long creationStart = System.currentTimeMillis();
143 cpe.open(cve);
144 LOGGER.info("CPE Index Created ({} ms)", System.currentTimeMillis() - creationStart);
145 } catch (IndexException ex) {
146 LOGGER.debug("IndexException", ex);
147 throw new DatabaseException(ex);
148 }
149 }
150
151
152
153
154 @Override
155 public void close() {
156 if (cpe != null) {
157 cpe.close();
158 cpe = null;
159 }
160 if (cve != null) {
161 cve.close();
162 cve = null;
163 }
164 }
165
166 public boolean isOpen() {
167 return cpe != null && cpe.isOpen();
168 }
169
170
171
172
173
174
175
176
177
178
179 protected void determineCPE(Dependency dependency) throws CorruptIndexException, IOException, ParseException {
180
181 String vendors = "";
182 String products = "";
183 for (Confidence confidence : Confidence.values()) {
184 if (dependency.getVendorEvidence().contains(confidence)) {
185 vendors = addEvidenceWithoutDuplicateTerms(vendors, dependency.getVendorEvidence(), confidence);
186 LOGGER.debug("vendor search: {}", vendors);
187 }
188 if (dependency.getProductEvidence().contains(confidence)) {
189 products = addEvidenceWithoutDuplicateTerms(products, dependency.getProductEvidence(), confidence);
190 LOGGER.debug("product search: {}", products);
191 }
192 if (!vendors.isEmpty() && !products.isEmpty()) {
193 final List<IndexEntry> entries = searchCPE(vendors, products, dependency.getProductEvidence().getWeighting(),
194 dependency.getVendorEvidence().getWeighting());
195 if (entries == null) {
196 continue;
197 }
198 boolean identifierAdded = false;
199 for (IndexEntry e : entries) {
200 LOGGER.debug("Verifying entry: {}", e);
201 if (verifyEntry(e, dependency)) {
202 final String vendor = e.getVendor();
203 final String product = e.getProduct();
204 LOGGER.debug("identified vendor/product: {}/{}", vendor, product);
205 identifierAdded |= determineIdentifiers(dependency, vendor, product, confidence);
206 }
207 }
208 if (identifierAdded) {
209 break;
210 }
211 }
212 }
213 }
214
215
216
217
218
219
220
221
222
223
224
225 private String addEvidenceWithoutDuplicateTerms(final String text, final EvidenceCollection ec, Confidence confidenceFilter) {
226 final String txt = (text == null) ? "" : text;
227 final StringBuilder sb = new StringBuilder(txt.length() + (20 * ec.size()));
228 sb.append(' ').append(txt).append(' ');
229 for (Evidence e : ec.iterator(confidenceFilter)) {
230 String value = e.getValue();
231
232
233
234 if (value.startsWith("http://")) {
235 value = value.substring(7).replaceAll("\\.", " ");
236 }
237 if (value.startsWith("https://")) {
238 value = value.substring(8).replaceAll("\\.", " ");
239 }
240 if (sb.indexOf(" " + value + " ") < 0) {
241 sb.append(value).append(' ');
242 }
243 }
244 return sb.toString().trim();
245 }
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262 protected List<IndexEntry> searchCPE(String vendor, String product,
263 Set<String> vendorWeightings, Set<String> productWeightings) {
264
265 final List<IndexEntry> ret = new ArrayList<IndexEntry>(MAX_QUERY_RESULTS);
266
267 final String searchString = buildSearch(vendor, product, vendorWeightings, productWeightings);
268 if (searchString == null) {
269 return ret;
270 }
271 try {
272 final TopDocs docs = cpe.search(searchString, MAX_QUERY_RESULTS);
273 for (ScoreDoc d : docs.scoreDocs) {
274 if (d.score >= 0.08) {
275 final Document doc = cpe.getDocument(d.doc);
276 final IndexEntry entry = new IndexEntry();
277 entry.setVendor(doc.get(Fields.VENDOR));
278 entry.setProduct(doc.get(Fields.PRODUCT));
279 entry.setSearchScore(d.score);
280 if (!ret.contains(entry)) {
281 ret.add(entry);
282 }
283 }
284 }
285 return ret;
286 } catch (ParseException ex) {
287 LOGGER.warn("An error occured querying the CPE data. See the log for more details.");
288 LOGGER.info("Unable to parse: {}", searchString, ex);
289 } catch (IOException ex) {
290 LOGGER.warn("An error occured reading CPE data. See the log for more details.");
291 LOGGER.info("IO Error with search string: {}", searchString, ex);
292 }
293 return null;
294 }
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310 protected String buildSearch(String vendor, String product,
311 Set<String> vendorWeighting, Set<String> productWeightings) {
312 final String v = vendor;
313 final String p = product;
314 final StringBuilder sb = new StringBuilder(v.length() + p.length()
315 + Fields.PRODUCT.length() + Fields.VENDOR.length() + STRING_BUILDER_BUFFER);
316
317 if (!appendWeightedSearch(sb, Fields.PRODUCT, p, productWeightings)) {
318 return null;
319 }
320 sb.append(" AND ");
321 if (!appendWeightedSearch(sb, Fields.VENDOR, v, vendorWeighting)) {
322 return null;
323 }
324 return sb.toString();
325 }
326
327
328
329
330
331
332
333
334
335
336
337 private boolean appendWeightedSearch(StringBuilder sb, String field, String searchText, Set<String> weightedText) {
338 sb.append(' ').append(field).append(":( ");
339
340 final String cleanText = cleanseText(searchText);
341
342 if (cleanText.isEmpty()) {
343 return false;
344 }
345
346 if (weightedText == null || weightedText.isEmpty()) {
347 LuceneUtils.appendEscapedLuceneQuery(sb, cleanText);
348 } else {
349 final StringTokenizer tokens = new StringTokenizer(cleanText);
350 while (tokens.hasMoreElements()) {
351 final String word = tokens.nextToken();
352 StringBuilder temp = null;
353 for (String weighted : weightedText) {
354 final String weightedStr = cleanseText(weighted);
355 if (equalsIgnoreCaseAndNonAlpha(word, weightedStr)) {
356 temp = new StringBuilder(word.length() + 2);
357 LuceneUtils.appendEscapedLuceneQuery(temp, word);
358 temp.append(WEIGHTING_BOOST);
359 if (!word.equalsIgnoreCase(weightedStr)) {
360 temp.append(' ');
361 LuceneUtils.appendEscapedLuceneQuery(temp, weightedStr);
362 temp.append(WEIGHTING_BOOST);
363 }
364 break;
365 }
366 }
367 sb.append(' ');
368 if (temp == null) {
369 LuceneUtils.appendEscapedLuceneQuery(sb, word);
370 } else {
371 sb.append(temp);
372 }
373 }
374 }
375 sb.append(" ) ");
376 return true;
377 }
378
379
380
381
382
383
384
385 private String cleanseText(String text) {
386 return text.replaceAll(CLEANSE_CHARACTER_RX, " ");
387 }
388
389
390
391
392
393
394
395
396 private boolean equalsIgnoreCaseAndNonAlpha(String l, String r) {
397 if (l == null || r == null) {
398 return false;
399 }
400
401 final String left = l.replaceAll(CLEANSE_NONALPHA_RX, "");
402 final String right = r.replaceAll(CLEANSE_NONALPHA_RX, "");
403 return left.equalsIgnoreCase(right);
404 }
405
406
407
408
409
410
411
412
413
414 private boolean verifyEntry(final IndexEntry entry, final Dependency dependency) {
415 boolean isValid = false;
416
417
418
419 if (collectionContainsString(dependency.getProductEvidence(), entry.getProduct())
420 && collectionContainsString(dependency.getVendorEvidence(), entry.getVendor())) {
421
422 isValid = true;
423 }
424 return isValid;
425 }
426
427
428
429
430
431
432
433
434 private boolean collectionContainsString(EvidenceCollection ec, String text) {
435
436 if (text == null) {
437 return false;
438 }
439 final String[] words = text.split("[\\s_-]");
440 final List<String> list = new ArrayList<String>();
441 String tempWord = null;
442 for (String word : words) {
443
444
445
446
447 if (tempWord != null) {
448 list.add(tempWord + word);
449 tempWord = null;
450 } else if (word.length() <= 2) {
451 tempWord = word;
452 } else {
453 list.add(word);
454 }
455 }
456 if (tempWord != null) {
457 if (!list.isEmpty()) {
458 final String tmp = list.get(list.size() - 1) + tempWord;
459 list.add(tmp);
460 } else {
461 list.add(tempWord);
462 }
463 }
464 if (list.isEmpty()) {
465 return false;
466 }
467 boolean contains = true;
468 for (String word : list) {
469 contains &= ec.containsUsedString(word);
470 }
471 return contains;
472 }
473
474
475
476
477
478
479
480
481 @Override
482 public void analyze(Dependency dependency, Engine engine) throws AnalysisException {
483 try {
484 determineCPE(dependency);
485 } catch (CorruptIndexException ex) {
486 throw new AnalysisException("CPE Index is corrupt.", ex);
487 } catch (IOException ex) {
488 throw new AnalysisException("Failure opening the CPE Index.", ex);
489 } catch (ParseException ex) {
490 throw new AnalysisException("Unable to parse the generated Lucene query for this dependency.", ex);
491 }
492 }
493
494
495
496
497
498
499
500
501
502
503
504
505
506 protected boolean determineIdentifiers(Dependency dependency, String vendor, String product,
507 Confidence currentConfidence) throws UnsupportedEncodingException {
508 final Set<VulnerableSoftware> cpes = cve.getCPEs(vendor, product);
509 DependencyVersion bestGuess = new DependencyVersion("-");
510 Confidence bestGuessConf = null;
511 boolean hasBroadMatch = false;
512 final List<IdentifierMatch> collected = new ArrayList<IdentifierMatch>();
513 for (Confidence conf : Confidence.values()) {
514
515
516
517 for (Evidence evidence : dependency.getVersionEvidence().iterator(conf)) {
518 final DependencyVersion evVer = DependencyVersionUtil.parseVersion(evidence.getValue());
519 if (evVer == null) {
520 continue;
521 }
522 for (VulnerableSoftware vs : cpes) {
523 DependencyVersion dbVer;
524 if (vs.getUpdate() != null && !vs.getUpdate().isEmpty()) {
525 dbVer = DependencyVersionUtil.parseVersion(vs.getVersion() + '.' + vs.getUpdate());
526 } else {
527 dbVer = DependencyVersionUtil.parseVersion(vs.getVersion());
528 }
529 if (dbVer == null) {
530 hasBroadMatch = true;
531 final String url = String.format(NVD_SEARCH_URL, URLEncoder.encode(vs.getName(), "UTF-8"));
532 final IdentifierMatch match = new IdentifierMatch("cpe", vs.getName(), url, IdentifierConfidence.BROAD_MATCH, conf);
533 collected.add(match);
534 } else if (evVer.equals(dbVer)) {
535 final String url = String.format(NVD_SEARCH_URL, URLEncoder.encode(vs.getName(), "UTF-8"));
536 final IdentifierMatch match = new IdentifierMatch("cpe", vs.getName(), url, IdentifierConfidence.EXACT_MATCH, conf);
537 collected.add(match);
538 } else {
539
540 if (evVer.getVersionParts().size() <= dbVer.getVersionParts().size()
541 && evVer.matchesAtLeastThreeLevels(dbVer)) {
542 if (bestGuessConf == null || bestGuessConf.compareTo(conf) > 0) {
543 if (bestGuess.getVersionParts().size() < dbVer.getVersionParts().size()) {
544 bestGuess = dbVer;
545 bestGuessConf = conf;
546 }
547 }
548 }
549 }
550 }
551 if (bestGuessConf == null || bestGuessConf.compareTo(conf) > 0) {
552 if (bestGuess.getVersionParts().size() < evVer.getVersionParts().size()) {
553 bestGuess = evVer;
554 bestGuessConf = conf;
555 }
556 }
557 }
558 }
559 final String cpeName = String.format("cpe:/a:%s:%s:%s", vendor, product, bestGuess.toString());
560 String url = null;
561 if (hasBroadMatch) {
562 final String cpeUrlName = String.format("cpe:/a:%s:%s", vendor, product);
563 url = String.format(NVD_SEARCH_URL, URLEncoder.encode(cpeUrlName, "UTF-8"));
564 }
565 if (bestGuessConf == null) {
566 bestGuessConf = Confidence.LOW;
567 }
568 final IdentifierMatch match = new IdentifierMatch("cpe", cpeName, url, IdentifierConfidence.BEST_GUESS, bestGuessConf);
569 collected.add(match);
570
571 Collections.sort(collected);
572 final IdentifierConfidence bestIdentifierQuality = collected.get(0).getConfidence();
573 final Confidence bestEvidenceQuality = collected.get(0).getEvidenceConfidence();
574 boolean identifierAdded = false;
575 for (IdentifierMatch m : collected) {
576 if (bestIdentifierQuality.equals(m.getConfidence())
577 && bestEvidenceQuality.equals(m.getEvidenceConfidence())) {
578 final Identifier i = m.getIdentifier();
579 if (bestIdentifierQuality == IdentifierConfidence.BEST_GUESS) {
580 i.setConfidence(Confidence.LOW);
581 } else {
582 i.setConfidence(bestEvidenceQuality);
583 }
584 dependency.addIdentifier(i);
585 identifierAdded = true;
586 }
587 }
588 return identifierAdded;
589 }
590
591
592
593
594 private enum IdentifierConfidence {
595
596
597
598
599 EXACT_MATCH,
600
601
602
603 BEST_GUESS,
604
605
606
607
608 BROAD_MATCH
609 }
610
611
612
613
614 private static class IdentifierMatch implements Comparable<IdentifierMatch> {
615
616
617
618
619
620
621
622
623
624
625 IdentifierMatch(String type, String value, String url, IdentifierConfidence identifierConfidence, Confidence evidenceConfidence) {
626 this.identifier = new Identifier(type, value, url);
627 this.confidence = identifierConfidence;
628 this.evidenceConfidence = evidenceConfidence;
629 }
630
631
632
633
634 private Confidence evidenceConfidence;
635
636
637
638
639
640
641 public Confidence getEvidenceConfidence() {
642 return evidenceConfidence;
643 }
644
645
646
647
648
649
650 public void setEvidenceConfidence(Confidence evidenceConfidence) {
651 this.evidenceConfidence = evidenceConfidence;
652 }
653
654
655
656 private IdentifierConfidence confidence;
657
658
659
660
661
662
663 public IdentifierConfidence getConfidence() {
664 return confidence;
665 }
666
667
668
669
670
671
672 public void setConfidence(IdentifierConfidence confidence) {
673 this.confidence = confidence;
674 }
675
676
677
678 private Identifier identifier;
679
680
681
682
683
684
685 public Identifier getIdentifier() {
686 return identifier;
687 }
688
689
690
691
692
693
694 public void setIdentifier(Identifier identifier) {
695 this.identifier = identifier;
696 }
697
698
699
700
701
702
703
704
705 @Override
706 public String toString() {
707 return "IdentifierMatch{" + "evidenceConfidence=" + evidenceConfidence
708 + ", confidence=" + confidence + ", identifier=" + identifier + '}';
709 }
710
711
712
713
714
715
716 @Override
717 public int hashCode() {
718 int hash = 5;
719 hash = 97 * hash + (this.evidenceConfidence != null ? this.evidenceConfidence.hashCode() : 0);
720 hash = 97 * hash + (this.confidence != null ? this.confidence.hashCode() : 0);
721 hash = 97 * hash + (this.identifier != null ? this.identifier.hashCode() : 0);
722 return hash;
723 }
724
725
726
727
728
729
730
731 @Override
732 public boolean equals(Object obj) {
733 if (obj == null) {
734 return false;
735 }
736 if (getClass() != obj.getClass()) {
737 return false;
738 }
739 final IdentifierMatch other = (IdentifierMatch) obj;
740 if (this.evidenceConfidence != other.evidenceConfidence) {
741 return false;
742 }
743 if (this.confidence != other.confidence) {
744 return false;
745 }
746 if (this.identifier != other.identifier && (this.identifier == null || !this.identifier.equals(other.identifier))) {
747 return false;
748 }
749 return true;
750 }
751
752
753
754
755
756
757
758
759 @Override
760 public int compareTo(IdentifierMatch o) {
761 int conf = this.confidence.compareTo(o.confidence);
762 if (conf == 0) {
763 conf = this.evidenceConfidence.compareTo(o.evidenceConfidence);
764 if (conf == 0) {
765 conf = identifier.compareTo(o.identifier);
766 }
767 }
768 return conf;
769 }
770 }
771 }