1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.owasp.dependencycheck.analyzer;
19
20 import java.io.IOException;
21 import java.io.UnsupportedEncodingException;
22 import java.net.URLEncoder;
23 import java.util.ArrayList;
24 import java.util.Collections;
25 import java.util.List;
26 import java.util.Set;
27 import java.util.StringTokenizer;
28 import org.apache.lucene.document.Document;
29 import org.apache.lucene.index.CorruptIndexException;
30 import org.apache.lucene.queryparser.classic.ParseException;
31 import org.apache.lucene.search.ScoreDoc;
32 import org.apache.lucene.search.TopDocs;
33 import org.owasp.dependencycheck.Engine;
34 import org.owasp.dependencycheck.analyzer.exception.AnalysisException;
35 import org.owasp.dependencycheck.data.cpe.CpeMemoryIndex;
36 import org.owasp.dependencycheck.data.cpe.Fields;
37 import org.owasp.dependencycheck.data.cpe.IndexEntry;
38 import org.owasp.dependencycheck.data.cpe.IndexException;
39 import org.owasp.dependencycheck.data.lucene.LuceneUtils;
40 import org.owasp.dependencycheck.data.nvdcve.CveDB;
41 import org.owasp.dependencycheck.data.nvdcve.DatabaseException;
42 import org.owasp.dependencycheck.dependency.Confidence;
43 import org.owasp.dependencycheck.dependency.Dependency;
44 import org.owasp.dependencycheck.dependency.Evidence;
45 import org.owasp.dependencycheck.dependency.EvidenceCollection;
46 import org.owasp.dependencycheck.dependency.Identifier;
47 import org.owasp.dependencycheck.dependency.VulnerableSoftware;
48 import org.owasp.dependencycheck.utils.DependencyVersion;
49 import org.owasp.dependencycheck.utils.DependencyVersionUtil;
50 import org.slf4j.Logger;
51 import org.slf4j.LoggerFactory;
52
53
54
55
56
57
58
59 public class CPEAnalyzer implements Analyzer {
60
61
62
63
64 private static final Logger LOGGER = LoggerFactory.getLogger(CPEAnalyzer.class);
65
66
67
68 static final int MAX_QUERY_RESULTS = 25;
69
70
71
72 static final String WEIGHTING_BOOST = "^5";
73
74
75
76 static final String CLEANSE_CHARACTER_RX = "[^A-Za-z0-9 ._-]";
77
78
79
80 static final String CLEANSE_NONALPHA_RX = "[^A-Za-z]*";
81
82
83
84 static final int STRING_BUILDER_BUFFER = 20;
85
86
87
88 private CpeMemoryIndex cpe;
89
90
91
92 private CveDB cve;
93
94
95
96
97 public static final String NVD_SEARCH_URL = "https://web.nvd.nist.gov/view/vuln/search-results?adv_search=true&cves=on&cpe_version=%s";
98
99
100
101
102
103
104 @Override
105 public String getName() {
106 return "CPE Analyzer";
107 }
108
109
110
111
112
113
114 @Override
115 public AnalysisPhase getAnalysisPhase() {
116 return AnalysisPhase.IDENTIFIER_ANALYSIS;
117 }
118
119
120
121
122
123
124 @Override
125 public void initialize() throws Exception {
126 this.open();
127 }
128
129
130
131
132
133
134
135
136 public void open() throws IOException, DatabaseException {
137 if (!isOpen()) {
138 cve = new CveDB();
139 cve.open();
140 cpe = CpeMemoryIndex.getInstance();
141 try {
142 LOGGER.info("Creating the CPE Index");
143 final long creationStart = System.currentTimeMillis();
144 cpe.open(cve);
145 LOGGER.info("CPE Index Created ({} ms)", System.currentTimeMillis() - creationStart);
146 } catch (IndexException ex) {
147 LOGGER.debug("IndexException", ex);
148 throw new DatabaseException(ex);
149 }
150 }
151 }
152
153
154
155
156 @Override
157 public void close() {
158 if (cpe != null) {
159 cpe.close();
160 cpe = null;
161 }
162 if (cve != null) {
163 cve.close();
164 cve = null;
165 }
166 }
167
168 public boolean isOpen() {
169 return cpe != null && cpe.isOpen();
170 }
171
172
173
174
175
176
177
178
179
180
181 protected void determineCPE(Dependency dependency) throws CorruptIndexException, IOException, ParseException {
182
183 String vendors = "";
184 String products = "";
185 for (Confidence confidence : Confidence.values()) {
186 if (dependency.getVendorEvidence().contains(confidence)) {
187 vendors = addEvidenceWithoutDuplicateTerms(vendors, dependency.getVendorEvidence(), confidence);
188 LOGGER.debug("vendor search: {}", vendors);
189 }
190 if (dependency.getProductEvidence().contains(confidence)) {
191 products = addEvidenceWithoutDuplicateTerms(products, dependency.getProductEvidence(), confidence);
192 LOGGER.debug("product search: {}", products);
193 }
194 if (!vendors.isEmpty() && !products.isEmpty()) {
195 final List<IndexEntry> entries = searchCPE(vendors, products, dependency.getProductEvidence().getWeighting(),
196 dependency.getVendorEvidence().getWeighting());
197 if (entries == null) {
198 continue;
199 }
200 boolean identifierAdded = false;
201 for (IndexEntry e : entries) {
202 LOGGER.debug("Verifying entry: {}", e);
203 if (verifyEntry(e, dependency)) {
204 final String vendor = e.getVendor();
205 final String product = e.getProduct();
206 LOGGER.debug("identified vendor/product: {}/{}", vendor, product);
207 identifierAdded |= determineIdentifiers(dependency, vendor, product, confidence);
208 }
209 }
210 if (identifierAdded) {
211 break;
212 }
213 }
214 }
215 }
216
217
218
219
220
221
222
223
224
225
226
227 private String addEvidenceWithoutDuplicateTerms(final String text, final EvidenceCollection ec, Confidence confidenceFilter) {
228 final String txt = (text == null) ? "" : text;
229 final StringBuilder sb = new StringBuilder(txt.length() + (20 * ec.size()));
230 sb.append(' ').append(txt).append(' ');
231 for (Evidence e : ec.iterator(confidenceFilter)) {
232 String value = e.getValue();
233
234
235
236 if (value.startsWith("http://")) {
237 value = value.substring(7).replaceAll("\\.", " ");
238 }
239 if (value.startsWith("https://")) {
240 value = value.substring(8).replaceAll("\\.", " ");
241 }
242 if (sb.indexOf(" " + value + " ") < 0) {
243 sb.append(value).append(' ');
244 }
245 }
246 return sb.toString().trim();
247 }
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264 protected List<IndexEntry> searchCPE(String vendor, String product,
265 Set<String> vendorWeightings, Set<String> productWeightings) {
266
267 final List<IndexEntry> ret = new ArrayList<IndexEntry>(MAX_QUERY_RESULTS);
268
269 final String searchString = buildSearch(vendor, product, vendorWeightings, productWeightings);
270 if (searchString == null) {
271 return ret;
272 }
273 try {
274 final TopDocs docs = cpe.search(searchString, MAX_QUERY_RESULTS);
275 for (ScoreDoc d : docs.scoreDocs) {
276 if (d.score >= 0.08) {
277 final Document doc = cpe.getDocument(d.doc);
278 final IndexEntry entry = new IndexEntry();
279 entry.setVendor(doc.get(Fields.VENDOR));
280 entry.setProduct(doc.get(Fields.PRODUCT));
281 entry.setSearchScore(d.score);
282 if (!ret.contains(entry)) {
283 ret.add(entry);
284 }
285 }
286 }
287 return ret;
288 } catch (ParseException ex) {
289 LOGGER.warn("An error occurred querying the CPE data. See the log for more details.");
290 LOGGER.info("Unable to parse: {}", searchString, ex);
291 } catch (IOException ex) {
292 LOGGER.warn("An error occurred reading CPE data. See the log for more details.");
293 LOGGER.info("IO Error with search string: {}", searchString, ex);
294 }
295 return null;
296 }
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312 protected String buildSearch(String vendor, String product,
313 Set<String> vendorWeighting, Set<String> productWeightings) {
314 final String v = vendor;
315 final String p = product;
316 final StringBuilder sb = new StringBuilder(v.length() + p.length()
317 + Fields.PRODUCT.length() + Fields.VENDOR.length() + STRING_BUILDER_BUFFER);
318
319 if (!appendWeightedSearch(sb, Fields.PRODUCT, p, productWeightings)) {
320 return null;
321 }
322 sb.append(" AND ");
323 if (!appendWeightedSearch(sb, Fields.VENDOR, v, vendorWeighting)) {
324 return null;
325 }
326 return sb.toString();
327 }
328
329
330
331
332
333
334
335
336
337
338
339 private boolean appendWeightedSearch(StringBuilder sb, String field, String searchText, Set<String> weightedText) {
340 sb.append(' ').append(field).append(":( ");
341
342 final String cleanText = cleanseText(searchText);
343
344 if (cleanText.isEmpty()) {
345 return false;
346 }
347
348 if (weightedText == null || weightedText.isEmpty()) {
349 LuceneUtils.appendEscapedLuceneQuery(sb, cleanText);
350 } else {
351 final StringTokenizer tokens = new StringTokenizer(cleanText);
352 while (tokens.hasMoreElements()) {
353 final String word = tokens.nextToken();
354 StringBuilder temp = null;
355 for (String weighted : weightedText) {
356 final String weightedStr = cleanseText(weighted);
357 if (equalsIgnoreCaseAndNonAlpha(word, weightedStr)) {
358 temp = new StringBuilder(word.length() + 2);
359 LuceneUtils.appendEscapedLuceneQuery(temp, word);
360 temp.append(WEIGHTING_BOOST);
361 if (!word.equalsIgnoreCase(weightedStr)) {
362 temp.append(' ');
363 LuceneUtils.appendEscapedLuceneQuery(temp, weightedStr);
364 temp.append(WEIGHTING_BOOST);
365 }
366 break;
367 }
368 }
369 sb.append(' ');
370 if (temp == null) {
371 LuceneUtils.appendEscapedLuceneQuery(sb, word);
372 } else {
373 sb.append(temp);
374 }
375 }
376 }
377 sb.append(" ) ");
378 return true;
379 }
380
381
382
383
384
385
386
387 private String cleanseText(String text) {
388 return text.replaceAll(CLEANSE_CHARACTER_RX, " ");
389 }
390
391
392
393
394
395
396
397
398 private boolean equalsIgnoreCaseAndNonAlpha(String l, String r) {
399 if (l == null || r == null) {
400 return false;
401 }
402
403 final String left = l.replaceAll(CLEANSE_NONALPHA_RX, "");
404 final String right = r.replaceAll(CLEANSE_NONALPHA_RX, "");
405 return left.equalsIgnoreCase(right);
406 }
407
408
409
410
411
412
413
414
415
416 private boolean verifyEntry(final IndexEntry entry, final Dependency dependency) {
417 boolean isValid = false;
418
419
420
421 if (collectionContainsString(dependency.getProductEvidence(), entry.getProduct())
422 && collectionContainsString(dependency.getVendorEvidence(), entry.getVendor())) {
423
424 isValid = true;
425 }
426 return isValid;
427 }
428
429
430
431
432
433
434
435
436 private boolean collectionContainsString(EvidenceCollection ec, String text) {
437
438 if (text == null) {
439 return false;
440 }
441 final String[] words = text.split("[\\s_-]");
442 final List<String> list = new ArrayList<String>();
443 String tempWord = null;
444 for (String word : words) {
445
446
447
448
449 if (tempWord != null) {
450 list.add(tempWord + word);
451 tempWord = null;
452 } else if (word.length() <= 2) {
453 tempWord = word;
454 } else {
455 list.add(word);
456 }
457 }
458 if (tempWord != null) {
459 if (!list.isEmpty()) {
460 final String tmp = list.get(list.size() - 1) + tempWord;
461 list.add(tmp);
462 } else {
463 list.add(tempWord);
464 }
465 }
466 if (list.isEmpty()) {
467 return false;
468 }
469 boolean contains = true;
470 for (String word : list) {
471 contains &= ec.containsUsedString(word);
472 }
473 return contains;
474 }
475
476
477
478
479
480
481
482
483 @Override
484 public synchronized void analyze(Dependency dependency, Engine engine) throws AnalysisException {
485 try {
486 determineCPE(dependency);
487 } catch (CorruptIndexException ex) {
488 throw new AnalysisException("CPE Index is corrupt.", ex);
489 } catch (IOException ex) {
490 throw new AnalysisException("Failure opening the CPE Index.", ex);
491 } catch (ParseException ex) {
492 throw new AnalysisException("Unable to parse the generated Lucene query for this dependency.", ex);
493 }
494 }
495
496
497
498
499
500
501
502
503
504
505
506
507
508 protected boolean determineIdentifiers(Dependency dependency, String vendor, String product,
509 Confidence currentConfidence) throws UnsupportedEncodingException {
510 final Set<VulnerableSoftware> cpes = cve.getCPEs(vendor, product);
511 DependencyVersion bestGuess = new DependencyVersion("-");
512 Confidence bestGuessConf = null;
513 boolean hasBroadMatch = false;
514 final List<IdentifierMatch> collected = new ArrayList<IdentifierMatch>();
515 for (Confidence conf : Confidence.values()) {
516
517
518
519 for (Evidence evidence : dependency.getVersionEvidence().iterator(conf)) {
520 final DependencyVersion evVer = DependencyVersionUtil.parseVersion(evidence.getValue());
521 if (evVer == null) {
522 continue;
523 }
524 for (VulnerableSoftware vs : cpes) {
525 DependencyVersion dbVer;
526 if (vs.getUpdate() != null && !vs.getUpdate().isEmpty()) {
527 dbVer = DependencyVersionUtil.parseVersion(vs.getVersion() + '.' + vs.getUpdate());
528 } else {
529 dbVer = DependencyVersionUtil.parseVersion(vs.getVersion());
530 }
531 if (dbVer == null) {
532 hasBroadMatch = true;
533 final String url = String.format(NVD_SEARCH_URL, URLEncoder.encode(vs.getName(), "UTF-8"));
534 final IdentifierMatch match = new IdentifierMatch("cpe", vs.getName(), url, IdentifierConfidence.BROAD_MATCH, conf);
535 collected.add(match);
536 } else if (evVer.equals(dbVer)) {
537 final String url = String.format(NVD_SEARCH_URL, URLEncoder.encode(vs.getName(), "UTF-8"));
538 final IdentifierMatch match = new IdentifierMatch("cpe", vs.getName(), url, IdentifierConfidence.EXACT_MATCH, conf);
539 collected.add(match);
540 } else {
541
542 if (evVer.getVersionParts().size() <= dbVer.getVersionParts().size()
543 && evVer.matchesAtLeastThreeLevels(dbVer)) {
544 if (bestGuessConf == null || bestGuessConf.compareTo(conf) > 0) {
545 if (bestGuess.getVersionParts().size() < dbVer.getVersionParts().size()) {
546 bestGuess = dbVer;
547 bestGuessConf = conf;
548 }
549 }
550 }
551 }
552 }
553 if (bestGuessConf == null || bestGuessConf.compareTo(conf) > 0) {
554 if (bestGuess.getVersionParts().size() < evVer.getVersionParts().size()) {
555 bestGuess = evVer;
556 bestGuessConf = conf;
557 }
558 }
559 }
560 }
561 final String cpeName = String.format("cpe:/a:%s:%s:%s", vendor, product, bestGuess.toString());
562 String url = null;
563 if (hasBroadMatch) {
564 final String cpeUrlName = String.format("cpe:/a:%s:%s", vendor, product);
565 url = String.format(NVD_SEARCH_URL, URLEncoder.encode(cpeUrlName, "UTF-8"));
566 }
567 if (bestGuessConf == null) {
568 bestGuessConf = Confidence.LOW;
569 }
570 final IdentifierMatch match = new IdentifierMatch("cpe", cpeName, url, IdentifierConfidence.BEST_GUESS, bestGuessConf);
571 collected.add(match);
572
573 Collections.sort(collected);
574 final IdentifierConfidence bestIdentifierQuality = collected.get(0).getConfidence();
575 final Confidence bestEvidenceQuality = collected.get(0).getEvidenceConfidence();
576 boolean identifierAdded = false;
577 for (IdentifierMatch m : collected) {
578 if (bestIdentifierQuality.equals(m.getConfidence())
579 && bestEvidenceQuality.equals(m.getEvidenceConfidence())) {
580 final Identifier i = m.getIdentifier();
581 if (bestIdentifierQuality == IdentifierConfidence.BEST_GUESS) {
582 i.setConfidence(Confidence.LOW);
583 } else {
584 i.setConfidence(bestEvidenceQuality);
585 }
586 dependency.addIdentifier(i);
587 identifierAdded = true;
588 }
589 }
590 return identifierAdded;
591 }
592
593
594
595
596 private enum IdentifierConfidence {
597
598
599
600
601 EXACT_MATCH,
602
603
604
605 BEST_GUESS,
606
607
608
609
610 BROAD_MATCH
611 }
612
613
614
615
616 private static class IdentifierMatch implements Comparable<IdentifierMatch> {
617
618
619
620
621
622
623
624
625
626
627 IdentifierMatch(String type, String value, String url, IdentifierConfidence identifierConfidence, Confidence evidenceConfidence) {
628 this.identifier = new Identifier(type, value, url);
629 this.confidence = identifierConfidence;
630 this.evidenceConfidence = evidenceConfidence;
631 }
632
633
634
635
636 private Confidence evidenceConfidence;
637
638
639
640
641
642
643 public Confidence getEvidenceConfidence() {
644 return evidenceConfidence;
645 }
646
647
648
649
650
651
652 public void setEvidenceConfidence(Confidence evidenceConfidence) {
653 this.evidenceConfidence = evidenceConfidence;
654 }
655
656
657
658 private IdentifierConfidence confidence;
659
660
661
662
663
664
665 public IdentifierConfidence getConfidence() {
666 return confidence;
667 }
668
669
670
671
672
673
674 public void setConfidence(IdentifierConfidence confidence) {
675 this.confidence = confidence;
676 }
677
678
679
680 private Identifier identifier;
681
682
683
684
685
686
687 public Identifier getIdentifier() {
688 return identifier;
689 }
690
691
692
693
694
695
696 public void setIdentifier(Identifier identifier) {
697 this.identifier = identifier;
698 }
699
700
701
702
703
704
705
706
707 @Override
708 public String toString() {
709 return "IdentifierMatch{" + "evidenceConfidence=" + evidenceConfidence
710 + ", confidence=" + confidence + ", identifier=" + identifier + '}';
711 }
712
713
714
715
716
717
718 @Override
719 public int hashCode() {
720 int hash = 5;
721 hash = 97 * hash + (this.evidenceConfidence != null ? this.evidenceConfidence.hashCode() : 0);
722 hash = 97 * hash + (this.confidence != null ? this.confidence.hashCode() : 0);
723 hash = 97 * hash + (this.identifier != null ? this.identifier.hashCode() : 0);
724 return hash;
725 }
726
727
728
729
730
731
732
733 @Override
734 public boolean equals(Object obj) {
735 if (obj == null) {
736 return false;
737 }
738 if (getClass() != obj.getClass()) {
739 return false;
740 }
741 final IdentifierMatch other = (IdentifierMatch) obj;
742 if (this.evidenceConfidence != other.evidenceConfidence) {
743 return false;
744 }
745 if (this.confidence != other.confidence) {
746 return false;
747 }
748 if (this.identifier != other.identifier && (this.identifier == null || !this.identifier.equals(other.identifier))) {
749 return false;
750 }
751 return true;
752 }
753
754
755
756
757
758
759
760
761 @Override
762 public int compareTo(IdentifierMatch o) {
763 int conf = this.confidence.compareTo(o.confidence);
764 if (conf == 0) {
765 conf = this.evidenceConfidence.compareTo(o.evidenceConfidence);
766 if (conf == 0) {
767 conf = identifier.compareTo(o.identifier);
768 }
769 }
770 return conf;
771 }
772 }
773 }