1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.owasp.dependencycheck.analyzer;
19
20 import java.io.IOException;
21 import java.io.UnsupportedEncodingException;
22 import java.net.URLEncoder;
23 import java.util.ArrayList;
24 import java.util.Collections;
25 import java.util.List;
26 import java.util.Set;
27 import java.util.StringTokenizer;
28 import java.util.concurrent.TimeUnit;
29 import org.apache.commons.lang3.builder.CompareToBuilder;
30 import org.apache.lucene.document.Document;
31 import org.apache.lucene.index.CorruptIndexException;
32 import org.apache.lucene.queryparser.classic.ParseException;
33 import org.apache.lucene.search.ScoreDoc;
34 import org.apache.lucene.search.TopDocs;
35 import org.owasp.dependencycheck.Engine;
36 import org.owasp.dependencycheck.analyzer.exception.AnalysisException;
37 import org.owasp.dependencycheck.data.cpe.CpeMemoryIndex;
38 import org.owasp.dependencycheck.data.cpe.Fields;
39 import org.owasp.dependencycheck.data.cpe.IndexEntry;
40 import org.owasp.dependencycheck.data.cpe.IndexException;
41 import org.owasp.dependencycheck.data.lucene.LuceneUtils;
42 import org.owasp.dependencycheck.data.nvdcve.CveDB;
43 import org.owasp.dependencycheck.data.nvdcve.DatabaseException;
44 import org.owasp.dependencycheck.dependency.Confidence;
45 import org.owasp.dependencycheck.dependency.Dependency;
46 import org.owasp.dependencycheck.dependency.Evidence;
47 import org.owasp.dependencycheck.dependency.EvidenceCollection;
48 import org.owasp.dependencycheck.dependency.Identifier;
49 import org.owasp.dependencycheck.dependency.VulnerableSoftware;
50 import org.owasp.dependencycheck.exception.InitializationException;
51 import org.owasp.dependencycheck.utils.DependencyVersion;
52 import org.owasp.dependencycheck.utils.DependencyVersionUtil;
53 import org.owasp.dependencycheck.utils.Settings;
54 import org.slf4j.Logger;
55 import org.slf4j.LoggerFactory;
56
57
58
59
60
61
62
63
64 public class CPEAnalyzer extends AbstractAnalyzer {
65
66
67
68
69 private static final Logger LOGGER = LoggerFactory.getLogger(CPEAnalyzer.class);
70
71
72
73 static final int MAX_QUERY_RESULTS = 25;
74
75
76
77 static final String WEIGHTING_BOOST = "^5";
78
79
80
81
82 static final String CLEANSE_CHARACTER_RX = "[^A-Za-z0-9 ._-]";
83
84
85
86
87 static final String CLEANSE_NONALPHA_RX = "[^A-Za-z]*";
88
89
90
91
92 static final int STRING_BUILDER_BUFFER = 20;
93
94
95
96 private CpeMemoryIndex cpe;
97
98
99
100 private CveDB cve;
101
102
103
104
105 public static final String NVD_SEARCH_URL = "https://web.nvd.nist.gov/view/vuln/search-results?adv_search=true&cves=on&cpe_version=%s";
106
107
108
109
110
111
112 @Override
113 public String getName() {
114 return "CPE Analyzer";
115 }
116
117
118
119
120
121
122 @Override
123 public AnalysisPhase getAnalysisPhase() {
124 return AnalysisPhase.IDENTIFIER_ANALYSIS;
125 }
126
127
128
129
130 @Override
131 public boolean supportsParallelProcessing() {
132 return false;
133 }
134
135
136
137
138
139
140 @Override
141 public void initializeAnalyzer() throws InitializationException {
142 try {
143 this.open();
144 } catch (IOException ex) {
145 LOGGER.debug("Exception initializing the Lucene Index", ex);
146 throw new InitializationException("An exception occurred initializing the Lucene Index", ex);
147 } catch (DatabaseException ex) {
148 LOGGER.debug("Exception accessing the database", ex);
149 throw new InitializationException("An exception occurred accessing the database", ex);
150 }
151 }
152
153
154
155
156
157
158
159
160
161 public void open() throws IOException, DatabaseException {
162 if (!isOpen()) {
163 cve = new CveDB();
164 cve.open();
165 cpe = CpeMemoryIndex.getInstance();
166 try {
167 final long creationStart = System.currentTimeMillis();
168 cpe.open(cve);
169 final long creationSeconds = TimeUnit.MILLISECONDS.toSeconds(System.currentTimeMillis() - creationStart);
170 LOGGER.info("Created CPE Index ({} seconds)", creationSeconds);
171 } catch (IndexException ex) {
172 LOGGER.debug("IndexException", ex);
173 throw new DatabaseException(ex);
174 }
175 }
176 }
177
178
179
180
181 @Override
182 public void closeAnalyzer() {
183 if (cpe != null) {
184 cpe.close();
185 cpe = null;
186 }
187 if (cve != null) {
188 cve.close();
189 cve = null;
190 }
191 }
192
193 public boolean isOpen() {
194 return cpe != null && cpe.isOpen();
195 }
196
197
198
199
200
201
202
203
204
205
206
207 protected void determineCPE(Dependency dependency) throws CorruptIndexException, IOException, ParseException {
208
209 String vendors = "";
210 String products = "";
211 for (Confidence confidence : Confidence.values()) {
212 if (dependency.getVendorEvidence().contains(confidence)) {
213 vendors = addEvidenceWithoutDuplicateTerms(vendors, dependency.getVendorEvidence(), confidence);
214 LOGGER.debug("vendor search: {}", vendors);
215 }
216 if (dependency.getProductEvidence().contains(confidence)) {
217 products = addEvidenceWithoutDuplicateTerms(products, dependency.getProductEvidence(), confidence);
218 LOGGER.debug("product search: {}", products);
219 }
220 if (!vendors.isEmpty() && !products.isEmpty()) {
221 final List<IndexEntry> entries = searchCPE(vendors, products, dependency.getVendorEvidence().getWeighting(),
222 dependency.getProductEvidence().getWeighting());
223 if (entries == null) {
224 continue;
225 }
226 boolean identifierAdded = false;
227 for (IndexEntry e : entries) {
228 LOGGER.debug("Verifying entry: {}", e);
229 if (verifyEntry(e, dependency)) {
230 final String vendor = e.getVendor();
231 final String product = e.getProduct();
232 LOGGER.debug("identified vendor/product: {}/{}", vendor, product);
233 identifierAdded |= determineIdentifiers(dependency, vendor, product, confidence);
234 }
235 }
236 if (identifierAdded) {
237 break;
238 }
239 }
240 }
241 }
242
243
244
245
246
247
248
249
250
251
252
253
254 private String addEvidenceWithoutDuplicateTerms(final String text, final EvidenceCollection ec, Confidence confidenceFilter) {
255 final String txt = (text == null) ? "" : text;
256 final StringBuilder sb = new StringBuilder(txt.length() + (20 * ec.size()));
257 sb.append(' ').append(txt).append(' ');
258 for (Evidence e : ec.iterator(confidenceFilter)) {
259 String value = e.getValue();
260
261
262
263 if (value.startsWith("http://")) {
264 value = value.substring(7).replaceAll("\\.", " ");
265 }
266 if (value.startsWith("https://")) {
267 value = value.substring(8).replaceAll("\\.", " ");
268 }
269 if (sb.indexOf(" " + value + " ") < 0) {
270 sb.append(value).append(' ');
271 }
272 }
273 return sb.toString().trim();
274 }
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293 protected List<IndexEntry> searchCPE(String vendor, String product,
294 Set<String> vendorWeightings, Set<String> productWeightings) {
295
296 final List<IndexEntry> ret = new ArrayList<IndexEntry>(MAX_QUERY_RESULTS);
297
298 final String searchString = buildSearch(vendor, product, vendorWeightings, productWeightings);
299 if (searchString == null) {
300 return ret;
301 }
302 try {
303 final TopDocs docs = cpe.search(searchString, MAX_QUERY_RESULTS);
304 for (ScoreDoc d : docs.scoreDocs) {
305 if (d.score >= 0.08) {
306 final Document doc = cpe.getDocument(d.doc);
307 final IndexEntry entry = new IndexEntry();
308 entry.setVendor(doc.get(Fields.VENDOR));
309 entry.setProduct(doc.get(Fields.PRODUCT));
310 entry.setSearchScore(d.score);
311 if (!ret.contains(entry)) {
312 ret.add(entry);
313 }
314 }
315 }
316 return ret;
317 } catch (ParseException ex) {
318 LOGGER.warn("An error occurred querying the CPE data. See the log for more details.");
319 LOGGER.info("Unable to parse: {}", searchString, ex);
320 } catch (IOException ex) {
321 LOGGER.warn("An error occurred reading CPE data. See the log for more details.");
322 LOGGER.info("IO Error with search string: {}", searchString, ex);
323 }
324 return null;
325 }
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345 protected String buildSearch(String vendor, String product,
346 Set<String> vendorWeighting, Set<String> productWeightings) {
347 final String v = vendor;
348 final String p = product;
349 final StringBuilder sb = new StringBuilder(v.length() + p.length()
350 + Fields.PRODUCT.length() + Fields.VENDOR.length() + STRING_BUILDER_BUFFER);
351
352 if (!appendWeightedSearch(sb, Fields.PRODUCT, p, productWeightings)) {
353 return null;
354 }
355 sb.append(" AND ");
356 if (!appendWeightedSearch(sb, Fields.VENDOR, v, vendorWeighting)) {
357 return null;
358 }
359 return sb.toString();
360 }
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376 private boolean appendWeightedSearch(StringBuilder sb, String field, String searchText, Set<String> weightedText) {
377 sb.append(' ').append(field).append(":( ");
378
379 final String cleanText = cleanseText(searchText);
380
381 if (cleanText.isEmpty()) {
382 return false;
383 }
384
385 if (weightedText == null || weightedText.isEmpty()) {
386 LuceneUtils.appendEscapedLuceneQuery(sb, cleanText);
387 } else {
388 final StringTokenizer tokens = new StringTokenizer(cleanText);
389 while (tokens.hasMoreElements()) {
390 final String word = tokens.nextToken();
391 StringBuilder temp = null;
392 for (String weighted : weightedText) {
393 final String weightedStr = cleanseText(weighted);
394 if (equalsIgnoreCaseAndNonAlpha(word, weightedStr)) {
395 temp = new StringBuilder(word.length() + 2);
396 LuceneUtils.appendEscapedLuceneQuery(temp, word);
397 temp.append(WEIGHTING_BOOST);
398 if (!word.equalsIgnoreCase(weightedStr)) {
399 temp.append(' ');
400 LuceneUtils.appendEscapedLuceneQuery(temp, weightedStr);
401 temp.append(WEIGHTING_BOOST);
402 }
403 break;
404 }
405 }
406 sb.append(' ');
407 if (temp == null) {
408 LuceneUtils.appendEscapedLuceneQuery(sb, word);
409 } else {
410 sb.append(temp);
411 }
412 }
413 }
414 sb.append(" ) ");
415 return true;
416 }
417
418
419
420
421
422
423
424
425 private String cleanseText(String text) {
426 return text.replaceAll(CLEANSE_CHARACTER_RX, " ");
427 }
428
429
430
431
432
433
434
435
436
437 private boolean equalsIgnoreCaseAndNonAlpha(String l, String r) {
438 if (l == null || r == null) {
439 return false;
440 }
441
442 final String left = l.replaceAll(CLEANSE_NONALPHA_RX, "");
443 final String right = r.replaceAll(CLEANSE_NONALPHA_RX, "");
444 return left.equalsIgnoreCase(right);
445 }
446
447
448
449
450
451
452
453
454
455
456 private boolean verifyEntry(final IndexEntry entry, final Dependency dependency) {
457 boolean isValid = false;
458
459
460
461 if (collectionContainsString(dependency.getProductEvidence(), entry.getProduct())
462 && collectionContainsString(dependency.getVendorEvidence(), entry.getVendor())) {
463
464 isValid = true;
465 }
466 return isValid;
467 }
468
469
470
471
472
473
474
475
476 private boolean collectionContainsString(EvidenceCollection ec, String text) {
477
478 if (text == null) {
479 return false;
480 }
481 final String[] words = text.split("[\\s_-]");
482 final List<String> list = new ArrayList<String>();
483 String tempWord = null;
484 for (String word : words) {
485
486
487
488
489 if (tempWord != null) {
490 list.add(tempWord + word);
491 tempWord = null;
492 } else if (word.length() <= 2) {
493 tempWord = word;
494 } else {
495 list.add(word);
496 }
497 }
498 if (tempWord != null) {
499 if (!list.isEmpty()) {
500 final String tmp = list.get(list.size() - 1) + tempWord;
501 list.add(tmp);
502 } else {
503 list.add(tempWord);
504 }
505 }
506 if (list.isEmpty()) {
507 return false;
508 }
509 boolean contains = true;
510 for (String word : list) {
511 contains &= ec.containsUsedString(word);
512 }
513 return contains;
514 }
515
516
517
518
519
520
521
522
523
524
525 @Override
526 protected synchronized void analyzeDependency(Dependency dependency, Engine engine) throws AnalysisException {
527 try {
528 determineCPE(dependency);
529 } catch (CorruptIndexException ex) {
530 throw new AnalysisException("CPE Index is corrupt.", ex);
531 } catch (IOException ex) {
532 throw new AnalysisException("Failure opening the CPE Index.", ex);
533 } catch (ParseException ex) {
534 throw new AnalysisException("Unable to parse the generated Lucene query for this dependency.", ex);
535 }
536 }
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554 protected boolean determineIdentifiers(Dependency dependency, String vendor, String product,
555 Confidence currentConfidence) throws UnsupportedEncodingException {
556 final Set<VulnerableSoftware> cpes = cve.getCPEs(vendor, product);
557 DependencyVersion bestGuess = new DependencyVersion("-");
558 Confidence bestGuessConf = null;
559 boolean hasBroadMatch = false;
560 final List<IdentifierMatch> collected = new ArrayList<IdentifierMatch>();
561
562
563
564
565 for (Confidence conf : Confidence.values()) {
566 for (Evidence evidence : dependency.getVersionEvidence().iterator(conf)) {
567 final DependencyVersion evVer = DependencyVersionUtil.parseVersion(evidence.getValue());
568 if (evVer == null) {
569 continue;
570 }
571 for (VulnerableSoftware vs : cpes) {
572 DependencyVersion dbVer;
573 if (vs.getUpdate() != null && !vs.getUpdate().isEmpty()) {
574 dbVer = DependencyVersionUtil.parseVersion(vs.getVersion() + '.' + vs.getUpdate());
575 } else {
576 dbVer = DependencyVersionUtil.parseVersion(vs.getVersion());
577 }
578 if (dbVer == null) {
579 hasBroadMatch = true;
580 final String url = String.format(NVD_SEARCH_URL, URLEncoder.encode(vs.getName(), "UTF-8"));
581 final IdentifierMatch match = new IdentifierMatch("cpe", vs.getName(), url, IdentifierConfidence.BROAD_MATCH, conf);
582 collected.add(match);
583 } else if (evVer.equals(dbVer)) {
584 final String url = String.format(NVD_SEARCH_URL, URLEncoder.encode(vs.getName(), "UTF-8"));
585 final IdentifierMatch match = new IdentifierMatch("cpe", vs.getName(), url, IdentifierConfidence.EXACT_MATCH, conf);
586 collected.add(match);
587
588
589 } else if (evVer.getVersionParts().size() <= dbVer.getVersionParts().size()
590 && evVer.matchesAtLeastThreeLevels(dbVer)) {
591 if (bestGuessConf == null || bestGuessConf.compareTo(conf) > 0) {
592 if (bestGuess.getVersionParts().size() < dbVer.getVersionParts().size()) {
593 bestGuess = dbVer;
594 bestGuessConf = conf;
595 }
596 }
597 }
598 }
599 if (bestGuessConf == null || bestGuessConf.compareTo(conf) > 0) {
600 if (bestGuess.getVersionParts().size() < evVer.getVersionParts().size()) {
601 bestGuess = evVer;
602 bestGuessConf = conf;
603 }
604 }
605 }
606 }
607 final String cpeName = String.format("cpe:/a:%s:%s:%s", vendor, product, bestGuess.toString());
608 String url = null;
609 if (hasBroadMatch) {
610 final String cpeUrlName = String.format("cpe:/a:%s:%s", vendor, product);
611 url = String.format(NVD_SEARCH_URL, URLEncoder.encode(cpeUrlName, "UTF-8"));
612 }
613 if (bestGuessConf == null) {
614 bestGuessConf = Confidence.LOW;
615 }
616 final IdentifierMatch match = new IdentifierMatch("cpe", cpeName, url, IdentifierConfidence.BEST_GUESS, bestGuessConf);
617 collected.add(match);
618
619 Collections.sort(collected);
620 final IdentifierConfidence bestIdentifierQuality = collected.get(0).getConfidence();
621 final Confidence bestEvidenceQuality = collected.get(0).getEvidenceConfidence();
622 boolean identifierAdded = false;
623 for (IdentifierMatch m : collected) {
624 if (bestIdentifierQuality.equals(m.getConfidence())
625 && bestEvidenceQuality.equals(m.getEvidenceConfidence())) {
626 final Identifier i = m.getIdentifier();
627 if (bestIdentifierQuality == IdentifierConfidence.BEST_GUESS) {
628 i.setConfidence(Confidence.LOW);
629 } else {
630 i.setConfidence(bestEvidenceQuality);
631 }
632 dependency.addIdentifier(i);
633 identifierAdded = true;
634 }
635 }
636 return identifierAdded;
637 }
638
639
640
641
642
643
644
645 @Override
646 protected String getAnalyzerEnabledSettingKey() {
647 return Settings.KEYS.ANALYZER_CPE_ENABLED;
648 }
649
650
651
652
653 private enum IdentifierConfidence {
654
655
656
657
658 EXACT_MATCH,
659
660
661
662 BEST_GUESS,
663
664
665
666
667
668 BROAD_MATCH
669 }
670
671
672
673
674
675 private static class IdentifierMatch implements Comparable<IdentifierMatch> {
676
677
678
679
680
681
682
683
684
685
686
687
688 IdentifierMatch(String type, String value, String url, IdentifierConfidence identifierConfidence, Confidence evidenceConfidence) {
689 this.identifier = new Identifier(type, value, url);
690 this.confidence = identifierConfidence;
691 this.evidenceConfidence = evidenceConfidence;
692 }
693
694
695
696
697 private Confidence evidenceConfidence;
698
699
700
701
702
703
704 public Confidence getEvidenceConfidence() {
705 return evidenceConfidence;
706 }
707
708
709
710
711
712
713 public void setEvidenceConfidence(Confidence evidenceConfidence) {
714 this.evidenceConfidence = evidenceConfidence;
715 }
716
717
718
719 private IdentifierConfidence confidence;
720
721
722
723
724
725
726 public IdentifierConfidence getConfidence() {
727 return confidence;
728 }
729
730
731
732
733
734
735 public void setConfidence(IdentifierConfidence confidence) {
736 this.confidence = confidence;
737 }
738
739
740
741 private Identifier identifier;
742
743
744
745
746
747
748 public Identifier getIdentifier() {
749 return identifier;
750 }
751
752
753
754
755
756
757 public void setIdentifier(Identifier identifier) {
758 this.identifier = identifier;
759 }
760
761
762
763
764
765
766
767
768 @Override
769 public String toString() {
770 return "IdentifierMatch{" + "evidenceConfidence=" + evidenceConfidence
771 + ", confidence=" + confidence + ", identifier=" + identifier + '}';
772 }
773
774
775
776
777
778
779 @Override
780 public int hashCode() {
781 int hash = 5;
782 hash = 97 * hash + (this.evidenceConfidence != null ? this.evidenceConfidence.hashCode() : 0);
783 hash = 97 * hash + (this.confidence != null ? this.confidence.hashCode() : 0);
784 hash = 97 * hash + (this.identifier != null ? this.identifier.hashCode() : 0);
785 return hash;
786 }
787
788
789
790
791
792
793
794 @Override
795 public boolean equals(Object obj) {
796 if (obj == null) {
797 return false;
798 }
799 if (getClass() != obj.getClass()) {
800 return false;
801 }
802 final IdentifierMatch other = (IdentifierMatch) obj;
803 if (this.evidenceConfidence != other.evidenceConfidence) {
804 return false;
805 }
806 if (this.confidence != other.confidence) {
807 return false;
808 }
809 if (this.identifier != other.identifier && (this.identifier == null || !this.identifier.equals(other.identifier))) {
810 return false;
811 }
812 return true;
813 }
814
815
816
817
818
819
820
821
822
823 @Override
824 public int compareTo(IdentifierMatch o) {
825 return new CompareToBuilder()
826 .append(confidence, o.confidence)
827 .append(evidenceConfidence, o.evidenceConfidence)
828 .append(identifier, o.identifier)
829 .toComparison();
830 }
831 }
832 }