1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.owasp.dependencycheck;
19
20 import org.owasp.dependencycheck.analyzer.AnalysisPhase;
21 import org.owasp.dependencycheck.analyzer.Analyzer;
22 import org.owasp.dependencycheck.analyzer.AnalyzerService;
23 import org.owasp.dependencycheck.analyzer.FileTypeAnalyzer;
24 import org.owasp.dependencycheck.data.nvdcve.ConnectionFactory;
25 import org.owasp.dependencycheck.data.nvdcve.CveDB;
26 import org.owasp.dependencycheck.data.nvdcve.DatabaseException;
27 import org.owasp.dependencycheck.data.update.CachedWebDataSource;
28 import org.owasp.dependencycheck.data.update.UpdateService;
29 import org.owasp.dependencycheck.data.update.exception.UpdateException;
30 import org.owasp.dependencycheck.dependency.Dependency;
31 import org.owasp.dependencycheck.exception.ExceptionCollection;
32 import org.owasp.dependencycheck.exception.InitializationException;
33 import org.owasp.dependencycheck.exception.NoDataException;
34 import org.owasp.dependencycheck.utils.InvalidSettingException;
35 import org.owasp.dependencycheck.utils.Settings;
36 import org.slf4j.Logger;
37 import org.slf4j.LoggerFactory;
38
39 import java.io.File;
40 import java.io.FileFilter;
41 import java.util.ArrayList;
42 import java.util.Collection;
43 import java.util.Collections;
44 import java.util.EnumMap;
45 import java.util.HashSet;
46 import java.util.Iterator;
47 import java.util.List;
48 import java.util.Map;
49 import java.util.Set;
50 import java.util.concurrent.CancellationException;
51 import java.util.concurrent.ExecutionException;
52 import java.util.concurrent.ExecutorService;
53 import java.util.concurrent.Executors;
54 import java.util.concurrent.Future;
55 import java.util.concurrent.TimeUnit;
56
57
58
59
60
61
62
63
64
65 public class Engine implements FileFilter {
66
67
68
69
70 private final List<Dependency> dependencies = Collections.synchronizedList(new ArrayList<Dependency>());
71
72
73
74 private final Map<AnalysisPhase, List<Analyzer>> analyzers = new EnumMap<AnalysisPhase, List<Analyzer>>(AnalysisPhase.class);
75
76
77
78
79 private final Set<FileTypeAnalyzer> fileTypeAnalyzers = new HashSet<FileTypeAnalyzer>();
80
81
82
83
84
85 private ClassLoader serviceClassLoader = Thread.currentThread().getContextClassLoader();
86
87
88
89 private static final Logger LOGGER = LoggerFactory.getLogger(Engine.class);
90
91
92
93
94
95
96
97 public Engine() throws DatabaseException {
98 initializeEngine();
99 }
100
101
102
103
104
105
106
107
108 public Engine(ClassLoader serviceClassLoader) throws DatabaseException {
109 this.serviceClassLoader = serviceClassLoader;
110 initializeEngine();
111 }
112
113
114
115
116
117
118
119
120 protected final void initializeEngine() throws DatabaseException {
121 ConnectionFactory.initialize();
122 loadAnalyzers();
123 }
124
125
126
127
128 public void cleanup() {
129 ConnectionFactory.cleanup();
130 }
131
132
133
134
135
136 private void loadAnalyzers() {
137 if (!analyzers.isEmpty()) {
138 return;
139 }
140 for (AnalysisPhase phase : AnalysisPhase.values()) {
141 analyzers.put(phase, new ArrayList<Analyzer>());
142 }
143
144 final AnalyzerService service = new AnalyzerService(serviceClassLoader);
145 final List<Analyzer> iterator = service.getAnalyzers();
146 for (Analyzer a : iterator) {
147 analyzers.get(a.getAnalysisPhase()).add(a);
148 if (a instanceof FileTypeAnalyzer) {
149 this.fileTypeAnalyzers.add((FileTypeAnalyzer) a);
150 }
151 }
152 }
153
154
155
156
157
158
159
160 public List<Analyzer> getAnalyzers(AnalysisPhase phase) {
161 return analyzers.get(phase);
162 }
163
164
165
166
167
168
169
170
171
172
173
174 public List<Dependency> getDependencies() {
175 return dependencies;
176 }
177
178
179
180
181
182
183 public void setDependencies(List<Dependency> dependencies) {
184 synchronized (this.dependencies) {
185 this.dependencies.clear();
186 this.dependencies.addAll(dependencies);
187 }
188 }
189
190
191
192
193
194
195
196
197
198
199 public List<Dependency> scan(String[] paths) {
200 return scan(paths, null);
201 }
202
203
204
205
206
207
208
209
210
211
212
213
214 public List<Dependency> scan(String[] paths, String projectReference) {
215 final List<Dependency> deps = new ArrayList<Dependency>();
216 for (String path : paths) {
217 final List<Dependency> d = scan(path, projectReference);
218 if (d != null) {
219 deps.addAll(d);
220 }
221 }
222 return deps;
223 }
224
225
226
227
228
229
230
231
232
233 public List<Dependency> scan(String path) {
234 return scan(path, null);
235 }
236
237
238
239
240
241
242
243
244
245
246
247
248 public List<Dependency> scan(String path, String projectReference) {
249 final File file = new File(path);
250 return scan(file, projectReference);
251 }
252
253
254
255
256
257
258
259
260
261
262 public List<Dependency> scan(File[] files) {
263 return scan(files, null);
264 }
265
266
267
268
269
270
271
272
273
274
275
276
277 public List<Dependency> scan(File[] files, String projectReference) {
278 final List<Dependency> deps = new ArrayList<Dependency>();
279 for (File file : files) {
280 final List<Dependency> d = scan(file, projectReference);
281 if (d != null) {
282 deps.addAll(d);
283 }
284 }
285 return deps;
286 }
287
288
289
290
291
292
293
294
295
296
297 public List<Dependency> scan(Collection<File> files) {
298 return scan(files, null);
299 }
300
301
302
303
304
305
306
307
308
309
310
311
312 public List<Dependency> scan(Collection<File> files, String projectReference) {
313 final List<Dependency> deps = new ArrayList<Dependency>();
314 for (File file : files) {
315 final List<Dependency> d = scan(file, projectReference);
316 if (d != null) {
317 deps.addAll(d);
318 }
319 }
320 return deps;
321 }
322
323
324
325
326
327
328
329
330
331
332 public List<Dependency> scan(File file) {
333 return scan(file, null);
334 }
335
336
337
338
339
340
341
342
343
344
345
346
347 public List<Dependency> scan(File file, String projectReference) {
348 if (file.exists()) {
349 if (file.isDirectory()) {
350 return scanDirectory(file, projectReference);
351 } else {
352 final Dependency d = scanFile(file, projectReference);
353 if (d != null) {
354 final List<Dependency> deps = new ArrayList<Dependency>();
355 deps.add(d);
356 return deps;
357 }
358 }
359 }
360 return null;
361 }
362
363
364
365
366
367
368
369
370 protected List<Dependency> scanDirectory(File dir) {
371 return scanDirectory(dir, null);
372 }
373
374
375
376
377
378
379
380
381
382
383
384 protected List<Dependency> scanDirectory(File dir, String projectReference) {
385 final File[] files = dir.listFiles();
386 final List<Dependency> deps = new ArrayList<Dependency>();
387 if (files != null) {
388 for (File f : files) {
389 if (f.isDirectory()) {
390 final List<Dependency> d = scanDirectory(f, projectReference);
391 if (d != null) {
392 deps.addAll(d);
393 }
394 } else {
395 final Dependency d = scanFile(f, projectReference);
396 deps.add(d);
397 }
398 }
399 }
400 return deps;
401 }
402
403
404
405
406
407
408
409
410 protected Dependency scanFile(File file) {
411 return scanFile(file, null);
412 }
413
414
415
416
417
418
419
420
421
422
423
424 protected Dependency scanFile(File file, String projectReference) {
425 Dependency dependency = null;
426 if (file.isFile()) {
427 if (accept(file)) {
428 dependency = new Dependency(file);
429 if (projectReference != null) {
430 dependency.addProjectReference(projectReference);
431 }
432 final String sha1 = dependency.getSha1sum();
433 boolean found = false;
434 synchronized (dependencies) {
435 if (sha1 != null) {
436 for (Dependency existing : dependencies) {
437 if (sha1.equals(existing.getSha1sum())) {
438 found = true;
439 if (projectReference != null) {
440 existing.addProjectReference(projectReference);
441 }
442 if (existing.getActualFilePath() != null && dependency.getActualFilePath() != null
443 && !existing.getActualFilePath().equals(dependency.getActualFilePath())) {
444 existing.addRelatedDependency(dependency);
445 } else {
446 dependency = existing;
447 }
448 break;
449 }
450 }
451 }
452 if (!found) {
453 dependencies.add(dependency);
454 }
455 }
456 } else {
457 LOGGER.debug("Path passed to scanFile(File) is not a file: {}. Skipping the file.", file);
458 }
459 }
460 return dependency;
461 }
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478 public void analyzeDependencies() throws ExceptionCollection {
479 final List<Throwable> exceptions = Collections.synchronizedList(new ArrayList<Throwable>());
480 boolean autoUpdate = true;
481 try {
482 autoUpdate = Settings.getBoolean(Settings.KEYS.AUTO_UPDATE);
483 } catch (InvalidSettingException ex) {
484 LOGGER.debug("Invalid setting for auto-update; using true.");
485 exceptions.add(ex);
486 }
487 if (autoUpdate) {
488 try {
489 doUpdates();
490 } catch (UpdateException ex) {
491 exceptions.add(ex);
492 LOGGER.warn("Unable to update Cached Web DataSource, using local "
493 + "data instead. Results may not include recent vulnerabilities.");
494 LOGGER.debug("Update Error", ex);
495 }
496 }
497
498
499 try {
500 ensureDataExists();
501 } catch (NoDataException ex) {
502 throwFatalExceptionCollection("Unable to continue dependency-check analysis.", ex, exceptions);
503 } catch (DatabaseException ex) {
504 throwFatalExceptionCollection("Unable to connect to the dependency-check database.", ex, exceptions);
505 }
506
507 LOGGER.debug("\n----------------------------------------------------\nBEGIN ANALYSIS\n----------------------------------------------------");
508 LOGGER.info("Analysis Started");
509 final long analysisStart = System.currentTimeMillis();
510
511
512 for (AnalysisPhase phase : AnalysisPhase.values()) {
513 final List<Analyzer> analyzerList = analyzers.get(phase);
514
515 for (final Analyzer analyzer : analyzerList) {
516 final long analyzerStart = System.currentTimeMillis();
517 try {
518 initializeAnalyzer(analyzer);
519 } catch (InitializationException ex) {
520 exceptions.add(ex);
521 continue;
522 }
523
524 executeAnalysisTasks(analyzer, exceptions);
525
526 final long analyzerDurationMillis = System.currentTimeMillis() - analyzerStart;
527 final long analyzerDurationSeconds = TimeUnit.MILLISECONDS.toSeconds(analyzerDurationMillis);
528 LOGGER.info("Finished {} ({} seconds)", analyzer.getName(), analyzerDurationSeconds);
529 }
530 }
531 for (AnalysisPhase phase : AnalysisPhase.values()) {
532 final List<Analyzer> analyzerList = analyzers.get(phase);
533
534 for (Analyzer a : analyzerList) {
535 closeAnalyzer(a);
536 }
537 }
538
539 LOGGER.debug("\n----------------------------------------------------\nEND ANALYSIS\n----------------------------------------------------");
540 final long analysisDurationSeconds = TimeUnit.MILLISECONDS.toSeconds(System.currentTimeMillis() - analysisStart);
541 LOGGER.info("Analysis Complete ({} seconds)", analysisDurationSeconds);
542 if (exceptions.size() > 0) {
543 throw new ExceptionCollection("One or more exceptions occurred during dependency-check analysis", exceptions);
544 }
545 }
546
547
548
549
550
551
552
553
554
555 void executeAnalysisTasks(Analyzer analyzer, List<Throwable> exceptions) throws ExceptionCollection {
556 LOGGER.debug("Starting {}", analyzer.getName());
557 final List<AnalysisTask> analysisTasks = getAnalysisTasks(analyzer, exceptions);
558 final ExecutorService executorService = getExecutorService(analyzer);
559
560 try {
561 final List<Future<Void>> results = executorService.invokeAll(analysisTasks, 10, TimeUnit.MINUTES);
562
563
564 for (Future<Void> result : results) {
565 try {
566 result.get();
567 } catch (ExecutionException e) {
568 throwFatalExceptionCollection("Analysis task failed with a fatal exception.", e, exceptions);
569 } catch (CancellationException e) {
570 throwFatalExceptionCollection("Analysis task timed out.", e, exceptions);
571 }
572 }
573 } catch (InterruptedException e) {
574 throwFatalExceptionCollection("Analysis has been interrupted.", e, exceptions);
575 } finally {
576 executorService.shutdown();
577 }
578 }
579
580
581
582
583
584
585
586
587 List<AnalysisTask> getAnalysisTasks(Analyzer analyzer, List<Throwable> exceptions) {
588 final List<AnalysisTask> result = new ArrayList<AnalysisTask>();
589 synchronized (dependencies) {
590 for (final Dependency dependency : dependencies) {
591 final AnalysisTask task = new AnalysisTask(analyzer, dependency, this, exceptions);
592 result.add(task);
593 }
594 }
595 return result;
596 }
597
598
599
600
601
602
603
604 ExecutorService getExecutorService(Analyzer analyzer) {
605 if (analyzer.supportsParallelProcessing()) {
606
607 final int maximumNumberOfThreads = 4 * Runtime.getRuntime().availableProcessors();
608
609 LOGGER.debug("Parallel processing with up to {} threads: {}.", maximumNumberOfThreads, analyzer.getName());
610 return Executors.newFixedThreadPool(maximumNumberOfThreads);
611 } else {
612 LOGGER.debug("Parallel processing is not supported: {}.", analyzer.getName());
613 return Executors.newSingleThreadExecutor();
614 }
615 }
616
617
618
619
620
621
622
623
624
625 protected Analyzer initializeAnalyzer(Analyzer analyzer) throws InitializationException {
626 try {
627 LOGGER.debug("Initializing {}", analyzer.getName());
628 analyzer.initialize();
629 } catch (InitializationException ex) {
630 LOGGER.error("Exception occurred initializing {}.", analyzer.getName());
631 LOGGER.debug("", ex);
632 try {
633 analyzer.close();
634 } catch (Throwable ex1) {
635 LOGGER.trace("", ex1);
636 }
637 throw ex;
638 } catch (Throwable ex) {
639 LOGGER.error("Unexpected exception occurred initializing {}.", analyzer.getName());
640 LOGGER.debug("", ex);
641 try {
642 analyzer.close();
643 } catch (Throwable ex1) {
644 LOGGER.trace("", ex1);
645 }
646 throw new InitializationException("Unexpected Exception", ex);
647 }
648 return analyzer;
649 }
650
651
652
653
654
655
656 protected void closeAnalyzer(Analyzer analyzer) {
657 LOGGER.debug("Closing Analyzer '{}'", analyzer.getName());
658 try {
659 analyzer.close();
660 } catch (Throwable ex) {
661 LOGGER.trace("", ex);
662 }
663 }
664
665
666
667
668
669
670
671 public void doUpdates() throws UpdateException {
672 LOGGER.info("Checking for updates");
673 final long updateStart = System.currentTimeMillis();
674 final UpdateService service = new UpdateService(serviceClassLoader);
675 final Iterator<CachedWebDataSource> iterator = service.getDataSources();
676 while (iterator.hasNext()) {
677 final CachedWebDataSource source = iterator.next();
678 source.update();
679 }
680 LOGGER.info("Check for updates complete ({} ms)", System.currentTimeMillis() - updateStart);
681 }
682
683
684
685
686
687
688
689 public List<Analyzer> getAnalyzers() {
690 final List<Analyzer> ret = new ArrayList<Analyzer>();
691 for (AnalysisPhase phase : AnalysisPhase.values()) {
692 final List<Analyzer> analyzerList = analyzers.get(phase);
693 ret.addAll(analyzerList);
694 }
695 return ret;
696 }
697
698
699
700
701
702
703
704
705 @Override
706 public boolean accept(File file) {
707 if (file == null) {
708 return false;
709 }
710 boolean scan = false;
711 for (FileTypeAnalyzer a : this.fileTypeAnalyzers) {
712
713
714 scan |= a.accept(file);
715 }
716 return scan;
717 }
718
719
720
721
722
723
724 public Set<FileTypeAnalyzer> getFileTypeAnalyzers() {
725 return this.fileTypeAnalyzers;
726 }
727
728
729
730
731
732
733
734 protected void addFileTypeAnalyzer(FileTypeAnalyzer fta) {
735 this.fileTypeAnalyzers.add(fta);
736 }
737
738
739
740
741
742
743
744
745
746 private void ensureDataExists() throws NoDataException, DatabaseException {
747 final CveDB cve = new CveDB();
748 try {
749 cve.open();
750 if (!cve.dataExists()) {
751 throw new NoDataException("No documents exist");
752 }
753 } catch (DatabaseException ex) {
754 throw new NoDataException(ex.getMessage(), ex);
755 } finally {
756 cve.close();
757 }
758 }
759
760
761
762
763
764
765
766
767
768
769 private void throwFatalExceptionCollection(String message, Throwable throwable, List<Throwable> exceptions) throws ExceptionCollection {
770 LOGGER.error("{}\n\n{}", throwable.getMessage(), message);
771 LOGGER.debug("", throwable);
772 exceptions.add(throwable);
773 throw new ExceptionCollection(message, exceptions, true);
774 }
775 }