1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.owasp.dependencycheck;
19
20 import org.owasp.dependencycheck.analyzer.AnalysisPhase;
21 import org.owasp.dependencycheck.analyzer.Analyzer;
22 import org.owasp.dependencycheck.analyzer.AnalyzerService;
23 import org.owasp.dependencycheck.analyzer.FileTypeAnalyzer;
24 import org.owasp.dependencycheck.data.nvdcve.ConnectionFactory;
25 import org.owasp.dependencycheck.data.nvdcve.CveDB;
26 import org.owasp.dependencycheck.data.nvdcve.DatabaseException;
27 import org.owasp.dependencycheck.data.update.CachedWebDataSource;
28 import org.owasp.dependencycheck.data.update.UpdateService;
29 import org.owasp.dependencycheck.data.update.exception.UpdateException;
30 import org.owasp.dependencycheck.dependency.Dependency;
31 import org.owasp.dependencycheck.exception.ExceptionCollection;
32 import org.owasp.dependencycheck.exception.InitializationException;
33 import org.owasp.dependencycheck.exception.NoDataException;
34 import org.owasp.dependencycheck.utils.InvalidSettingException;
35 import org.owasp.dependencycheck.utils.Settings;
36 import org.slf4j.Logger;
37 import org.slf4j.LoggerFactory;
38
39 import java.io.File;
40 import java.io.FileFilter;
41 import java.util.ArrayList;
42 import java.util.Collection;
43 import java.util.Collections;
44 import java.util.EnumMap;
45 import java.util.HashSet;
46 import java.util.Iterator;
47 import java.util.List;
48 import java.util.Map;
49 import java.util.Set;
50 import java.util.concurrent.CancellationException;
51 import java.util.concurrent.ExecutionException;
52 import java.util.concurrent.ExecutorService;
53 import java.util.concurrent.Executors;
54 import java.util.concurrent.Future;
55 import java.util.concurrent.TimeUnit;
56
57
58
59
60
61
62
63
64
65 public class Engine implements FileFilter {
66
67
68
69
70 private final List<Dependency> dependencies = Collections.synchronizedList(new ArrayList<Dependency>());
71
72
73
74 private final Map<AnalysisPhase, List<Analyzer>> analyzers = new EnumMap<AnalysisPhase, List<Analyzer>>(AnalysisPhase.class);
75
76
77
78
79 private final Set<FileTypeAnalyzer> fileTypeAnalyzers = new HashSet<FileTypeAnalyzer>();
80
81
82
83
84
85 private ClassLoader serviceClassLoader = Thread.currentThread().getContextClassLoader();
86
87
88
89 private static final Logger LOGGER = LoggerFactory.getLogger(Engine.class);
90
91
92
93
94
95
96
97 public Engine() throws DatabaseException {
98 initializeEngine();
99 }
100
101
102
103
104
105
106
107
108 public Engine(ClassLoader serviceClassLoader) throws DatabaseException {
109 this.serviceClassLoader = serviceClassLoader;
110 initializeEngine();
111 }
112
113
114
115
116
117
118
119
120 protected final void initializeEngine() throws DatabaseException {
121 ConnectionFactory.initialize();
122 loadAnalyzers();
123 }
124
125
126
127
128 public void cleanup() {
129 ConnectionFactory.cleanup();
130 }
131
132
133
134
135
136 private void loadAnalyzers() {
137 if (!analyzers.isEmpty()) {
138 return;
139 }
140 for (AnalysisPhase phase : AnalysisPhase.values()) {
141 analyzers.put(phase, new ArrayList<Analyzer>());
142 }
143
144 final AnalyzerService service = new AnalyzerService(serviceClassLoader);
145 final List<Analyzer> iterator = service.getAnalyzers();
146 for (Analyzer a : iterator) {
147 analyzers.get(a.getAnalysisPhase()).add(a);
148 if (a instanceof FileTypeAnalyzer) {
149 this.fileTypeAnalyzers.add((FileTypeAnalyzer) a);
150 }
151 }
152 }
153
154
155
156
157
158
159
160 public List<Analyzer> getAnalyzers(AnalysisPhase phase) {
161 return analyzers.get(phase);
162 }
163
164
165
166
167
168
169
170
171
172
173
174
175 public synchronized List<Dependency> getDependencies() {
176 return dependencies;
177 }
178
179
180
181
182
183
184 public void setDependencies(List<Dependency> dependencies) {
185 synchronized (this.dependencies) {
186 this.dependencies.clear();
187 this.dependencies.addAll(dependencies);
188 }
189 }
190
191
192
193
194
195
196
197
198
199
200 public List<Dependency> scan(String[] paths) {
201 return scan(paths, null);
202 }
203
204
205
206
207
208
209
210
211
212
213
214
215 public List<Dependency> scan(String[] paths, String projectReference) {
216 final List<Dependency> deps = new ArrayList<Dependency>();
217 for (String path : paths) {
218 final List<Dependency> d = scan(path, projectReference);
219 if (d != null) {
220 deps.addAll(d);
221 }
222 }
223 return deps;
224 }
225
226
227
228
229
230
231
232
233
234 public List<Dependency> scan(String path) {
235 return scan(path, null);
236 }
237
238
239
240
241
242
243
244
245
246
247
248
249 public List<Dependency> scan(String path, String projectReference) {
250 final File file = new File(path);
251 return scan(file, projectReference);
252 }
253
254
255
256
257
258
259
260
261
262
263 public List<Dependency> scan(File[] files) {
264 return scan(files, null);
265 }
266
267
268
269
270
271
272
273
274
275
276
277
278 public List<Dependency> scan(File[] files, String projectReference) {
279 final List<Dependency> deps = new ArrayList<Dependency>();
280 for (File file : files) {
281 final List<Dependency> d = scan(file, projectReference);
282 if (d != null) {
283 deps.addAll(d);
284 }
285 }
286 return deps;
287 }
288
289
290
291
292
293
294
295
296
297
298 public List<Dependency> scan(Collection<File> files) {
299 return scan(files, null);
300 }
301
302
303
304
305
306
307
308
309
310
311
312
313 public List<Dependency> scan(Collection<File> files, String projectReference) {
314 final List<Dependency> deps = new ArrayList<Dependency>();
315 for (File file : files) {
316 final List<Dependency> d = scan(file, projectReference);
317 if (d != null) {
318 deps.addAll(d);
319 }
320 }
321 return deps;
322 }
323
324
325
326
327
328
329
330
331
332
333 public List<Dependency> scan(File file) {
334 return scan(file, null);
335 }
336
337
338
339
340
341
342
343
344
345
346
347
348 public List<Dependency> scan(File file, String projectReference) {
349 if (file.exists()) {
350 if (file.isDirectory()) {
351 return scanDirectory(file, projectReference);
352 } else {
353 final Dependency d = scanFile(file, projectReference);
354 if (d != null) {
355 final List<Dependency> deps = new ArrayList<Dependency>();
356 deps.add(d);
357 return deps;
358 }
359 }
360 }
361 return null;
362 }
363
364
365
366
367
368
369
370
371 protected List<Dependency> scanDirectory(File dir) {
372 return scanDirectory(dir, null);
373 }
374
375
376
377
378
379
380
381
382
383
384
385 protected List<Dependency> scanDirectory(File dir, String projectReference) {
386 final File[] files = dir.listFiles();
387 final List<Dependency> deps = new ArrayList<Dependency>();
388 if (files != null) {
389 for (File f : files) {
390 if (f.isDirectory()) {
391 final List<Dependency> d = scanDirectory(f, projectReference);
392 if (d != null) {
393 deps.addAll(d);
394 }
395 } else {
396 final Dependency d = scanFile(f, projectReference);
397 deps.add(d);
398 }
399 }
400 }
401 return deps;
402 }
403
404
405
406
407
408
409
410
411 protected Dependency scanFile(File file) {
412 return scanFile(file, null);
413 }
414
415
416
417
418
419
420
421
422
423
424
425 protected Dependency scanFile(File file, String projectReference) {
426 Dependency dependency = null;
427 if (file.isFile()) {
428 if (accept(file)) {
429 dependency = new Dependency(file);
430 if (projectReference != null) {
431 dependency.addProjectReference(projectReference);
432 }
433 final String sha1 = dependency.getSha1sum();
434 boolean found = false;
435 synchronized (dependencies) {
436 if (sha1 != null) {
437 for (Dependency existing : dependencies) {
438 if (sha1.equals(existing.getSha1sum())) {
439 found = true;
440 if (projectReference != null) {
441 existing.addProjectReference(projectReference);
442 }
443 if (existing.getActualFilePath() != null && dependency.getActualFilePath() != null
444 && !existing.getActualFilePath().equals(dependency.getActualFilePath())) {
445 existing.addRelatedDependency(dependency);
446 } else {
447 dependency = existing;
448 }
449 break;
450 }
451 }
452 }
453 if (!found) {
454 dependencies.add(dependency);
455 }
456 }
457 } else {
458 LOGGER.debug("Path passed to scanFile(File) is not a file: {}. Skipping the file.", file);
459 }
460 }
461 return dependency;
462 }
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479 public void analyzeDependencies() throws ExceptionCollection {
480 final List<Throwable> exceptions = Collections.synchronizedList(new ArrayList<Throwable>());
481 boolean autoUpdate = true;
482 try {
483 autoUpdate = Settings.getBoolean(Settings.KEYS.AUTO_UPDATE);
484 } catch (InvalidSettingException ex) {
485 LOGGER.debug("Invalid setting for auto-update; using true.");
486 exceptions.add(ex);
487 }
488 if (autoUpdate) {
489 try {
490 doUpdates();
491 } catch (UpdateException ex) {
492 exceptions.add(ex);
493 LOGGER.warn("Unable to update Cached Web DataSource, using local "
494 + "data instead. Results may not include recent vulnerabilities.");
495 LOGGER.debug("Update Error", ex);
496 }
497 }
498
499
500 try {
501 ensureDataExists();
502 } catch (NoDataException ex) {
503 throwFatalExceptionCollection("Unable to continue dependency-check analysis.", ex, exceptions);
504 } catch (DatabaseException ex) {
505 throwFatalExceptionCollection("Unable to connect to the dependency-check database.", ex, exceptions);
506 }
507
508 LOGGER.debug("\n----------------------------------------------------\nBEGIN ANALYSIS\n----------------------------------------------------");
509 LOGGER.info("Analysis Started");
510 final long analysisStart = System.currentTimeMillis();
511
512
513 for (AnalysisPhase phase : AnalysisPhase.values()) {
514 final List<Analyzer> analyzerList = analyzers.get(phase);
515
516 for (final Analyzer analyzer : analyzerList) {
517 final long analyzerStart = System.currentTimeMillis();
518 try {
519 initializeAnalyzer(analyzer);
520 } catch (InitializationException ex) {
521 exceptions.add(ex);
522 continue;
523 }
524
525 if (analyzer.isEnabled()) {
526 executeAnalysisTasks(analyzer, exceptions);
527
528 final long analyzerDurationMillis = System.currentTimeMillis() - analyzerStart;
529 final long analyzerDurationSeconds = TimeUnit.MILLISECONDS.toSeconds(analyzerDurationMillis);
530 LOGGER.info("Finished {} ({} seconds)", analyzer.getName(), analyzerDurationSeconds);
531 } else {
532 LOGGER.debug("Skipping {} (not enabled)", analyzer.getName());
533 }
534 }
535 }
536 for (AnalysisPhase phase : AnalysisPhase.values()) {
537 final List<Analyzer> analyzerList = analyzers.get(phase);
538
539 for (Analyzer a : analyzerList) {
540 closeAnalyzer(a);
541 }
542 }
543
544 LOGGER.debug("\n----------------------------------------------------\nEND ANALYSIS\n----------------------------------------------------");
545 final long analysisDurationSeconds = TimeUnit.MILLISECONDS.toSeconds(System.currentTimeMillis() - analysisStart);
546 LOGGER.info("Analysis Complete ({} seconds)", analysisDurationSeconds);
547 if (exceptions.size() > 0) {
548 throw new ExceptionCollection("One or more exceptions occurred during dependency-check analysis", exceptions);
549 }
550 }
551
552
553
554
555
556
557
558
559
560 void executeAnalysisTasks(Analyzer analyzer, List<Throwable> exceptions) throws ExceptionCollection {
561 LOGGER.debug("Starting {}", analyzer.getName());
562 final List<AnalysisTask> analysisTasks = getAnalysisTasks(analyzer, exceptions);
563 final ExecutorService executorService = getExecutorService(analyzer);
564
565 try {
566 final List<Future<Void>> results = executorService.invokeAll(analysisTasks, 10, TimeUnit.MINUTES);
567
568
569 for (Future<Void> result : results) {
570 try {
571 result.get();
572 } catch (ExecutionException e) {
573 throwFatalExceptionCollection("Analysis task failed with a fatal exception.", e, exceptions);
574 } catch (CancellationException e) {
575 throwFatalExceptionCollection("Analysis task timed out.", e, exceptions);
576 }
577 }
578 } catch (InterruptedException e) {
579 throwFatalExceptionCollection("Analysis has been interrupted.", e, exceptions);
580 } finally {
581 executorService.shutdown();
582 }
583 }
584
585
586
587
588
589
590
591
592 List<AnalysisTask> getAnalysisTasks(Analyzer analyzer, List<Throwable> exceptions) {
593 final List<AnalysisTask> result = new ArrayList<AnalysisTask>();
594 synchronized (dependencies) {
595 for (final Dependency dependency : dependencies) {
596 final AnalysisTask task = new AnalysisTask(analyzer, dependency, this, exceptions, Settings.getInstance());
597 result.add(task);
598 }
599 }
600 return result;
601 }
602
603
604
605
606
607
608
609 ExecutorService getExecutorService(Analyzer analyzer) {
610 if (analyzer.supportsParallelProcessing()) {
611
612 final int maximumNumberOfThreads = 4 * Runtime.getRuntime().availableProcessors();
613
614 LOGGER.debug("Parallel processing with up to {} threads: {}.", maximumNumberOfThreads, analyzer.getName());
615 return Executors.newFixedThreadPool(maximumNumberOfThreads);
616 } else {
617 LOGGER.debug("Parallel processing is not supported: {}.", analyzer.getName());
618 return Executors.newSingleThreadExecutor();
619 }
620 }
621
622
623
624
625
626
627
628
629
630 protected Analyzer initializeAnalyzer(Analyzer analyzer) throws InitializationException {
631 try {
632 LOGGER.debug("Initializing {}", analyzer.getName());
633 analyzer.initialize();
634 } catch (InitializationException ex) {
635 LOGGER.error("Exception occurred initializing {}.", analyzer.getName());
636 LOGGER.debug("", ex);
637 try {
638 analyzer.close();
639 } catch (Throwable ex1) {
640 LOGGER.trace("", ex1);
641 }
642 throw ex;
643 } catch (Throwable ex) {
644 LOGGER.error("Unexpected exception occurred initializing {}.", analyzer.getName());
645 LOGGER.debug("", ex);
646 try {
647 analyzer.close();
648 } catch (Throwable ex1) {
649 LOGGER.trace("", ex1);
650 }
651 throw new InitializationException("Unexpected Exception", ex);
652 }
653 return analyzer;
654 }
655
656
657
658
659
660
661 protected void closeAnalyzer(Analyzer analyzer) {
662 LOGGER.debug("Closing Analyzer '{}'", analyzer.getName());
663 try {
664 analyzer.close();
665 } catch (Throwable ex) {
666 LOGGER.trace("", ex);
667 }
668 }
669
670
671
672
673
674
675
676 public void doUpdates() throws UpdateException {
677 LOGGER.info("Checking for updates");
678 final long updateStart = System.currentTimeMillis();
679 final UpdateService service = new UpdateService(serviceClassLoader);
680 final Iterator<CachedWebDataSource> iterator = service.getDataSources();
681 while (iterator.hasNext()) {
682 final CachedWebDataSource source = iterator.next();
683 source.update();
684 }
685 LOGGER.info("Check for updates complete ({} ms)", System.currentTimeMillis() - updateStart);
686 }
687
688
689
690
691
692
693
694 public List<Analyzer> getAnalyzers() {
695 final List<Analyzer> ret = new ArrayList<Analyzer>();
696 for (AnalysisPhase phase : AnalysisPhase.values()) {
697 final List<Analyzer> analyzerList = analyzers.get(phase);
698 ret.addAll(analyzerList);
699 }
700 return ret;
701 }
702
703
704
705
706
707
708
709
710 @Override
711 public boolean accept(File file) {
712 if (file == null) {
713 return false;
714 }
715 boolean scan = false;
716 for (FileTypeAnalyzer a : this.fileTypeAnalyzers) {
717
718
719 scan |= a.accept(file);
720 }
721 return scan;
722 }
723
724
725
726
727
728
729 public Set<FileTypeAnalyzer> getFileTypeAnalyzers() {
730 return this.fileTypeAnalyzers;
731 }
732
733
734
735
736
737
738
739 protected void addFileTypeAnalyzer(FileTypeAnalyzer fta) {
740 this.fileTypeAnalyzers.add(fta);
741 }
742
743
744
745
746
747
748
749
750
751 private void ensureDataExists() throws NoDataException, DatabaseException {
752 final CveDB cve = new CveDB();
753 try {
754 cve.open();
755 if (!cve.dataExists()) {
756 throw new NoDataException("No documents exist");
757 }
758 } catch (DatabaseException ex) {
759 throw new NoDataException(ex.getMessage(), ex);
760 } finally {
761 cve.close();
762 }
763 }
764
765
766
767
768
769
770
771
772
773
774 private void throwFatalExceptionCollection(String message, Throwable throwable, List<Throwable> exceptions) throws ExceptionCollection {
775 LOGGER.error("{}\n\n{}", throwable.getMessage(), message);
776 LOGGER.debug("", throwable);
777 exceptions.add(throwable);
778 throw new ExceptionCollection(message, exceptions, true);
779 }
780 }