Coverage Report - org.owasp.dependencycheck.Engine
 
Classes in this File Line Coverage Branch Coverage Complexity
Engine
63%
163/255
63%
53/84
2.914
 
 1  
 /*
 2  
  * This file is part of dependency-check-core.
 3  
  *
 4  
  * Licensed under the Apache License, Version 2.0 (the "License");
 5  
  * you may not use this file except in compliance with the License.
 6  
  * You may obtain a copy of the License at
 7  
  *
 8  
  *     http://www.apache.org/licenses/LICENSE-2.0
 9  
  *
 10  
  * Unless required by applicable law or agreed to in writing, software
 11  
  * distributed under the License is distributed on an "AS IS" BASIS,
 12  
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 13  
  * See the License for the specific language governing permissions and
 14  
  * limitations under the License.
 15  
  *
 16  
  * Copyright (c) 2012 Jeremy Long. All Rights Reserved.
 17  
  */
 18  
 package org.owasp.dependencycheck;
 19  
 
 20  
 import org.owasp.dependencycheck.analyzer.AnalysisPhase;
 21  
 import org.owasp.dependencycheck.analyzer.Analyzer;
 22  
 import org.owasp.dependencycheck.analyzer.AnalyzerService;
 23  
 import org.owasp.dependencycheck.analyzer.FileTypeAnalyzer;
 24  
 import org.owasp.dependencycheck.data.nvdcve.ConnectionFactory;
 25  
 import org.owasp.dependencycheck.data.nvdcve.CveDB;
 26  
 import org.owasp.dependencycheck.data.nvdcve.DatabaseException;
 27  
 import org.owasp.dependencycheck.data.update.CachedWebDataSource;
 28  
 import org.owasp.dependencycheck.data.update.UpdateService;
 29  
 import org.owasp.dependencycheck.data.update.exception.UpdateException;
 30  
 import org.owasp.dependencycheck.dependency.Dependency;
 31  
 import org.owasp.dependencycheck.exception.ExceptionCollection;
 32  
 import org.owasp.dependencycheck.exception.InitializationException;
 33  
 import org.owasp.dependencycheck.exception.NoDataException;
 34  
 import org.owasp.dependencycheck.utils.InvalidSettingException;
 35  
 import org.owasp.dependencycheck.utils.Settings;
 36  
 import org.slf4j.Logger;
 37  
 import org.slf4j.LoggerFactory;
 38  
 
 39  
 import java.io.File;
 40  
 import java.io.FileFilter;
 41  
 import java.util.ArrayList;
 42  
 import java.util.Collection;
 43  
 import java.util.Collections;
 44  
 import java.util.EnumMap;
 45  
 import java.util.HashSet;
 46  
 import java.util.Iterator;
 47  
 import java.util.List;
 48  
 import java.util.Map;
 49  
 import java.util.Set;
 50  
 import java.util.concurrent.CancellationException;
 51  
 import java.util.concurrent.ExecutionException;
 52  
 import java.util.concurrent.ExecutorService;
 53  
 import java.util.concurrent.Executors;
 54  
 import java.util.concurrent.Future;
 55  
 import java.util.concurrent.TimeUnit;
 56  
 
 57  
 /**
 58  
  * Scans files, directories, etc. for Dependencies. Analyzers are loaded and
 59  
  * used to process the files found by the scan, if a file is encountered and an
 60  
  * Analyzer is associated with the file type then the file is turned into a
 61  
  * dependency.
 62  
  *
 63  
  * @author Jeremy Long
 64  
  */
 65  
 public class Engine implements FileFilter {
 66  
 
 67  
     /**
 68  
      * The list of dependencies.
 69  
      */
 70  6
     private final List<Dependency> dependencies = Collections.synchronizedList(new ArrayList<Dependency>());
 71  
     /**
 72  
      * A Map of analyzers grouped by Analysis phase.
 73  
      */
 74  6
     private final Map<AnalysisPhase, List<Analyzer>> analyzers = new EnumMap<AnalysisPhase, List<Analyzer>>(AnalysisPhase.class);
 75  
 
 76  
     /**
 77  
      * A Map of analyzers grouped by Analysis phase.
 78  
      */
 79  6
     private final Set<FileTypeAnalyzer> fileTypeAnalyzers = new HashSet<FileTypeAnalyzer>();
 80  
 
 81  
     /**
 82  
      * The ClassLoader to use when dynamically loading Analyzer and Update
 83  
      * services.
 84  
      */
 85  6
     private ClassLoader serviceClassLoader = Thread.currentThread().getContextClassLoader();
 86  
     /**
 87  
      * The Logger for use throughout the class.
 88  
      */
 89  1
     private static final Logger LOGGER = LoggerFactory.getLogger(Engine.class);
 90  
 
 91  
     /**
 92  
      * Creates a new Engine.
 93  
      *
 94  
      * @throws DatabaseException thrown if there is an error connecting to the
 95  
      * database
 96  
      */
 97  6
     public Engine() throws DatabaseException {
 98  6
         initializeEngine();
 99  6
     }
 100  
 
 101  
     /**
 102  
      * Creates a new Engine.
 103  
      *
 104  
      * @param serviceClassLoader a reference the class loader being used
 105  
      * @throws DatabaseException thrown if there is an error connecting to the
 106  
      * database
 107  
      */
 108  0
     public Engine(ClassLoader serviceClassLoader) throws DatabaseException {
 109  0
         this.serviceClassLoader = serviceClassLoader;
 110  0
         initializeEngine();
 111  0
     }
 112  
 
 113  
     /**
 114  
      * Creates a new Engine using the specified classloader to dynamically load
 115  
      * Analyzer and Update services.
 116  
      *
 117  
      * @throws DatabaseException thrown if there is an error connecting to the
 118  
      * database
 119  
      */
 120  
     protected final void initializeEngine() throws DatabaseException {
 121  6
         ConnectionFactory.initialize();
 122  6
         loadAnalyzers();
 123  6
     }
 124  
 
 125  
     /**
 126  
      * Properly cleans up resources allocated during analysis.
 127  
      */
 128  
     public void cleanup() {
 129  0
         ConnectionFactory.cleanup();
 130  0
     }
 131  
 
 132  
     /**
 133  
      * Loads the analyzers specified in the configuration file (or system
 134  
      * properties).
 135  
      */
 136  
     private void loadAnalyzers() {
 137  6
         if (!analyzers.isEmpty()) {
 138  0
             return;
 139  
         }
 140  66
         for (AnalysisPhase phase : AnalysisPhase.values()) {
 141  60
             analyzers.put(phase, new ArrayList<Analyzer>());
 142  
         }
 143  
 
 144  6
         final AnalyzerService service = new AnalyzerService(serviceClassLoader);
 145  6
         final List<Analyzer> iterator = service.getAnalyzers();
 146  6
         for (Analyzer a : iterator) {
 147  156
             analyzers.get(a.getAnalysisPhase()).add(a);
 148  156
             if (a instanceof FileTypeAnalyzer) {
 149  108
                 this.fileTypeAnalyzers.add((FileTypeAnalyzer) a);
 150  
             }
 151  156
         }
 152  6
     }
 153  
 
 154  
     /**
 155  
      * Get the List of the analyzers for a specific phase of analysis.
 156  
      *
 157  
      * @param phase the phase to get the configured analyzers.
 158  
      * @return the analyzers loaded
 159  
      */
 160  
     public List<Analyzer> getAnalyzers(AnalysisPhase phase) {
 161  0
         return analyzers.get(phase);
 162  
     }
 163  
 
 164  
     /**
 165  
      * Get the dependencies identified. The returned list is a reference to the
 166  
      * engine's synchronized list. You must synchronize on it, when you modify
 167  
      * and iterate over it from multiple threads. E.g. this holds for analyzers
 168  
      * supporting parallel processing during their analysis phase.
 169  
      *
 170  
      * @return the dependencies identified
 171  
      * @see Collections#synchronizedList(List)
 172  
      * @see Analyzer#supportsParallelProcessing()
 173  
      */
 174  
     public List<Dependency> getDependencies() {
 175  46
         return dependencies;
 176  
     }
 177  
 
 178  
     /**
 179  
      * Sets the dependencies.
 180  
      *
 181  
      * @param dependencies the dependencies
 182  
      */
 183  
     public void setDependencies(List<Dependency> dependencies) {
 184  0
         synchronized (this.dependencies) {
 185  0
             this.dependencies.clear();
 186  0
             this.dependencies.addAll(dependencies);
 187  0
         }
 188  0
     }
 189  
 
 190  
     /**
 191  
      * Scans an array of files or directories. If a directory is specified, it
 192  
      * will be scanned recursively. Any dependencies identified are added to the
 193  
      * dependency collection.
 194  
      *
 195  
      * @param paths an array of paths to files or directories to be analyzed
 196  
      * @return the list of dependencies scanned
 197  
      * @since v0.3.2.5
 198  
      */
 199  
     public List<Dependency> scan(String[] paths) {
 200  0
         return scan(paths, null);
 201  
     }
 202  
 
 203  
     /**
 204  
      * Scans an array of files or directories. If a directory is specified, it
 205  
      * will be scanned recursively. Any dependencies identified are added to the
 206  
      * dependency collection.
 207  
      *
 208  
      * @param paths an array of paths to files or directories to be analyzed
 209  
      * @param projectReference the name of the project or scope in which the
 210  
      * dependency was identified
 211  
      * @return the list of dependencies scanned
 212  
      * @since v1.4.4
 213  
      */
 214  
     public List<Dependency> scan(String[] paths, String projectReference) {
 215  0
         final List<Dependency> deps = new ArrayList<Dependency>();
 216  0
         for (String path : paths) {
 217  0
             final List<Dependency> d = scan(path, projectReference);
 218  0
             if (d != null) {
 219  0
                 deps.addAll(d);
 220  
             }
 221  
         }
 222  0
         return deps;
 223  
     }
 224  
 
 225  
     /**
 226  
      * Scans a given file or directory. If a directory is specified, it will be
 227  
      * scanned recursively. Any dependencies identified are added to the
 228  
      * dependency collection.
 229  
      *
 230  
      * @param path the path to a file or directory to be analyzed
 231  
      * @return the list of dependencies scanned
 232  
      */
 233  
     public List<Dependency> scan(String path) {
 234  0
         return scan(path, null);
 235  
     }
 236  
 
 237  
     /**
 238  
      * Scans a given file or directory. If a directory is specified, it will be
 239  
      * scanned recursively. Any dependencies identified are added to the
 240  
      * dependency collection.
 241  
      *
 242  
      * @param path the path to a file or directory to be analyzed
 243  
      * @param projectReference the name of the project or scope in which the
 244  
      * dependency was identified
 245  
      * @return the list of dependencies scanned
 246  
      * @since v1.4.4
 247  
      */
 248  
     public List<Dependency> scan(String path, String projectReference) {
 249  0
         final File file = new File(path);
 250  0
         return scan(file, projectReference);
 251  
     }
 252  
 
 253  
     /**
 254  
      * Scans an array of files or directories. If a directory is specified, it
 255  
      * will be scanned recursively. Any dependencies identified are added to the
 256  
      * dependency collection.
 257  
      *
 258  
      * @param files an array of paths to files or directories to be analyzed.
 259  
      * @return the list of dependencies
 260  
      * @since v0.3.2.5
 261  
      */
 262  
     public List<Dependency> scan(File[] files) {
 263  0
         return scan(files, null);
 264  
     }
 265  
 
 266  
     /**
 267  
      * Scans an array of files or directories. If a directory is specified, it
 268  
      * will be scanned recursively. Any dependencies identified are added to the
 269  
      * dependency collection.
 270  
      *
 271  
      * @param files an array of paths to files or directories to be analyzed.
 272  
      * @param projectReference the name of the project or scope in which the
 273  
      * dependency was identified
 274  
      * @return the list of dependencies
 275  
      * @since v1.4.4
 276  
      */
 277  
     public List<Dependency> scan(File[] files, String projectReference) {
 278  0
         final List<Dependency> deps = new ArrayList<Dependency>();
 279  0
         for (File file : files) {
 280  0
             final List<Dependency> d = scan(file, projectReference);
 281  0
             if (d != null) {
 282  0
                 deps.addAll(d);
 283  
             }
 284  
         }
 285  0
         return deps;
 286  
     }
 287  
 
 288  
     /**
 289  
      * Scans a collection of files or directories. If a directory is specified,
 290  
      * it will be scanned recursively. Any dependencies identified are added to
 291  
      * the dependency collection.
 292  
      *
 293  
      * @param files a set of paths to files or directories to be analyzed
 294  
      * @return the list of dependencies scanned
 295  
      * @since v0.3.2.5
 296  
      */
 297  
     public List<Dependency> scan(Collection<File> files) {
 298  0
         return scan(files, null);
 299  
     }
 300  
 
 301  
     /**
 302  
      * Scans a collection of files or directories. If a directory is specified,
 303  
      * it will be scanned recursively. Any dependencies identified are added to
 304  
      * the dependency collection.
 305  
      *
 306  
      * @param files a set of paths to files or directories to be analyzed
 307  
      * @param projectReference the name of the project or scope in which the
 308  
      * dependency was identified
 309  
      * @return the list of dependencies scanned
 310  
      * @since v1.4.4
 311  
      */
 312  
     public List<Dependency> scan(Collection<File> files, String projectReference) {
 313  0
         final List<Dependency> deps = new ArrayList<Dependency>();
 314  0
         for (File file : files) {
 315  0
             final List<Dependency> d = scan(file, projectReference);
 316  0
             if (d != null) {
 317  0
                 deps.addAll(d);
 318  
             }
 319  0
         }
 320  0
         return deps;
 321  
     }
 322  
 
 323  
     /**
 324  
      * Scans a given file or directory. If a directory is specified, it will be
 325  
      * scanned recursively. Any dependencies identified are added to the
 326  
      * dependency collection.
 327  
      *
 328  
      * @param file the path to a file or directory to be analyzed
 329  
      * @return the list of dependencies scanned
 330  
      * @since v0.3.2.4
 331  
      */
 332  
     public List<Dependency> scan(File file) {
 333  5
         return scan(file, null);
 334  
     }
 335  
 
 336  
     /**
 337  
      * Scans a given file or directory. If a directory is specified, it will be
 338  
      * scanned recursively. Any dependencies identified are added to the
 339  
      * dependency collection.
 340  
      *
 341  
      * @param file the path to a file or directory to be analyzed
 342  
      * @param projectReference the name of the project or scope in which the
 343  
      * dependency was identified
 344  
      * @return the list of dependencies scanned
 345  
      * @since v1.4.4
 346  
      */
 347  
     public List<Dependency> scan(File file, String projectReference) {
 348  5
         if (file.exists()) {
 349  5
             if (file.isDirectory()) {
 350  3
                 return scanDirectory(file, projectReference);
 351  
             } else {
 352  2
                 final Dependency d = scanFile(file, projectReference);
 353  2
                 if (d != null) {
 354  2
                     final List<Dependency> deps = new ArrayList<Dependency>();
 355  2
                     deps.add(d);
 356  2
                     return deps;
 357  
                 }
 358  
             }
 359  
         }
 360  0
         return null;
 361  
     }
 362  
 
 363  
     /**
 364  
      * Recursively scans files and directories. Any dependencies identified are
 365  
      * added to the dependency collection.
 366  
      *
 367  
      * @param dir the directory to scan
 368  
      * @return the list of Dependency objects scanned
 369  
      */
 370  
     protected List<Dependency> scanDirectory(File dir) {
 371  0
         return scanDirectory(dir, null);
 372  
     }
 373  
 
 374  
     /**
 375  
      * Recursively scans files and directories. Any dependencies identified are
 376  
      * added to the dependency collection.
 377  
      *
 378  
      * @param dir the directory to scan
 379  
      * @param projectReference the name of the project or scope in which the
 380  
      * dependency was identified
 381  
      * @return the list of Dependency objects scanned
 382  
      * @since v1.4.4
 383  
      */
 384  
     protected List<Dependency> scanDirectory(File dir, String projectReference) {
 385  46
         final File[] files = dir.listFiles();
 386  46
         final List<Dependency> deps = new ArrayList<Dependency>();
 387  46
         if (files != null) {
 388  93
             for (File f : files) {
 389  47
                 if (f.isDirectory()) {
 390  43
                     final List<Dependency> d = scanDirectory(f, projectReference);
 391  43
                     if (d != null) {
 392  43
                         deps.addAll(d);
 393  
                     }
 394  43
                 } else {
 395  4
                     final Dependency d = scanFile(f, projectReference);
 396  4
                     deps.add(d);
 397  
                 }
 398  
             }
 399  
         }
 400  46
         return deps;
 401  
     }
 402  
 
 403  
     /**
 404  
      * Scans a specified file. If a dependency is identified it is added to the
 405  
      * dependency collection.
 406  
      *
 407  
      * @param file The file to scan
 408  
      * @return the scanned dependency
 409  
      */
 410  
     protected Dependency scanFile(File file) {
 411  3
         return scanFile(file, null);
 412  
     }
 413  
 
 414  
     /**
 415  
      * Scans a specified file. If a dependency is identified it is added to the
 416  
      * dependency collection.
 417  
      *
 418  
      * @param file The file to scan
 419  
      * @param projectReference the name of the project or scope in which the
 420  
      * dependency was identified
 421  
      * @return the scanned dependency
 422  
      * @since v1.4.4
 423  
      */
 424  
     protected Dependency scanFile(File file, String projectReference) {
 425  9
         Dependency dependency = null;
 426  9
         if (file.isFile()) {
 427  9
             if (accept(file)) {
 428  7
                 dependency = new Dependency(file);
 429  7
                 if (projectReference != null) {
 430  0
                     dependency.addProjectReference(projectReference);
 431  
                 }
 432  7
                 final String sha1 = dependency.getSha1sum();
 433  7
                 boolean found = false;
 434  7
                 synchronized (dependencies) {
 435  7
                     if (sha1 != null) {
 436  7
                         for (Dependency existing : dependencies) {
 437  4
                             if (sha1.equals(existing.getSha1sum())) {
 438  1
                                 found = true;
 439  1
                                 if (projectReference != null) {
 440  0
                                     existing.addProjectReference(projectReference);
 441  
                                 }
 442  1
                                 if (existing.getActualFilePath() != null && dependency.getActualFilePath() != null
 443  1
                                         && !existing.getActualFilePath().equals(dependency.getActualFilePath())) {
 444  0
                                     existing.addRelatedDependency(dependency);
 445  
                                 } else {
 446  1
                                     dependency = existing;
 447  
                                 }
 448  1
                                 break;
 449  
                             }
 450  3
                         }
 451  
                     }
 452  7
                     if (!found) {
 453  6
                         dependencies.add(dependency);
 454  
                     }
 455  7
                 }
 456  7
             } else {
 457  2
                 LOGGER.debug("Path passed to scanFile(File) is not a file: {}. Skipping the file.", file);
 458  
             }
 459  
         }
 460  9
         return dependency;
 461  
     }
 462  
 
 463  
     /**
 464  
      * Runs the analyzers against all of the dependencies. Since the mutable
 465  
      * dependencies list is exposed via {@link #getDependencies()}, this method
 466  
      * iterates over a copy of the dependencies list. Thus, the potential for
 467  
      * {@link java.util.ConcurrentModificationException}s is avoided, and
 468  
      * analyzers may safely add or remove entries from the dependencies list.
 469  
      * <p>
 470  
      * Every effort is made to complete analysis on the dependencies. In some
 471  
      * cases an exception will occur with part of the analysis being performed
 472  
      * which may not affect the entire analysis. If an exception occurs it will
 473  
      * be included in the thrown exception collection.
 474  
      *
 475  
      * @throws ExceptionCollection a collections of any exceptions that occurred
 476  
      * during analysis
 477  
      */
 478  
     public void analyzeDependencies() throws ExceptionCollection {
 479  2
         final List<Throwable> exceptions = Collections.synchronizedList(new ArrayList<Throwable>());
 480  2
         boolean autoUpdate = true;
 481  
         try {
 482  2
             autoUpdate = Settings.getBoolean(Settings.KEYS.AUTO_UPDATE);
 483  0
         } catch (InvalidSettingException ex) {
 484  0
             LOGGER.debug("Invalid setting for auto-update; using true.");
 485  0
             exceptions.add(ex);
 486  2
         }
 487  2
         if (autoUpdate) {
 488  
             try {
 489  0
                 doUpdates();
 490  0
             } catch (UpdateException ex) {
 491  0
                 exceptions.add(ex);
 492  0
                 LOGGER.warn("Unable to update Cached Web DataSource, using local "
 493  
                         + "data instead. Results may not include recent vulnerabilities.");
 494  0
                 LOGGER.debug("Update Error", ex);
 495  0
             }
 496  
         }
 497  
 
 498  
         //need to ensure that data exists
 499  
         try {
 500  2
             ensureDataExists();
 501  0
         } catch (NoDataException ex) {
 502  0
             throwFatalExceptionCollection("Unable to continue dependency-check analysis.", ex, exceptions);
 503  0
         } catch (DatabaseException ex) {
 504  0
             throwFatalExceptionCollection("Unable to connect to the dependency-check database.", ex, exceptions);
 505  2
         }
 506  
 
 507  2
         LOGGER.debug("\n----------------------------------------------------\nBEGIN ANALYSIS\n----------------------------------------------------");
 508  2
         LOGGER.info("Analysis Started");
 509  2
         final long analysisStart = System.currentTimeMillis();
 510  
 
 511  
         // analysis phases
 512  22
         for (AnalysisPhase phase : AnalysisPhase.values()) {
 513  20
             final List<Analyzer> analyzerList = analyzers.get(phase);
 514  
 
 515  20
             for (final Analyzer analyzer : analyzerList) {
 516  52
                 final long analyzerStart = System.currentTimeMillis();
 517  
                 try {
 518  52
                     initializeAnalyzer(analyzer);
 519  1
                 } catch (InitializationException ex) {
 520  1
                     exceptions.add(ex);
 521  1
                     continue;
 522  51
                 }
 523  
 
 524  51
                 executeAnalysisTasks(analyzer, exceptions);
 525  
 
 526  51
                 final long analyzerDurationMillis = System.currentTimeMillis() - analyzerStart;
 527  51
                 final long analyzerDurationSeconds = TimeUnit.MILLISECONDS.toSeconds(analyzerDurationMillis);
 528  51
                 LOGGER.info("Finished {} ({} seconds)", analyzer.getName(), analyzerDurationSeconds);
 529  51
             }
 530  
         }
 531  22
         for (AnalysisPhase phase : AnalysisPhase.values()) {
 532  20
             final List<Analyzer> analyzerList = analyzers.get(phase);
 533  
 
 534  20
             for (Analyzer a : analyzerList) {
 535  52
                 closeAnalyzer(a);
 536  52
             }
 537  
         }
 538  
 
 539  2
         LOGGER.debug("\n----------------------------------------------------\nEND ANALYSIS\n----------------------------------------------------");
 540  2
         final long analysisDurationSeconds = TimeUnit.MILLISECONDS.toSeconds(System.currentTimeMillis() - analysisStart);
 541  2
         LOGGER.info("Analysis Complete ({} seconds)", analysisDurationSeconds);
 542  2
         if (exceptions.size() > 0) {
 543  1
             throw new ExceptionCollection("One or more exceptions occurred during dependency-check analysis", exceptions);
 544  
         }
 545  1
     }
 546  
 
 547  
     /**
 548  
      * Executes executes the analyzer using multiple threads.
 549  
      *
 550  
      * @param exceptions a collection of exceptions that occurred during
 551  
      * analysis
 552  
      * @param analyzer the analyzer to execute
 553  
      * @throws ExceptionCollection thrown if exceptions occurred during analysis
 554  
      */
 555  
     void executeAnalysisTasks(Analyzer analyzer, List<Throwable> exceptions) throws ExceptionCollection {
 556  52
         LOGGER.debug("Starting {}", analyzer.getName());
 557  52
         final List<AnalysisTask> analysisTasks = getAnalysisTasks(analyzer, exceptions);
 558  52
         final ExecutorService executorService = getExecutorService(analyzer);
 559  
 
 560  
         try {
 561  52
             final List<Future<Void>> results = executorService.invokeAll(analysisTasks, 10, TimeUnit.MINUTES);
 562  
 
 563  
             // ensure there was no exception during execution
 564  52
             for (Future<Void> result : results) {
 565  
                 try {
 566  103
                     result.get();
 567  1
                 } catch (ExecutionException e) {
 568  1
                     throwFatalExceptionCollection("Analysis task failed with a fatal exception.", e, exceptions);
 569  0
                 } catch (CancellationException e) {
 570  0
                     throwFatalExceptionCollection("Analysis task timed out.", e, exceptions);
 571  102
                 }
 572  102
             }
 573  0
         } catch (InterruptedException e) {
 574  0
             throwFatalExceptionCollection("Analysis has been interrupted.", e, exceptions);
 575  
         } finally {
 576  52
             executorService.shutdown();
 577  51
         }
 578  51
     }
 579  
 
 580  
     /**
 581  
      * Returns the analysis tasks for the dependencies.
 582  
      *
 583  
      * @param analyzer the analyzer to create tasks for
 584  
      * @param exceptions the collection of exceptions to collect
 585  
      * @return a collection of analysis tasks
 586  
      */
 587  
     List<AnalysisTask> getAnalysisTasks(Analyzer analyzer, List<Throwable> exceptions) {
 588  51
         final List<AnalysisTask> result = new ArrayList<AnalysisTask>();
 589  51
         synchronized (dependencies) {
 590  51
             for (final Dependency dependency : dependencies) {
 591  102
                 final AnalysisTask task = new AnalysisTask(analyzer, dependency, this, exceptions);
 592  102
                 result.add(task);
 593  102
             }
 594  51
         }
 595  51
         return result;
 596  
     }
 597  
 
 598  
     /**
 599  
      * Returns the executor service for a given analyzer.
 600  
      *
 601  
      * @param analyzer the analyzer to obtain an executor
 602  
      * @return the executor service
 603  
      */
 604  
     ExecutorService getExecutorService(Analyzer analyzer) {
 605  51
         if (analyzer.supportsParallelProcessing()) {
 606  
             // just a fair trade-off that should be reasonable for all analyzer types
 607  47
             final int maximumNumberOfThreads = 4 * Runtime.getRuntime().availableProcessors();
 608  
 
 609  47
             LOGGER.debug("Parallel processing with up to {} threads: {}.", maximumNumberOfThreads, analyzer.getName());
 610  47
             return Executors.newFixedThreadPool(maximumNumberOfThreads);
 611  
         } else {
 612  4
             LOGGER.debug("Parallel processing is not supported: {}.", analyzer.getName());
 613  4
             return Executors.newSingleThreadExecutor();
 614  
         }
 615  
     }
 616  
 
 617  
     /**
 618  
      * Initializes the given analyzer.
 619  
      *
 620  
      * @param analyzer the analyzer to initialize
 621  
      * @return the initialized analyzer
 622  
      * @throws InitializationException thrown when there is a problem
 623  
      * initializing the analyzer
 624  
      */
 625  
     protected Analyzer initializeAnalyzer(Analyzer analyzer) throws InitializationException {
 626  
         try {
 627  52
             LOGGER.debug("Initializing {}", analyzer.getName());
 628  52
             analyzer.initialize();
 629  1
         } catch (InitializationException ex) {
 630  1
             LOGGER.error("Exception occurred initializing {}.", analyzer.getName());
 631  1
             LOGGER.debug("", ex);
 632  
             try {
 633  1
                 analyzer.close();
 634  0
             } catch (Throwable ex1) {
 635  0
                 LOGGER.trace("", ex1);
 636  1
             }
 637  1
             throw ex;
 638  0
         } catch (Throwable ex) {
 639  0
             LOGGER.error("Unexpected exception occurred initializing {}.", analyzer.getName());
 640  0
             LOGGER.debug("", ex);
 641  
             try {
 642  0
                 analyzer.close();
 643  0
             } catch (Throwable ex1) {
 644  0
                 LOGGER.trace("", ex1);
 645  0
             }
 646  0
             throw new InitializationException("Unexpected Exception", ex);
 647  51
         }
 648  51
         return analyzer;
 649  
     }
 650  
 
 651  
     /**
 652  
      * Closes the given analyzer.
 653  
      *
 654  
      * @param analyzer the analyzer to close
 655  
      */
 656  
     protected void closeAnalyzer(Analyzer analyzer) {
 657  52
         LOGGER.debug("Closing Analyzer '{}'", analyzer.getName());
 658  
         try {
 659  52
             analyzer.close();
 660  0
         } catch (Throwable ex) {
 661  0
             LOGGER.trace("", ex);
 662  52
         }
 663  52
     }
 664  
 
 665  
     /**
 666  
      * Cycles through the cached web data sources and calls update on all of
 667  
      * them.
 668  
      *
 669  
      * @throws UpdateException thrown if the operation fails
 670  
      */
 671  
     public void doUpdates() throws UpdateException {
 672  0
         LOGGER.info("Checking for updates");
 673  0
         final long updateStart = System.currentTimeMillis();
 674  0
         final UpdateService service = new UpdateService(serviceClassLoader);
 675  0
         final Iterator<CachedWebDataSource> iterator = service.getDataSources();
 676  0
         while (iterator.hasNext()) {
 677  0
             final CachedWebDataSource source = iterator.next();
 678  0
             source.update();
 679  0
         }
 680  0
         LOGGER.info("Check for updates complete ({} ms)", System.currentTimeMillis() - updateStart);
 681  0
     }
 682  
 
 683  
     /**
 684  
      * Returns a full list of all of the analyzers. This is useful for reporting
 685  
      * which analyzers where used.
 686  
      *
 687  
      * @return a list of Analyzers
 688  
      */
 689  
     public List<Analyzer> getAnalyzers() {
 690  0
         final List<Analyzer> ret = new ArrayList<Analyzer>();
 691  0
         for (AnalysisPhase phase : AnalysisPhase.values()) {
 692  0
             final List<Analyzer> analyzerList = analyzers.get(phase);
 693  0
             ret.addAll(analyzerList);
 694  
         }
 695  0
         return ret;
 696  
     }
 697  
 
 698  
     /**
 699  
      * Checks all analyzers to see if an extension is supported.
 700  
      *
 701  
      * @param file a file extension
 702  
      * @return true or false depending on whether or not the file extension is
 703  
      * supported
 704  
      */
 705  
     @Override
 706  
     public boolean accept(File file) {
 707  858
         if (file == null) {
 708  0
             return false;
 709  
         }
 710  858
         boolean scan = false;
 711  858
         for (FileTypeAnalyzer a : this.fileTypeAnalyzers) {
 712  
             /* note, we can't break early on this loop as the analyzers need to know if
 713  
              they have files to work on prior to initialization */
 714  15447
             scan |= a.accept(file);
 715  15447
         }
 716  858
         return scan;
 717  
     }
 718  
 
 719  
     /**
 720  
      * Returns the set of file type analyzers.
 721  
      *
 722  
      * @return the set of file type analyzers
 723  
      */
 724  
     public Set<FileTypeAnalyzer> getFileTypeAnalyzers() {
 725  0
         return this.fileTypeAnalyzers;
 726  
     }
 727  
 
 728  
     /**
 729  
      * Adds a file type analyzer. This has been added solely to assist in unit
 730  
      * testing the Engine.
 731  
      *
 732  
      * @param fta the file type analyzer to add
 733  
      */
 734  
     protected void addFileTypeAnalyzer(FileTypeAnalyzer fta) {
 735  1
         this.fileTypeAnalyzers.add(fta);
 736  1
     }
 737  
 
 738  
     /**
 739  
      * Checks the CPE Index to ensure documents exists. If none exist a
 740  
      * NoDataException is thrown.
 741  
      *
 742  
      * @throws NoDataException thrown if no data exists in the CPE Index
 743  
      * @throws DatabaseException thrown if there is an exception opening the
 744  
      * database
 745  
      */
 746  
     private void ensureDataExists() throws NoDataException, DatabaseException {
 747  2
         final CveDB cve = new CveDB();
 748  
         try {
 749  2
             cve.open();
 750  2
             if (!cve.dataExists()) {
 751  0
                 throw new NoDataException("No documents exist");
 752  
             }
 753  0
         } catch (DatabaseException ex) {
 754  0
             throw new NoDataException(ex.getMessage(), ex);
 755  
         } finally {
 756  2
             cve.close();
 757  2
         }
 758  2
     }
 759  
 
 760  
     /**
 761  
      * Constructs and throws a fatal exception collection.
 762  
      *
 763  
      * @param message the exception message
 764  
      * @param throwable the cause
 765  
      * @param exceptions a collection of exception to include
 766  
      * @throws ExceptionCollection a collection of exceptions that occurred
 767  
      * during analysis
 768  
      */
 769  
     private void throwFatalExceptionCollection(String message, Throwable throwable, List<Throwable> exceptions) throws ExceptionCollection {
 770  1
         LOGGER.error("{}\n\n{}", throwable.getMessage(), message);
 771  1
         LOGGER.debug("", throwable);
 772  1
         exceptions.add(throwable);
 773  1
         throw new ExceptionCollection(message, exceptions, true);
 774  
     }
 775  
 }