Coverage Report - org.owasp.dependencycheck.Engine
 
Classes in this File Line Coverage Branch Coverage Complexity
Engine
64%
166/258
63%
55/86
2.943
 
 1  
 /*
 2  
  * This file is part of dependency-check-core.
 3  
  *
 4  
  * Licensed under the Apache License, Version 2.0 (the "License");
 5  
  * you may not use this file except in compliance with the License.
 6  
  * You may obtain a copy of the License at
 7  
  *
 8  
  *     http://www.apache.org/licenses/LICENSE-2.0
 9  
  *
 10  
  * Unless required by applicable law or agreed to in writing, software
 11  
  * distributed under the License is distributed on an "AS IS" BASIS,
 12  
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 13  
  * See the License for the specific language governing permissions and
 14  
  * limitations under the License.
 15  
  *
 16  
  * Copyright (c) 2012 Jeremy Long. All Rights Reserved.
 17  
  */
 18  
 package org.owasp.dependencycheck;
 19  
 
 20  
 import org.owasp.dependencycheck.analyzer.AnalysisPhase;
 21  
 import org.owasp.dependencycheck.analyzer.Analyzer;
 22  
 import org.owasp.dependencycheck.analyzer.AnalyzerService;
 23  
 import org.owasp.dependencycheck.analyzer.FileTypeAnalyzer;
 24  
 import org.owasp.dependencycheck.data.nvdcve.ConnectionFactory;
 25  
 import org.owasp.dependencycheck.data.nvdcve.CveDB;
 26  
 import org.owasp.dependencycheck.data.nvdcve.DatabaseException;
 27  
 import org.owasp.dependencycheck.data.update.CachedWebDataSource;
 28  
 import org.owasp.dependencycheck.data.update.UpdateService;
 29  
 import org.owasp.dependencycheck.data.update.exception.UpdateException;
 30  
 import org.owasp.dependencycheck.dependency.Dependency;
 31  
 import org.owasp.dependencycheck.exception.ExceptionCollection;
 32  
 import org.owasp.dependencycheck.exception.InitializationException;
 33  
 import org.owasp.dependencycheck.exception.NoDataException;
 34  
 import org.owasp.dependencycheck.utils.InvalidSettingException;
 35  
 import org.owasp.dependencycheck.utils.Settings;
 36  
 import org.slf4j.Logger;
 37  
 import org.slf4j.LoggerFactory;
 38  
 
 39  
 import java.io.File;
 40  
 import java.io.FileFilter;
 41  
 import java.util.ArrayList;
 42  
 import java.util.Collection;
 43  
 import java.util.Collections;
 44  
 import java.util.EnumMap;
 45  
 import java.util.HashSet;
 46  
 import java.util.Iterator;
 47  
 import java.util.List;
 48  
 import java.util.Map;
 49  
 import java.util.Set;
 50  
 import java.util.concurrent.CancellationException;
 51  
 import java.util.concurrent.ExecutionException;
 52  
 import java.util.concurrent.ExecutorService;
 53  
 import java.util.concurrent.Executors;
 54  
 import java.util.concurrent.Future;
 55  
 import java.util.concurrent.TimeUnit;
 56  
 
 57  
 /**
 58  
  * Scans files, directories, etc. for Dependencies. Analyzers are loaded and
 59  
  * used to process the files found by the scan, if a file is encountered and an
 60  
  * Analyzer is associated with the file type then the file is turned into a
 61  
  * dependency.
 62  
  *
 63  
  * @author Jeremy Long
 64  
  */
 65  
 public class Engine implements FileFilter {
 66  
 
 67  
     /**
 68  
      * The list of dependencies.
 69  
      */
 70  6
     private final List<Dependency> dependencies = Collections.synchronizedList(new ArrayList<Dependency>());
 71  
     /**
 72  
      * A Map of analyzers grouped by Analysis phase.
 73  
      */
 74  6
     private final Map<AnalysisPhase, List<Analyzer>> analyzers = new EnumMap<AnalysisPhase, List<Analyzer>>(AnalysisPhase.class);
 75  
 
 76  
     /**
 77  
      * A Map of analyzers grouped by Analysis phase.
 78  
      */
 79  6
     private final Set<FileTypeAnalyzer> fileTypeAnalyzers = new HashSet<FileTypeAnalyzer>();
 80  
 
 81  
     /**
 82  
      * The ClassLoader to use when dynamically loading Analyzer and Update
 83  
      * services.
 84  
      */
 85  6
     private ClassLoader serviceClassLoader = Thread.currentThread().getContextClassLoader();
 86  
     /**
 87  
      * The Logger for use throughout the class.
 88  
      */
 89  1
     private static final Logger LOGGER = LoggerFactory.getLogger(Engine.class);
 90  
 
 91  
     /**
 92  
      * Creates a new Engine.
 93  
      *
 94  
      * @throws DatabaseException thrown if there is an error connecting to the
 95  
      * database
 96  
      */
 97  6
     public Engine() throws DatabaseException {
 98  6
         initializeEngine();
 99  6
     }
 100  
 
 101  
     /**
 102  
      * Creates a new Engine.
 103  
      *
 104  
      * @param serviceClassLoader a reference the class loader being used
 105  
      * @throws DatabaseException thrown if there is an error connecting to the
 106  
      * database
 107  
      */
 108  0
     public Engine(ClassLoader serviceClassLoader) throws DatabaseException {
 109  0
         this.serviceClassLoader = serviceClassLoader;
 110  0
         initializeEngine();
 111  0
     }
 112  
 
 113  
     /**
 114  
      * Creates a new Engine using the specified classloader to dynamically load
 115  
      * Analyzer and Update services.
 116  
      *
 117  
      * @throws DatabaseException thrown if there is an error connecting to the
 118  
      * database
 119  
      */
 120  
     protected final void initializeEngine() throws DatabaseException {
 121  6
         ConnectionFactory.initialize();
 122  6
         loadAnalyzers();
 123  6
     }
 124  
 
 125  
     /**
 126  
      * Properly cleans up resources allocated during analysis.
 127  
      */
 128  
     public void cleanup() {
 129  0
         ConnectionFactory.cleanup();
 130  0
     }
 131  
 
 132  
     /**
 133  
      * Loads the analyzers specified in the configuration file (or system
 134  
      * properties).
 135  
      */
 136  
     private void loadAnalyzers() {
 137  6
         if (!analyzers.isEmpty()) {
 138  0
             return;
 139  
         }
 140  72
         for (AnalysisPhase phase : AnalysisPhase.values()) {
 141  66
             analyzers.put(phase, new ArrayList<Analyzer>());
 142  
         }
 143  
 
 144  6
         final AnalyzerService service = new AnalyzerService(serviceClassLoader);
 145  6
         final List<Analyzer> iterator = service.getAnalyzers();
 146  6
         for (Analyzer a : iterator) {
 147  168
             analyzers.get(a.getAnalysisPhase()).add(a);
 148  168
             if (a instanceof FileTypeAnalyzer) {
 149  108
                 this.fileTypeAnalyzers.add((FileTypeAnalyzer) a);
 150  
             }
 151  168
         }
 152  6
     }
 153  
 
 154  
     /**
 155  
      * Get the List of the analyzers for a specific phase of analysis.
 156  
      *
 157  
      * @param phase the phase to get the configured analyzers.
 158  
      * @return the analyzers loaded
 159  
      */
 160  
     public List<Analyzer> getAnalyzers(AnalysisPhase phase) {
 161  0
         return analyzers.get(phase);
 162  
     }
 163  
 
 164  
     /**
 165  
      * Get the dependencies identified. The returned list is a reference to the
 166  
      * engine's synchronized list. <b>You must synchronize on the returned
 167  
      * list</b> when you modify and iterate over it from multiple threads. E.g.
 168  
      * this holds for analyzers supporting parallel processing during their
 169  
      * analysis phase.
 170  
      *
 171  
      * @return the dependencies identified
 172  
      * @see Collections#synchronizedList(List)
 173  
      * @see Analyzer#supportsParallelProcessing()
 174  
      */
 175  
     public synchronized List<Dependency> getDependencies() {
 176  52
         return dependencies;
 177  
     }
 178  
 
 179  
     /**
 180  
      * Sets the dependencies.
 181  
      *
 182  
      * @param dependencies the dependencies
 183  
      */
 184  
     public void setDependencies(List<Dependency> dependencies) {
 185  0
         synchronized (this.dependencies) {
 186  0
             this.dependencies.clear();
 187  0
             this.dependencies.addAll(dependencies);
 188  0
         }
 189  0
     }
 190  
 
 191  
     /**
 192  
      * Scans an array of files or directories. If a directory is specified, it
 193  
      * will be scanned recursively. Any dependencies identified are added to the
 194  
      * dependency collection.
 195  
      *
 196  
      * @param paths an array of paths to files or directories to be analyzed
 197  
      * @return the list of dependencies scanned
 198  
      * @since v0.3.2.5
 199  
      */
 200  
     public List<Dependency> scan(String[] paths) {
 201  0
         return scan(paths, null);
 202  
     }
 203  
 
 204  
     /**
 205  
      * Scans an array of files or directories. If a directory is specified, it
 206  
      * will be scanned recursively. Any dependencies identified are added to the
 207  
      * dependency collection.
 208  
      *
 209  
      * @param paths an array of paths to files or directories to be analyzed
 210  
      * @param projectReference the name of the project or scope in which the
 211  
      * dependency was identified
 212  
      * @return the list of dependencies scanned
 213  
      * @since v1.4.4
 214  
      */
 215  
     public List<Dependency> scan(String[] paths, String projectReference) {
 216  0
         final List<Dependency> deps = new ArrayList<Dependency>();
 217  0
         for (String path : paths) {
 218  0
             final List<Dependency> d = scan(path, projectReference);
 219  0
             if (d != null) {
 220  0
                 deps.addAll(d);
 221  
             }
 222  
         }
 223  0
         return deps;
 224  
     }
 225  
 
 226  
     /**
 227  
      * Scans a given file or directory. If a directory is specified, it will be
 228  
      * scanned recursively. Any dependencies identified are added to the
 229  
      * dependency collection.
 230  
      *
 231  
      * @param path the path to a file or directory to be analyzed
 232  
      * @return the list of dependencies scanned
 233  
      */
 234  
     public List<Dependency> scan(String path) {
 235  0
         return scan(path, null);
 236  
     }
 237  
 
 238  
     /**
 239  
      * Scans a given file or directory. If a directory is specified, it will be
 240  
      * scanned recursively. Any dependencies identified are added to the
 241  
      * dependency collection.
 242  
      *
 243  
      * @param path the path to a file or directory to be analyzed
 244  
      * @param projectReference the name of the project or scope in which the
 245  
      * dependency was identified
 246  
      * @return the list of dependencies scanned
 247  
      * @since v1.4.4
 248  
      */
 249  
     public List<Dependency> scan(String path, String projectReference) {
 250  0
         final File file = new File(path);
 251  0
         return scan(file, projectReference);
 252  
     }
 253  
 
 254  
     /**
 255  
      * Scans an array of files or directories. If a directory is specified, it
 256  
      * will be scanned recursively. Any dependencies identified are added to the
 257  
      * dependency collection.
 258  
      *
 259  
      * @param files an array of paths to files or directories to be analyzed.
 260  
      * @return the list of dependencies
 261  
      * @since v0.3.2.5
 262  
      */
 263  
     public List<Dependency> scan(File[] files) {
 264  0
         return scan(files, null);
 265  
     }
 266  
 
 267  
     /**
 268  
      * Scans an array of files or directories. If a directory is specified, it
 269  
      * will be scanned recursively. Any dependencies identified are added to the
 270  
      * dependency collection.
 271  
      *
 272  
      * @param files an array of paths to files or directories to be analyzed.
 273  
      * @param projectReference the name of the project or scope in which the
 274  
      * dependency was identified
 275  
      * @return the list of dependencies
 276  
      * @since v1.4.4
 277  
      */
 278  
     public List<Dependency> scan(File[] files, String projectReference) {
 279  0
         final List<Dependency> deps = new ArrayList<Dependency>();
 280  0
         for (File file : files) {
 281  0
             final List<Dependency> d = scan(file, projectReference);
 282  0
             if (d != null) {
 283  0
                 deps.addAll(d);
 284  
             }
 285  
         }
 286  0
         return deps;
 287  
     }
 288  
 
 289  
     /**
 290  
      * Scans a collection of files or directories. If a directory is specified,
 291  
      * it will be scanned recursively. Any dependencies identified are added to
 292  
      * the dependency collection.
 293  
      *
 294  
      * @param files a set of paths to files or directories to be analyzed
 295  
      * @return the list of dependencies scanned
 296  
      * @since v0.3.2.5
 297  
      */
 298  
     public List<Dependency> scan(Collection<File> files) {
 299  0
         return scan(files, null);
 300  
     }
 301  
 
 302  
     /**
 303  
      * Scans a collection of files or directories. If a directory is specified,
 304  
      * it will be scanned recursively. Any dependencies identified are added to
 305  
      * the dependency collection.
 306  
      *
 307  
      * @param files a set of paths to files or directories to be analyzed
 308  
      * @param projectReference the name of the project or scope in which the
 309  
      * dependency was identified
 310  
      * @return the list of dependencies scanned
 311  
      * @since v1.4.4
 312  
      */
 313  
     public List<Dependency> scan(Collection<File> files, String projectReference) {
 314  0
         final List<Dependency> deps = new ArrayList<Dependency>();
 315  0
         for (File file : files) {
 316  0
             final List<Dependency> d = scan(file, projectReference);
 317  0
             if (d != null) {
 318  0
                 deps.addAll(d);
 319  
             }
 320  0
         }
 321  0
         return deps;
 322  
     }
 323  
 
 324  
     /**
 325  
      * Scans a given file or directory. If a directory is specified, it will be
 326  
      * scanned recursively. Any dependencies identified are added to the
 327  
      * dependency collection.
 328  
      *
 329  
      * @param file the path to a file or directory to be analyzed
 330  
      * @return the list of dependencies scanned
 331  
      * @since v0.3.2.4
 332  
      */
 333  
     public List<Dependency> scan(File file) {
 334  5
         return scan(file, null);
 335  
     }
 336  
 
 337  
     /**
 338  
      * Scans a given file or directory. If a directory is specified, it will be
 339  
      * scanned recursively. Any dependencies identified are added to the
 340  
      * dependency collection.
 341  
      *
 342  
      * @param file the path to a file or directory to be analyzed
 343  
      * @param projectReference the name of the project or scope in which the
 344  
      * dependency was identified
 345  
      * @return the list of dependencies scanned
 346  
      * @since v1.4.4
 347  
      */
 348  
     public List<Dependency> scan(File file, String projectReference) {
 349  5
         if (file.exists()) {
 350  5
             if (file.isDirectory()) {
 351  3
                 return scanDirectory(file, projectReference);
 352  
             } else {
 353  2
                 final Dependency d = scanFile(file, projectReference);
 354  2
                 if (d != null) {
 355  2
                     final List<Dependency> deps = new ArrayList<Dependency>();
 356  2
                     deps.add(d);
 357  2
                     return deps;
 358  
                 }
 359  
             }
 360  
         }
 361  0
         return null;
 362  
     }
 363  
 
 364  
     /**
 365  
      * Recursively scans files and directories. Any dependencies identified are
 366  
      * added to the dependency collection.
 367  
      *
 368  
      * @param dir the directory to scan
 369  
      * @return the list of Dependency objects scanned
 370  
      */
 371  
     protected List<Dependency> scanDirectory(File dir) {
 372  0
         return scanDirectory(dir, null);
 373  
     }
 374  
 
 375  
     /**
 376  
      * Recursively scans files and directories. Any dependencies identified are
 377  
      * added to the dependency collection.
 378  
      *
 379  
      * @param dir the directory to scan
 380  
      * @param projectReference the name of the project or scope in which the
 381  
      * dependency was identified
 382  
      * @return the list of Dependency objects scanned
 383  
      * @since v1.4.4
 384  
      */
 385  
     protected List<Dependency> scanDirectory(File dir, String projectReference) {
 386  46
         final File[] files = dir.listFiles();
 387  46
         final List<Dependency> deps = new ArrayList<Dependency>();
 388  46
         if (files != null) {
 389  93
             for (File f : files) {
 390  47
                 if (f.isDirectory()) {
 391  43
                     final List<Dependency> d = scanDirectory(f, projectReference);
 392  43
                     if (d != null) {
 393  43
                         deps.addAll(d);
 394  
                     }
 395  43
                 } else {
 396  4
                     final Dependency d = scanFile(f, projectReference);
 397  4
                     deps.add(d);
 398  
                 }
 399  
             }
 400  
         }
 401  46
         return deps;
 402  
     }
 403  
 
 404  
     /**
 405  
      * Scans a specified file. If a dependency is identified it is added to the
 406  
      * dependency collection.
 407  
      *
 408  
      * @param file The file to scan
 409  
      * @return the scanned dependency
 410  
      */
 411  
     protected Dependency scanFile(File file) {
 412  3
         return scanFile(file, null);
 413  
     }
 414  
 
 415  
     /**
 416  
      * Scans a specified file. If a dependency is identified it is added to the
 417  
      * dependency collection.
 418  
      *
 419  
      * @param file The file to scan
 420  
      * @param projectReference the name of the project or scope in which the
 421  
      * dependency was identified
 422  
      * @return the scanned dependency
 423  
      * @since v1.4.4
 424  
      */
 425  
     protected Dependency scanFile(File file, String projectReference) {
 426  9
         Dependency dependency = null;
 427  9
         if (file.isFile()) {
 428  9
             if (accept(file)) {
 429  7
                 dependency = new Dependency(file);
 430  7
                 if (projectReference != null) {
 431  0
                     dependency.addProjectReference(projectReference);
 432  
                 }
 433  7
                 final String sha1 = dependency.getSha1sum();
 434  7
                 boolean found = false;
 435  7
                 synchronized (dependencies) {
 436  7
                     if (sha1 != null) {
 437  7
                         for (Dependency existing : dependencies) {
 438  4
                             if (sha1.equals(existing.getSha1sum())) {
 439  1
                                 found = true;
 440  1
                                 if (projectReference != null) {
 441  0
                                     existing.addProjectReference(projectReference);
 442  
                                 }
 443  1
                                 if (existing.getActualFilePath() != null && dependency.getActualFilePath() != null
 444  1
                                         && !existing.getActualFilePath().equals(dependency.getActualFilePath())) {
 445  0
                                     existing.addRelatedDependency(dependency);
 446  
                                 } else {
 447  1
                                     dependency = existing;
 448  
                                 }
 449  1
                                 break;
 450  
                             }
 451  3
                         }
 452  
                     }
 453  7
                     if (!found) {
 454  6
                         dependencies.add(dependency);
 455  
                     }
 456  7
                 }
 457  7
             } else {
 458  2
                 LOGGER.debug("Path passed to scanFile(File) is not a file: {}. Skipping the file.", file);
 459  
             }
 460  
         }
 461  9
         return dependency;
 462  
     }
 463  
 
 464  
     /**
 465  
      * Runs the analyzers against all of the dependencies. Since the mutable
 466  
      * dependencies list is exposed via {@link #getDependencies()}, this method
 467  
      * iterates over a copy of the dependencies list. Thus, the potential for
 468  
      * {@link java.util.ConcurrentModificationException}s is avoided, and
 469  
      * analyzers may safely add or remove entries from the dependencies list.
 470  
      * <p>
 471  
      * Every effort is made to complete analysis on the dependencies. In some
 472  
      * cases an exception will occur with part of the analysis being performed
 473  
      * which may not affect the entire analysis. If an exception occurs it will
 474  
      * be included in the thrown exception collection.
 475  
      *
 476  
      * @throws ExceptionCollection a collections of any exceptions that occurred
 477  
      * during analysis
 478  
      */
 479  
     public void analyzeDependencies() throws ExceptionCollection {
 480  2
         final List<Throwable> exceptions = Collections.synchronizedList(new ArrayList<Throwable>());
 481  2
         boolean autoUpdate = true;
 482  
         try {
 483  2
             autoUpdate = Settings.getBoolean(Settings.KEYS.AUTO_UPDATE);
 484  0
         } catch (InvalidSettingException ex) {
 485  0
             LOGGER.debug("Invalid setting for auto-update; using true.");
 486  0
             exceptions.add(ex);
 487  2
         }
 488  2
         if (autoUpdate) {
 489  
             try {
 490  0
                 doUpdates();
 491  0
             } catch (UpdateException ex) {
 492  0
                 exceptions.add(ex);
 493  0
                 LOGGER.warn("Unable to update Cached Web DataSource, using local "
 494  
                         + "data instead. Results may not include recent vulnerabilities.");
 495  0
                 LOGGER.debug("Update Error", ex);
 496  0
             }
 497  
         }
 498  
 
 499  
         //need to ensure that data exists
 500  
         try {
 501  2
             ensureDataExists();
 502  0
         } catch (NoDataException ex) {
 503  0
             throwFatalExceptionCollection("Unable to continue dependency-check analysis.", ex, exceptions);
 504  0
         } catch (DatabaseException ex) {
 505  0
             throwFatalExceptionCollection("Unable to connect to the dependency-check database.", ex, exceptions);
 506  2
         }
 507  
 
 508  2
         LOGGER.debug("\n----------------------------------------------------\nBEGIN ANALYSIS\n----------------------------------------------------");
 509  2
         LOGGER.info("Analysis Started");
 510  2
         final long analysisStart = System.currentTimeMillis();
 511  
 
 512  
         // analysis phases
 513  24
         for (AnalysisPhase phase : AnalysisPhase.values()) {
 514  22
             final List<Analyzer> analyzerList = analyzers.get(phase);
 515  
 
 516  22
             for (final Analyzer analyzer : analyzerList) {
 517  56
                 final long analyzerStart = System.currentTimeMillis();
 518  
                 try {
 519  56
                     initializeAnalyzer(analyzer);
 520  1
                 } catch (InitializationException ex) {
 521  1
                     exceptions.add(ex);
 522  1
                     continue;
 523  55
                 }
 524  
 
 525  55
                 if (analyzer.isEnabled()) {
 526  24
                     executeAnalysisTasks(analyzer, exceptions);
 527  
 
 528  24
                     final long analyzerDurationMillis = System.currentTimeMillis() - analyzerStart;
 529  24
                     final long analyzerDurationSeconds = TimeUnit.MILLISECONDS.toSeconds(analyzerDurationMillis);
 530  24
                     LOGGER.info("Finished {} ({} seconds)", analyzer.getName(), analyzerDurationSeconds);
 531  24
                 } else {
 532  31
                     LOGGER.debug("Skipping {} (not enabled)", analyzer.getName());
 533  
                 }
 534  55
             }
 535  
         }
 536  24
         for (AnalysisPhase phase : AnalysisPhase.values()) {
 537  22
             final List<Analyzer> analyzerList = analyzers.get(phase);
 538  
 
 539  22
             for (Analyzer a : analyzerList) {
 540  56
                 closeAnalyzer(a);
 541  56
             }
 542  
         }
 543  
 
 544  2
         LOGGER.debug("\n----------------------------------------------------\nEND ANALYSIS\n----------------------------------------------------");
 545  2
         final long analysisDurationSeconds = TimeUnit.MILLISECONDS.toSeconds(System.currentTimeMillis() - analysisStart);
 546  2
         LOGGER.info("Analysis Complete ({} seconds)", analysisDurationSeconds);
 547  2
         if (exceptions.size() > 0) {
 548  1
             throw new ExceptionCollection("One or more exceptions occurred during dependency-check analysis", exceptions);
 549  
         }
 550  1
     }
 551  
 
 552  
     /**
 553  
      * Executes executes the analyzer using multiple threads.
 554  
      *
 555  
      * @param exceptions a collection of exceptions that occurred during
 556  
      * analysis
 557  
      * @param analyzer the analyzer to execute
 558  
      * @throws ExceptionCollection thrown if exceptions occurred during analysis
 559  
      */
 560  
     void executeAnalysisTasks(Analyzer analyzer, List<Throwable> exceptions) throws ExceptionCollection {
 561  25
         LOGGER.debug("Starting {}", analyzer.getName());
 562  25
         final List<AnalysisTask> analysisTasks = getAnalysisTasks(analyzer, exceptions);
 563  25
         final ExecutorService executorService = getExecutorService(analyzer);
 564  
 
 565  
         try {
 566  25
             final List<Future<Void>> results = executorService.invokeAll(analysisTasks, 10, TimeUnit.MINUTES);
 567  
 
 568  
             // ensure there was no exception during execution
 569  25
             for (Future<Void> result : results) {
 570  
                 try {
 571  49
                     result.get();
 572  1
                 } catch (ExecutionException e) {
 573  1
                     throwFatalExceptionCollection("Analysis task failed with a fatal exception.", e, exceptions);
 574  0
                 } catch (CancellationException e) {
 575  0
                     throwFatalExceptionCollection("Analysis task timed out.", e, exceptions);
 576  48
                 }
 577  48
             }
 578  0
         } catch (InterruptedException e) {
 579  0
             throwFatalExceptionCollection("Analysis has been interrupted.", e, exceptions);
 580  
         } finally {
 581  25
             executorService.shutdown();
 582  24
         }
 583  24
     }
 584  
 
 585  
     /**
 586  
      * Returns the analysis tasks for the dependencies.
 587  
      *
 588  
      * @param analyzer the analyzer to create tasks for
 589  
      * @param exceptions the collection of exceptions to collect
 590  
      * @return a collection of analysis tasks
 591  
      */
 592  
     List<AnalysisTask> getAnalysisTasks(Analyzer analyzer, List<Throwable> exceptions) {
 593  24
         final List<AnalysisTask> result = new ArrayList<AnalysisTask>();
 594  24
         synchronized (dependencies) {
 595  24
             for (final Dependency dependency : dependencies) {
 596  48
                 final AnalysisTask task = new AnalysisTask(analyzer, dependency, this, exceptions, Settings.getInstance());
 597  48
                 result.add(task);
 598  48
             }
 599  24
         }
 600  24
         return result;
 601  
     }
 602  
 
 603  
     /**
 604  
      * Returns the executor service for a given analyzer.
 605  
      *
 606  
      * @param analyzer the analyzer to obtain an executor
 607  
      * @return the executor service
 608  
      */
 609  
     ExecutorService getExecutorService(Analyzer analyzer) {
 610  24
         if (analyzer.supportsParallelProcessing()) {
 611  
             // just a fair trade-off that should be reasonable for all analyzer types
 612  17
             final int maximumNumberOfThreads = 4 * Runtime.getRuntime().availableProcessors();
 613  
 
 614  17
             LOGGER.debug("Parallel processing with up to {} threads: {}.", maximumNumberOfThreads, analyzer.getName());
 615  17
             return Executors.newFixedThreadPool(maximumNumberOfThreads);
 616  
         } else {
 617  7
             LOGGER.debug("Parallel processing is not supported: {}.", analyzer.getName());
 618  7
             return Executors.newSingleThreadExecutor();
 619  
         }
 620  
     }
 621  
 
 622  
     /**
 623  
      * Initializes the given analyzer.
 624  
      *
 625  
      * @param analyzer the analyzer to initialize
 626  
      * @return the initialized analyzer
 627  
      * @throws InitializationException thrown when there is a problem
 628  
      * initializing the analyzer
 629  
      */
 630  
     protected Analyzer initializeAnalyzer(Analyzer analyzer) throws InitializationException {
 631  
         try {
 632  56
             LOGGER.debug("Initializing {}", analyzer.getName());
 633  56
             analyzer.initialize();
 634  1
         } catch (InitializationException ex) {
 635  1
             LOGGER.error("Exception occurred initializing {}.", analyzer.getName());
 636  1
             LOGGER.debug("", ex);
 637  
             try {
 638  1
                 analyzer.close();
 639  0
             } catch (Throwable ex1) {
 640  0
                 LOGGER.trace("", ex1);
 641  1
             }
 642  1
             throw ex;
 643  0
         } catch (Throwable ex) {
 644  0
             LOGGER.error("Unexpected exception occurred initializing {}.", analyzer.getName());
 645  0
             LOGGER.debug("", ex);
 646  
             try {
 647  0
                 analyzer.close();
 648  0
             } catch (Throwable ex1) {
 649  0
                 LOGGER.trace("", ex1);
 650  0
             }
 651  0
             throw new InitializationException("Unexpected Exception", ex);
 652  55
         }
 653  55
         return analyzer;
 654  
     }
 655  
 
 656  
     /**
 657  
      * Closes the given analyzer.
 658  
      *
 659  
      * @param analyzer the analyzer to close
 660  
      */
 661  
     protected void closeAnalyzer(Analyzer analyzer) {
 662  56
         LOGGER.debug("Closing Analyzer '{}'", analyzer.getName());
 663  
         try {
 664  56
             analyzer.close();
 665  0
         } catch (Throwable ex) {
 666  0
             LOGGER.trace("", ex);
 667  56
         }
 668  56
     }
 669  
 
 670  
     /**
 671  
      * Cycles through the cached web data sources and calls update on all of
 672  
      * them.
 673  
      *
 674  
      * @throws UpdateException thrown if the operation fails
 675  
      */
 676  
     public void doUpdates() throws UpdateException {
 677  0
         LOGGER.info("Checking for updates");
 678  0
         final long updateStart = System.currentTimeMillis();
 679  0
         final UpdateService service = new UpdateService(serviceClassLoader);
 680  0
         final Iterator<CachedWebDataSource> iterator = service.getDataSources();
 681  0
         while (iterator.hasNext()) {
 682  0
             final CachedWebDataSource source = iterator.next();
 683  0
             source.update();
 684  0
         }
 685  0
         LOGGER.info("Check for updates complete ({} ms)", System.currentTimeMillis() - updateStart);
 686  0
     }
 687  
 
 688  
     /**
 689  
      * Returns a full list of all of the analyzers. This is useful for reporting
 690  
      * which analyzers where used.
 691  
      *
 692  
      * @return a list of Analyzers
 693  
      */
 694  
     public List<Analyzer> getAnalyzers() {
 695  0
         final List<Analyzer> ret = new ArrayList<Analyzer>();
 696  0
         for (AnalysisPhase phase : AnalysisPhase.values()) {
 697  0
             final List<Analyzer> analyzerList = analyzers.get(phase);
 698  0
             ret.addAll(analyzerList);
 699  
         }
 700  0
         return ret;
 701  
     }
 702  
 
 703  
     /**
 704  
      * Checks all analyzers to see if an extension is supported.
 705  
      *
 706  
      * @param file a file extension
 707  
      * @return true or false depending on whether or not the file extension is
 708  
      * supported
 709  
      */
 710  
     @Override
 711  
     public boolean accept(File file) {
 712  858
         if (file == null) {
 713  0
             return false;
 714  
         }
 715  858
         boolean scan = false;
 716  858
         for (FileTypeAnalyzer a : this.fileTypeAnalyzers) {
 717  
             /* note, we can't break early on this loop as the analyzers need to know if
 718  
              they have files to work on prior to initialization */
 719  15447
             scan |= a.accept(file);
 720  15447
         }
 721  858
         return scan;
 722  
     }
 723  
 
 724  
     /**
 725  
      * Returns the set of file type analyzers.
 726  
      *
 727  
      * @return the set of file type analyzers
 728  
      */
 729  
     public Set<FileTypeAnalyzer> getFileTypeAnalyzers() {
 730  0
         return this.fileTypeAnalyzers;
 731  
     }
 732  
 
 733  
     /**
 734  
      * Adds a file type analyzer. This has been added solely to assist in unit
 735  
      * testing the Engine.
 736  
      *
 737  
      * @param fta the file type analyzer to add
 738  
      */
 739  
     protected void addFileTypeAnalyzer(FileTypeAnalyzer fta) {
 740  1
         this.fileTypeAnalyzers.add(fta);
 741  1
     }
 742  
 
 743  
     /**
 744  
      * Checks the CPE Index to ensure documents exists. If none exist a
 745  
      * NoDataException is thrown.
 746  
      *
 747  
      * @throws NoDataException thrown if no data exists in the CPE Index
 748  
      * @throws DatabaseException thrown if there is an exception opening the
 749  
      * database
 750  
      */
 751  
     private void ensureDataExists() throws NoDataException, DatabaseException {
 752  2
         final CveDB cve = new CveDB();
 753  
         try {
 754  2
             cve.open();
 755  2
             if (!cve.dataExists()) {
 756  0
                 throw new NoDataException("No documents exist");
 757  
             }
 758  0
         } catch (DatabaseException ex) {
 759  0
             throw new NoDataException(ex.getMessage(), ex);
 760  
         } finally {
 761  2
             cve.close();
 762  2
         }
 763  2
     }
 764  
 
 765  
     /**
 766  
      * Constructs and throws a fatal exception collection.
 767  
      *
 768  
      * @param message the exception message
 769  
      * @param throwable the cause
 770  
      * @param exceptions a collection of exception to include
 771  
      * @throws ExceptionCollection a collection of exceptions that occurred
 772  
      * during analysis
 773  
      */
 774  
     private void throwFatalExceptionCollection(String message, Throwable throwable, List<Throwable> exceptions) throws ExceptionCollection {
 775  1
         LOGGER.error("{}\n\n{}", throwable.getMessage(), message);
 776  1
         LOGGER.debug("", throwable);
 777  1
         exceptions.add(throwable);
 778  1
         throw new ExceptionCollection(message, exceptions, true);
 779  
     }
 780  
 }