View Javadoc

    1   /*
    2    * This file is part of dependency-check-core.
    3    *
    4    * Dependency-check-core is free software: you can redistribute it and/or modify it
    5    * under the terms of the GNU General Public License as published by the Free
    6    * Software Foundation, either version 3 of the License, or (at your option) any
    7    * later version.
    8    *
    9    * Dependency-check-core is distributed in the hope that it will be useful, but
    10   * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
    11   * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
    12   * details.
    13   *
    14   * You should have received a copy of the GNU General Public License along with
    15   * dependency-check-core. If not, see http://www.gnu.org/licenses/.
    16   *
    17   * Copyright (c) 2012 Jeremy Long. All Rights Reserved.
    18   */
    19  package org.owasp.dependencycheck;
    20
    21  import java.util.EnumMap;
    22  import java.io.File;
    23  import java.io.IOException;
    24  import java.util.ArrayList;
    25  import java.util.HashSet;
    26  import java.util.Iterator;
    27  import java.util.List;
    28  import java.util.Set;
    29  import java.util.logging.Level;
    30  import java.util.logging.Logger;
    31  import org.owasp.dependencycheck.analyzer.AnalysisException;
    32  import org.owasp.dependencycheck.analyzer.AnalysisPhase;
    33  import org.owasp.dependencycheck.analyzer.Analyzer;
    34  import org.owasp.dependencycheck.analyzer.AnalyzerService;
    35  import org.owasp.dependencycheck.data.CachedWebDataSource;
    36  import org.owasp.dependencycheck.data.NoDataException;
    37  import org.owasp.dependencycheck.data.UpdateException;
    38  import org.owasp.dependencycheck.data.UpdateService;
    39  import org.owasp.dependencycheck.data.cpe.CpeIndexReader;
    40  import org.owasp.dependencycheck.dependency.Dependency;
    41  import org.owasp.dependencycheck.utils.FileUtils;
    42  import org.owasp.dependencycheck.utils.InvalidSettingException;
    43  import org.owasp.dependencycheck.utils.Settings;
    44
    45  /**
    46   * Scans files, directories, etc. for Dependencies. Analyzers are loaded and
    47   * used to process the files found by the scan, if a file is encountered and an
    48   * Analyzer is associated with the file type then the file is turned into a
    49   * dependency.
    50   *
    51   * @author Jeremy Long (jeremy.long@owasp.org)
    52   */
    53  public class Engine {
    54
    55      /**
    56       * The list of dependencies.
    57       */
    58      private final List<Dependency> dependencies = new ArrayList<Dependency>();
    59      /**
    60       * A Map of analyzers grouped by Analysis phase.
    61       */
    62      private final EnumMap<AnalysisPhase, List<Analyzer>> analyzers =
    63              new EnumMap<AnalysisPhase, List<Analyzer>>(AnalysisPhase.class);
    64      /**
    65       * A set of extensions supported by the analyzers.
    66       */
    67      private final Set<String> extensions = new HashSet<String>();
    68
    69      /**
    70       * Creates a new Engine.
    71       */
    72      public Engine() {
    73          boolean autoUpdate = true;
    74          try {
    75              autoUpdate = Settings.getBoolean(Settings.KEYS.AUTO_UPDATE);
    76          } catch (InvalidSettingException ex) {
    77              Logger.getLogger(Engine.class.getName()).log(Level.FINE, "Invalid setting for auto-update; using true.");
    78          }
    79          if (autoUpdate) {
    80              doUpdates();
    81          }
    82          loadAnalyzers();
    83      }
    84
    85      /**
    86       * Creates a new Engine.
    87       *
    88       * @param autoUpdate indicates whether or not data should be updated from
    89       * the Internet
    90       * @deprecated This function should no longer be used; the autoupdate flag
    91       * should be set using:
    92       * <code>Settings.setBoolean(Settings.KEYS.AUTO_UPDATE, value);</code>
    93       */
    94      @Deprecated
    95      public Engine(boolean autoUpdate) {
    96          if (autoUpdate) {
    97              doUpdates();
    98          }
    99          loadAnalyzers();
    100     }
    101
    102     /**
    103      * Loads the analyzers specified in the configuration file (or system
    104      * properties).
    105      */
    106     private void loadAnalyzers() {
    107
    108         for (AnalysisPhase phase : AnalysisPhase.values()) {
    109             analyzers.put(phase, new ArrayList<Analyzer>());
    110         }
    111
    112         final AnalyzerService service = AnalyzerService.getInstance();
    113         final Iterator<Analyzer> iterator = service.getAnalyzers();
    114         while (iterator.hasNext()) {
    115             final Analyzer a = iterator.next();
    116             analyzers.get(a.getAnalysisPhase()).add(a);
    117             if (a.getSupportedExtensions() != null) {
    118                 extensions.addAll(a.getSupportedExtensions());
    119             }
    120         }
    121     }
    122
    123     /**
    124      * Get the List of the analyzers for a specific phase of analysis.
    125      *
    126      * @param phase the phase to get the configured analyzers.
    127      * @return the analyzers loaded
    128      */
    129     public List<Analyzer> getAnalyzers(AnalysisPhase phase) {
    130         return analyzers.get(phase);
    131     }
    132
    133     /**
    134      * Get the dependencies identified.
    135      *
    136      * @return the dependencies identified
    137      */
    138     public List<Dependency> getDependencies() {
    139         return dependencies;
    140     }
    141
    142     /**
    143      * Scans an array of files or directories. If a directory is specified, it
    144      * will be scanned recursively. Any dependencies identified are added to the
    145      * dependency collection.
    146      *
    147      * @since v0.3.2.5
    148      *
    149      * @param paths an array of paths to files or directories to be analyzed.
    150      */
    151     public void scan(String[] paths) {
    152         for (String path : paths) {
    153             final File file = new File(path);
    154             scan(file);
    155         }
    156     }
    157
    158     /**
    159      * Scans a given file or directory. If a directory is specified, it will be
    160      * scanned recursively. Any dependencies identified are added to the
    161      * dependency collection.
    162      *
    163      * @param path the path to a file or directory to be analyzed.
    164      */
    165     public void scan(String path) {
    166         final File file = new File(path);
    167         scan(file);
    168     }
    169
    170     /**
    171      * Scans an array of files or directories. If a directory is specified, it
    172      * will be scanned recursively. Any dependencies identified are added to the
    173      * dependency collection.
    174      *
    175      * @since v0.3.2.5
    176      *
    177      * @param files an array of paths to files or directories to be analyzed.
    178      */
    179     public void scan(File[] files) {
    180         for (File file : files) {
    181             scan(file);
    182         }
    183     }
    184
    185     /**
    186      * Scans a list of files or directories. If a directory is specified, it
    187      * will be scanned recursively. Any dependencies identified are added to the
    188      * dependency collection.
    189      *
    190      * @since v0.3.2.5
    191      *
    192      * @param files a set of paths to files or directories to be analyzed.
    193      */
    194     public void scan(Set<File> files) {
    195         for (File file : files) {
    196             scan(file);
    197         }
    198     }
    199
    200     /**
    201      * Scans a list of files or directories. If a directory is specified, it
    202      * will be scanned recursively. Any dependencies identified are added to the
    203      * dependency collection.
    204      *
    205      * @since v0.3.2.5
    206      *
    207      * @param files a set of paths to files or directories to be analyzed.
    208      */
    209     public void scan(List<File> files) {
    210         for (File file : files) {
    211             scan(file);
    212         }
    213     }
    214
    215     /**
    216      * Scans a given file or directory. If a directory is specified, it will be
    217      * scanned recursively. Any dependencies identified are added to the
    218      * dependency collection.
    219      *
    220      * @since v0.3.2.4
    221      *
    222      * @param file the path to a file or directory to be analyzed.
    223      */
    224     public void scan(File file) {
    225         if (file.exists()) {
    226             if (file.isDirectory()) {
    227                 scanDirectory(file);
    228             } else {
    229                 scanFile(file);
    230             }
    231         }
    232     }
    233
    234     /**
    235      * Recursively scans files and directories. Any dependencies identified are
    236      * added to the dependency collection.
    237      *
    238      * @param dir the directory to scan.
    239      */
    240     protected void scanDirectory(File dir) {
    241         final File[] files = dir.listFiles();
    242         if (files != null) {
    243             for (File f : files) {
    244                 if (f.isDirectory()) {
    245                     scanDirectory(f);
    246                 } else {
    247                     scanFile(f);
    248                 }
    249             }
    250         }
    251     }
    252
    253     /**
    254      * Scans a specified file. If a dependency is identified it is added to the
    255      * dependency collection.
    256      *
    257      * @param file The file to scan.
    258      */
    259     protected void scanFile(File file) {
    260         if (!file.isFile()) {
    261             final String msg = String.format("Path passed to scanFile(File) is not a file: %s. Skipping the file.", file.toString());
    262             Logger.getLogger(Engine.class.getName()).log(Level.FINE, msg);
    263             return;
    264         }
    265         final String fileName = file.getName();
    266         final String extension = FileUtils.getFileExtension(fileName);
    267         if (extension != null) {
    268             if (extensions.contains(extension)) {
    269                 final Dependency dependency = new Dependency(file);
    270                 dependencies.add(dependency);
    271             }
    272         } else {
    273             final String msg = String.format("No file extension found on file '%s'. The file was not analyzed.",
    274                     file.toString());
    275             Logger.getLogger(Engine.class.getName()).log(Level.FINEST, msg);
    276         }
    277     }
    278
    279     /**
    280      * Runs the analyzers against all of the dependencies.
    281      */
    282     public void analyzeDependencies() {
    283         //need to ensure that data exists
    284         try {
    285             ensureDataExists();
    286         } catch (NoDataException ex) {
    287             final String msg = String.format("%n%n%s%n%nUnable to continue dependency-check analysis.", ex.getMessage());
    288             Logger.getLogger(Engine.class.getName()).log(Level.SEVERE, msg);
    289             Logger.getLogger(Engine.class.getName()).log(Level.FINE, null, ex);
    290             return;
    291         }
    292
    293         //phase one initialize
    294         for (AnalysisPhase phase : AnalysisPhase.values()) {
    295             final List<Analyzer> analyzerList = analyzers.get(phase);
    296             for (Analyzer a : analyzerList) {
    297                 try {
    298                     final String msg = String.format("Initializing %s", a.getName());
    299                     Logger.getLogger(Engine.class.getName()).log(Level.FINE, msg);
    300                     a.initialize();
    301                 } catch (Exception ex) {
    302                     final String msg = String.format("Exception occurred initializing %s.", a.getName());
    303                     Logger.getLogger(Engine.class.getName()).log(Level.SEVERE, msg);
    304                     Logger.getLogger(Engine.class.getName()).log(Level.INFO, null, ex);
    305                     try {
    306                         a.close();
    307                     } catch (Exception ex1) {
    308                         Logger.getLogger(Engine.class.getName()).log(Level.FINEST, null, ex1);
    309                     }
    310                 }
    311             }
    312         }
    313
    314         // analysis phases
    315         for (AnalysisPhase phase : AnalysisPhase.values()) {
    316             final List<Analyzer> analyzerList = analyzers.get(phase);
    317
    318             for (Analyzer a : analyzerList) {
    319                 /* need to create a copy of the collection because some of the
    320                  * analyzers may modify it. This prevents ConcurrentModificationExceptions.
    321                  * This is okay for adds/deletes because it happens per analyzer.
    322                  */
    323                 final String msg = String.format("Begin Analyzer '%s'", a.getName());
    324                 Logger.getLogger(Engine.class.getName()).log(Level.FINE, msg);
    325                 final Set<Dependency> dependencySet = new HashSet<Dependency>();
    326                 dependencySet.addAll(dependencies);
    327                 for (Dependency d : dependencySet) {
    328                     final String msgFile = String.format("Begin Analysis of '%s'", d.getActualFilePath());
    329                     Logger.getLogger(Engine.class.getName()).log(Level.FINE, msgFile);
    330                     if (a.supportsExtension(d.getFileExtension())) {
    331                         try {
    332                             a.analyze(d, this);
    333                         } catch (AnalysisException ex) {
    334                             d.addAnalysisException(ex);
    335                         }
    336                     }
    337                 }
    338             }
    339         }
    340
    341         //close/cleanup
    342         for (AnalysisPhase phase : AnalysisPhase.values()) {
    343             final List<Analyzer> analyzerList = analyzers.get(phase);
    344             for (Analyzer a : analyzerList) {
    345                 final String msg = String.format("Closing Analyzer '%s'", a.getName());
    346                 Logger.getLogger(Engine.class.getName()).log(Level.FINE, msg);
    347                 try {
    348                     a.close();
    349                 } catch (Exception ex) {
    350                     Logger.getLogger(Engine.class.getName()).log(Level.FINEST, null, ex);
    351                 }
    352             }
    353         }
    354     }
    355
    356     /**
    357      * Cycles through the cached web data sources and calls update on all of
    358      * them.
    359      */
    360     private void doUpdates() {
    361         final UpdateService service = UpdateService.getInstance();
    362         final Iterator<CachedWebDataSource> iterator = service.getDataSources();
    363         while (iterator.hasNext()) {
    364             final CachedWebDataSource source = iterator.next();
    365             try {
    366                 source.update();
    367             } catch (UpdateException ex) {
    368                 Logger.getLogger(Engine.class.getName()).log(Level.WARNING,
    369                         "Unable to update Cached Web DataSource, using local data instead. Results may not include recent vulnerabilities.");
    370                 Logger.getLogger(Engine.class.getName()).log(Level.FINE,
    371                         String.format("Unable to update details for %s", source.getClass().getName()), ex);
    372             }
    373         }
    374     }
    375
    376     /**
    377      * Returns a full list of all of the analyzers. This is useful for reporting
    378      * which analyzers where used.
    379      *
    380      * @return a list of Analyzers
    381      */
    382     public List<Analyzer> getAnalyzers() {
    383         final List<Analyzer> ret = new ArrayList<Analyzer>();
    384         for (AnalysisPhase phase : AnalysisPhase.values()) {
    385             final List<Analyzer> analyzerList = analyzers.get(phase);
    386             ret.addAll(analyzerList);
    387         }
    388         return ret;
    389     }
    390
    391     /**
    392      * Checks all analyzers to see if an extension is supported.
    393      *
    394      * @param ext a file extension
    395      * @return true or false depending on whether or not the file extension is
    396      * supported
    397      */
    398     public boolean supportsExtension(String ext) {
    399         if (ext == null) {
    400             return false;
    401         }
    402         for (AnalysisPhase phase : AnalysisPhase.values()) {
    403             final List<Analyzer> analyzerList = analyzers.get(phase);
    404             for (Analyzer a : analyzerList) {
    405                 if (a.getSupportedExtensions() != null && a.supportsExtension(ext)) {
    406                     return true;
    407                 }
    408             }
    409         }
    410         return false;
    411     }
    412
    413     /**
    414      * Checks the CPE Index to ensure documents exists. If none exist a
    415      * NoDataException is thrown.
    416      *
    417      * @throws NoDataException thrown if no data exists in the CPE Index
    418      */
    419     private void ensureDataExists() throws NoDataException {
    420         CpeIndexReader cpe = null;
    421         boolean noDataExists = false;
    422         try {
    423             cpe = new CpeIndexReader();
    424             cpe.open();
    425             if (cpe.numDocs() <= 0) {
    426                 noDataExists = true;
    427             }
    428         } catch (IOException ex) {
    429             noDataExists = true;
    430         } catch (NullPointerException ex) {
    431             noDataExists = true;
    432         } finally {
    433             if (cpe != null) {
    434                 cpe.close();
    435             }
    436         }
    437         if (noDataExists) {
    438             throw new NoDataException("No data exists in the data store. Please check that you are able to connect "
    439                     + "to the Internet and re-run dependency-check. If the problem persists determine whether you need "
    440                     + "to set a proxy url and port.\\n\\nIf you are unable to solve this problem please contact the mailing "
    441                     + "list for help: dependency-check@googlegroups.com");
    442
    443         }
    444     }
    445 }