1 /*
2 * This file is part of dependency-check-core.
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 *
16 * Copyright (c) 2012 Jeremy Long. All Rights Reserved.
17 */
18 package org.owasp.dependencycheck.analyzer;
19
20 import java.io.File;
21 import java.util.HashSet;
22 import java.util.Iterator;
23 import java.util.ListIterator;
24 import java.util.Set;
25 import java.util.regex.Matcher;
26 import java.util.regex.Pattern;
27 import org.owasp.dependencycheck.Engine;
28 import org.owasp.dependencycheck.analyzer.exception.AnalysisException;
29 import org.owasp.dependencycheck.dependency.Dependency;
30 import org.owasp.dependencycheck.dependency.Identifier;
31 import org.owasp.dependencycheck.utils.DependencyVersion;
32 import org.owasp.dependencycheck.utils.DependencyVersionUtil;
33 import org.slf4j.Logger;
34 import org.slf4j.LoggerFactory;
35
36 /**
37 * <p>
38 * This analyzer ensures dependencies that should be grouped together, to remove
39 * excess noise from the report, are grouped. An example would be Spring, Spring
40 * Beans, Spring MVC, etc. If they are all for the same version and have the
41 * same relative path then these should be grouped into a single dependency
42 * under the core/main library.</p>
43 * <p>
44 * Note, this grouping only works on dependencies with identified CVE
45 * entries</p>
46 *
47 * @author Jeremy Long
48 */
49 public class DependencyBundlingAnalyzer extends AbstractAnalyzer {
50
51 /**
52 * The Logger.
53 */
54 private static final Logger LOGGER = LoggerFactory.getLogger(DependencyBundlingAnalyzer.class);
55
56 //<editor-fold defaultstate="collapsed" desc="Constants and Member Variables">
57 /**
58 * A pattern for obtaining the first part of a filename.
59 */
60 private static final Pattern STARTING_TEXT_PATTERN = Pattern.compile("^[a-zA-Z0-9]*");
61
62 /**
63 * a flag indicating if this analyzer has run. This analyzer only runs once.
64 */
65 private boolean analyzed = false;
66
67 /**
68 * Returns a flag indicating if this analyzer has run. This analyzer only
69 * runs once. Note this is currently only used in the unit tests.
70 *
71 * @return a flag indicating if this analyzer has run. This analyzer only
72 * runs once
73 */
74 protected boolean getAnalyzed() {
75 return analyzed;
76 }
77
78 //</editor-fold>
79 //<editor-fold defaultstate="collapsed" desc="All standard implementation details of Analyzer">
80 /**
81 * The name of the analyzer.
82 */
83 private static final String ANALYZER_NAME = "Dependency Bundling Analyzer";
84 /**
85 * The phase that this analyzer is intended to run in.
86 */
87 private static final AnalysisPhase ANALYSIS_PHASE = AnalysisPhase.PRE_FINDING_ANALYSIS;
88
89 /**
90 * Returns the name of the analyzer.
91 *
92 * @return the name of the analyzer.
93 */
94 @Override
95 public String getName() {
96 return ANALYZER_NAME;
97 }
98
99 /**
100 * Returns the phase that the analyzer is intended to run in.
101 *
102 * @return the phase that the analyzer is intended to run in.
103 */
104 @Override
105 public AnalysisPhase getAnalysisPhase() {
106 return ANALYSIS_PHASE;
107 }
108 //</editor-fold>
109
110 /**
111 * Does not support parallel processing as it only runs once and then
112 * operates on <em>all</em> dependencies.
113 *
114 * @return whether or not parallel processing is enabled
115 * @see #analyze(Dependency, Engine)
116 */
117 @Override
118 public boolean supportsParallelProcessing() {
119 return false;
120 }
121
122 /**
123 * Analyzes a set of dependencies. If they have been found to have the same
124 * base path and the same set of identifiers they are likely related. The
125 * related dependencies are bundled into a single reportable item.
126 *
127 * @param ignore this analyzer ignores the dependency being analyzed
128 * @param engine the engine that is scanning the dependencies
129 * @throws AnalysisException is thrown if there is an error reading the JAR
130 * file.
131 */
132 @Override
133 public void analyze(Dependency ignore, Engine engine) throws AnalysisException {
134 if (!analyzed) {
135 analyzed = true;
136 final Set<Dependency> dependenciesToRemove = new HashSet<Dependency>();
137 final ListIterator<Dependency> mainIterator = engine.getDependencies().listIterator();
138 //for (Dependency nextDependency : engine.getDependencies()) {
139 while (mainIterator.hasNext()) {
140 final Dependency dependency = mainIterator.next();
141 if (mainIterator.hasNext() && !dependenciesToRemove.contains(dependency)) {
142 final ListIterator<Dependency> subIterator = engine.getDependencies().listIterator(mainIterator.nextIndex());
143 while (subIterator.hasNext()) {
144 final Dependency nextDependency = subIterator.next();
145 Dependency main = null;
146 if (hashesMatch(dependency, nextDependency) && !containedInWar(dependency.getFilePath())
147 && !containedInWar(nextDependency.getFilePath())) {
148 if (firstPathIsShortest(dependency.getFilePath(), nextDependency.getFilePath())) {
149 mergeDependencies(dependency, nextDependency, dependenciesToRemove);
150 } else {
151 mergeDependencies(nextDependency, dependency, dependenciesToRemove);
152 break; //since we merged into the next dependency - skip forward to the next in mainIterator
153 }
154 } else if (isShadedJar(dependency, nextDependency)) {
155 if (dependency.getFileName().toLowerCase().endsWith("pom.xml")) {
156 mergeDependencies(nextDependency, dependency, dependenciesToRemove);
157 nextDependency.getRelatedDependencies().remove(dependency);
158 break;
159 } else {
160 mergeDependencies(dependency, nextDependency, dependenciesToRemove);
161 dependency.getRelatedDependencies().remove(nextDependency);
162 }
163 } else if (cpeIdentifiersMatch(dependency, nextDependency)
164 && hasSameBasePath(dependency, nextDependency)
165 && fileNameMatch(dependency, nextDependency)) {
166 if (isCore(dependency, nextDependency)) {
167 mergeDependencies(dependency, nextDependency, dependenciesToRemove);
168 } else {
169 mergeDependencies(nextDependency, dependency, dependenciesToRemove);
170 break; //since we merged into the next dependency - skip forward to the next in mainIterator
171 }
172 } else if ((main = getMainGemspecDependency(dependency, nextDependency)) != null) {
173 if (main == dependency) {
174 mergeDependencies(dependency, nextDependency, dependenciesToRemove);
175 } else {
176 mergeDependencies(nextDependency, dependency, dependenciesToRemove);
177 break; //since we merged into the next dependency - skip forward to the next in mainIterator
178 }
179 } else if ((main = getMainSwiftDependency(dependency, nextDependency)) != null) {
180 if (main == dependency) {
181 mergeDependencies(dependency, nextDependency, dependenciesToRemove);
182 } else {
183 mergeDependencies(nextDependency, dependency, dependenciesToRemove);
184 break; //since we merged into the next dependency - skip forward to the next in mainIterator
185 }
186 }
187 }
188 }
189 }
190 //removing dependencies here as ensuring correctness and avoiding ConcurrentUpdateExceptions
191 // was difficult because of the inner iterator.
192 engine.getDependencies().removeAll(dependenciesToRemove);
193 }
194 }
195
196 /**
197 * Adds the relatedDependency to the dependency's related dependencies.
198 *
199 * @param dependency the main dependency
200 * @param relatedDependency a collection of dependencies to be removed from
201 * the main analysis loop, this is the source of dependencies to remove
202 * @param dependenciesToRemove a collection of dependencies that will be
203 * removed from the main analysis loop, this function adds to this
204 * collection
205 */
206 private void mergeDependencies(final Dependency dependency, final Dependency relatedDependency, final Set<Dependency> dependenciesToRemove) {
207 dependency.addRelatedDependency(relatedDependency);
208 final Iterator<Dependency> i = relatedDependency.getRelatedDependencies().iterator();
209 while (i.hasNext()) {
210 dependency.addRelatedDependency(i.next());
211 i.remove();
212 }
213 if (dependency.getSha1sum().equals(relatedDependency.getSha1sum())) {
214 dependency.addAllProjectReferences(relatedDependency.getProjectReferences());
215 }
216 dependenciesToRemove.add(relatedDependency);
217 }
218
219 /**
220 * Attempts to trim a maven repo to a common base path. This is typically
221 * [drive]\[repo_location]\repository\[path1]\[path2].
222 *
223 * @param path the path to trim
224 * @return a string representing the base path.
225 */
226 private String getBaseRepoPath(final String path) {
227 int pos = path.indexOf("repository" + File.separator) + 11;
228 if (pos < 0) {
229 return path;
230 }
231 int tmp = path.indexOf(File.separator, pos);
232 if (tmp <= 0) {
233 return path;
234 }
235 if (tmp > 0) {
236 pos = tmp + 1;
237 }
238 tmp = path.indexOf(File.separator, pos);
239 if (tmp > 0) {
240 pos = tmp + 1;
241 }
242 return path.substring(0, pos);
243 }
244
245 /**
246 * Returns true if the file names (and version if it exists) of the two
247 * dependencies are sufficiently similar.
248 *
249 * @param dependency1 a dependency2 to compare
250 * @param dependency2 a dependency2 to compare
251 * @return true if the identifiers in the two supplied dependencies are
252 * equal
253 */
254 private boolean fileNameMatch(Dependency dependency1, Dependency dependency2) {
255 if (dependency1 == null || dependency1.getFileName() == null
256 || dependency2 == null || dependency2.getFileName() == null) {
257 return false;
258 }
259 final String fileName1 = dependency1.getActualFile().getName();
260 final String fileName2 = dependency2.getActualFile().getName();
261
262 //version check
263 final DependencyVersion version1 = DependencyVersionUtil.parseVersion(fileName1);
264 final DependencyVersion version2 = DependencyVersionUtil.parseVersion(fileName2);
265 if (version1 != null && version2 != null && !version1.equals(version2)) {
266 return false;
267 }
268
269 //filename check
270 final Matcher match1 = STARTING_TEXT_PATTERN.matcher(fileName1);
271 final Matcher match2 = STARTING_TEXT_PATTERN.matcher(fileName2);
272 if (match1.find() && match2.find()) {
273 return match1.group().equals(match2.group());
274 }
275
276 return false;
277 }
278
279 /**
280 * Returns true if the CPE identifiers in the two supplied dependencies are
281 * equal.
282 *
283 * @param dependency1 a dependency2 to compare
284 * @param dependency2 a dependency2 to compare
285 * @return true if the identifiers in the two supplied dependencies are
286 * equal
287 */
288 private boolean cpeIdentifiersMatch(Dependency dependency1, Dependency dependency2) {
289 if (dependency1 == null || dependency1.getIdentifiers() == null
290 || dependency2 == null || dependency2.getIdentifiers() == null) {
291 return false;
292 }
293 boolean matches = false;
294 int cpeCount1 = 0;
295 int cpeCount2 = 0;
296 for (Identifier i : dependency1.getIdentifiers()) {
297 if ("cpe".equals(i.getType())) {
298 cpeCount1 += 1;
299 }
300 }
301 for (Identifier i : dependency2.getIdentifiers()) {
302 if ("cpe".equals(i.getType())) {
303 cpeCount2 += 1;
304 }
305 }
306 if (cpeCount1 > 0 && cpeCount1 == cpeCount2) {
307 for (Identifier i : dependency1.getIdentifiers()) {
308 if ("cpe".equals(i.getType())) {
309 matches |= dependency2.getIdentifiers().contains(i);
310 if (!matches) {
311 break;
312 }
313 }
314 }
315 }
316 LOGGER.debug("IdentifiersMatch={} ({}, {})", matches, dependency1.getFileName(), dependency2.getFileName());
317 return matches;
318 }
319
320 /**
321 * Determines if the two dependencies have the same base path.
322 *
323 * @param dependency1 a Dependency object
324 * @param dependency2 a Dependency object
325 * @return true if the base paths of the dependencies are identical
326 */
327 private boolean hasSameBasePath(Dependency dependency1, Dependency dependency2) {
328 if (dependency1 == null || dependency2 == null) {
329 return false;
330 }
331 final File lFile = new File(dependency1.getFilePath());
332 String left = lFile.getParent();
333 final File rFile = new File(dependency2.getFilePath());
334 String right = rFile.getParent();
335 if (left == null) {
336 return right == null;
337 } else if (right == null) {
338 return false;
339 }
340 if (left.equalsIgnoreCase(right)) {
341 return true;
342 }
343
344 if (left.matches(".*[/\\\\]repository[/\\\\].*") && right.matches(".*[/\\\\]repository[/\\\\].*")) {
345 left = getBaseRepoPath(left);
346 right = getBaseRepoPath(right);
347 }
348 if (left.equalsIgnoreCase(right)) {
349 return true;
350 }
351 //new code
352 for (Dependency child : dependency2.getRelatedDependencies()) {
353 if (hasSameBasePath(dependency1, child)) {
354 return true;
355 }
356 }
357 return false;
358 }
359
360 /**
361 * Bundling Ruby gems that are identified from different .gemspec files but
362 * denote the same package path. This happens when Ruby bundler installs an
363 * application's dependencies by running "bundle install".
364 *
365 * @param dependency1 dependency to compare
366 * @param dependency2 dependency to compare
367 * @return true if the the dependencies being analyzed appear to be the
368 * same; otherwise false
369 */
370 private boolean isSameRubyGem(Dependency dependency1, Dependency dependency2) {
371 if (dependency1 == null || dependency2 == null
372 || !dependency1.getFileName().endsWith(".gemspec")
373 || !dependency2.getFileName().endsWith(".gemspec")
374 || dependency1.getPackagePath() == null
375 || dependency2.getPackagePath() == null) {
376 return false;
377 }
378 return dependency1.getPackagePath().equalsIgnoreCase(dependency2.getPackagePath());
379 }
380
381 /**
382 * Ruby gems installed by "bundle install" can have zero or more *.gemspec
383 * files, all of which have the same packagePath and should be grouped. If
384 * one of these gemspec is from <parent>/specifications/*.gemspec, because
385 * it is a stub with fully resolved gem meta-data created by Ruby bundler,
386 * this dependency should be the main one. Otherwise, use dependency2 as
387 * main.
388 *
389 * This method returns null if any dependency is not from *.gemspec, or the
390 * two do not have the same packagePath. In this case, they should not be
391 * grouped.
392 *
393 * @param dependency1 dependency to compare
394 * @param dependency2 dependency to compare
395 * @return the main dependency; or null if a gemspec is not included in the
396 * analysis
397 */
398 private Dependency getMainGemspecDependency(Dependency dependency1, Dependency dependency2) {
399 if (isSameRubyGem(dependency1, dependency2)) {
400 final File lFile = dependency1.getActualFile();
401 final File left = lFile.getParentFile();
402 if (left != null && left.getName().equalsIgnoreCase("specifications")) {
403 return dependency1;
404 }
405 return dependency2;
406 }
407 return null;
408 }
409
410 /**
411 * Bundling same swift dependencies with the same packagePath but identified
412 * by different analyzers.
413 *
414 * @param dependency1 dependency to test
415 * @param dependency2 dependency to test
416 * @return <code>true</code> if the dependencies appear to be the same;
417 * otherwise <code>false</code>
418 */
419 private boolean isSameSwiftPackage(Dependency dependency1, Dependency dependency2) {
420 if (dependency1 == null || dependency2 == null
421 || (!dependency1.getFileName().endsWith(".podspec")
422 && !dependency1.getFileName().equals("Package.swift"))
423 || (!dependency2.getFileName().endsWith(".podspec")
424 && !dependency2.getFileName().equals("Package.swift"))
425 || dependency1.getPackagePath() == null
426 || dependency2.getPackagePath() == null) {
427 return false;
428 }
429 return dependency1.getPackagePath().equalsIgnoreCase(dependency2.getPackagePath());
430 }
431
432 /**
433 * Determines which of the swift dependencies should be considered the
434 * primary.
435 *
436 * @param dependency1 the first swift dependency to compare
437 * @param dependency2 the second swift dependency to compare
438 * @return the primary swift dependency
439 */
440 private Dependency getMainSwiftDependency(Dependency dependency1, Dependency dependency2) {
441 if (isSameSwiftPackage(dependency1, dependency2)) {
442 if (dependency1.getFileName().endsWith(".podspec")) {
443 return dependency1;
444 }
445 return dependency2;
446 }
447 return null;
448 }
449
450 /**
451 * This is likely a very broken attempt at determining if the 'left'
452 * dependency is the 'core' library in comparison to the 'right' library.
453 *
454 * @param left the dependency to test
455 * @param right the dependency to test against
456 * @return a boolean indicating whether or not the left dependency should be
457 * considered the "core" version.
458 */
459 boolean isCore(Dependency left, Dependency right) {
460 final String leftName = left.getFileName().toLowerCase();
461 final String rightName = right.getFileName().toLowerCase();
462
463 final boolean returnVal;
464 if (!rightName.matches(".*\\.(tar|tgz|gz|zip|ear|war).+") && leftName.matches(".*\\.(tar|tgz|gz|zip|ear|war).+")
465 || rightName.contains("core") && !leftName.contains("core")
466 || rightName.contains("kernel") && !leftName.contains("kernel")) {
467 returnVal = false;
468 } else if (rightName.matches(".*\\.(tar|tgz|gz|zip|ear|war).+") && !leftName.matches(".*\\.(tar|tgz|gz|zip|ear|war).+")
469 || !rightName.contains("core") && leftName.contains("core")
470 || !rightName.contains("kernel") && leftName.contains("kernel")) {
471 returnVal = true;
472 // } else if (leftName.matches(".*struts2\\-core.*") && rightName.matches(".*xwork\\-core.*")) {
473 // returnVal = true;
474 // } else if (rightName.matches(".*struts2\\-core.*") && leftName.matches(".*xwork\\-core.*")) {
475 // returnVal = false;
476 } else {
477 /*
478 * considered splitting the names up and comparing the components,
479 * but decided that the file name length should be sufficient as the
480 * "core" component, if this follows a normal naming protocol should
481 * be shorter:
482 * axis2-saaj-1.4.1.jar
483 * axis2-1.4.1.jar <-----
484 * axis2-kernel-1.4.1.jar
485 */
486 returnVal = leftName.length() <= rightName.length();
487 }
488 LOGGER.debug("IsCore={} ({}, {})", returnVal, left.getFileName(), right.getFileName());
489 return returnVal;
490 }
491
492 /**
493 * Compares the SHA1 hashes of two dependencies to determine if they are
494 * equal.
495 *
496 * @param dependency1 a dependency object to compare
497 * @param dependency2 a dependency object to compare
498 * @return true if the sha1 hashes of the two dependencies match; otherwise
499 * false
500 */
501 private boolean hashesMatch(Dependency dependency1, Dependency dependency2) {
502 if (dependency1 == null || dependency2 == null || dependency1.getSha1sum() == null || dependency2.getSha1sum() == null) {
503 return false;
504 }
505 return dependency1.getSha1sum().equals(dependency2.getSha1sum());
506 }
507
508 /**
509 * Determines if the jar is shaded and the created pom.xml identified the
510 * same CPE as the jar - if so, the pom.xml dependency should be removed.
511 *
512 * @param dependency a dependency to check
513 * @param nextDependency another dependency to check
514 * @return true if on of the dependencies is a pom.xml and the identifiers
515 * between the two collections match; otherwise false
516 */
517 private boolean isShadedJar(Dependency dependency, Dependency nextDependency) {
518 final String mainName = dependency.getFileName().toLowerCase();
519 final String nextName = nextDependency.getFileName().toLowerCase();
520 if (mainName.endsWith(".jar") && nextName.endsWith("pom.xml")) {
521 return dependency.getIdentifiers().containsAll(nextDependency.getIdentifiers());
522 } else if (nextName.endsWith(".jar") && mainName.endsWith("pom.xml")) {
523 return nextDependency.getIdentifiers().containsAll(dependency.getIdentifiers());
524 }
525 return false;
526 }
527
528 /**
529 * Determines which path is shortest; if path lengths are equal then we use
530 * compareTo of the string method to determine if the first path is smaller.
531 *
532 * @param left the first path to compare
533 * @param right the second path to compare
534 * @return <code>true</code> if the leftPath is the shortest; otherwise
535 * <code>false</code>
536 */
537 protected boolean firstPathIsShortest(String left, String right) {
538 if (left.contains("dctemp")) {
539 return false;
540 }
541 final String leftPath = left.replace('\\', '/');
542 final String rightPath = right.replace('\\', '/');
543
544 final int leftCount = countChar(leftPath, '/');
545 final int rightCount = countChar(rightPath, '/');
546 if (leftCount == rightCount) {
547 return leftPath.compareTo(rightPath) <= 0;
548 } else {
549 return leftCount < rightCount;
550 }
551 }
552
553 /**
554 * Counts the number of times the character is present in the string.
555 *
556 * @param string the string to count the characters in
557 * @param c the character to count
558 * @return the number of times the character is present in the string
559 */
560 private int countChar(String string, char c) {
561 int count = 0;
562 final int max = string.length();
563 for (int i = 0; i < max; i++) {
564 if (c == string.charAt(i)) {
565 count++;
566 }
567 }
568 return count;
569 }
570
571 /**
572 * Checks if the given file path is contained within a war or ear file.
573 *
574 * @param filePath the file path to check
575 * @return true if the path contains '.war\' or '.ear\'.
576 */
577 private boolean containedInWar(String filePath) {
578 return filePath == null ? false : filePath.matches(".*\\.(ear|war)[\\\\/].*");
579 }
580 }