From b91d0863401378a1a8ab3eee8ba744e7f4d6cebe Mon Sep 17 00:00:00 2001 From: Jeremy Long Date: Wed, 20 Dec 2017 06:25:34 -0500 Subject: [PATCH 1/5] updated FP list while working on #632 --- .../src/main/resources/dependencycheck-base-suppression.xml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/dependency-check-core/src/main/resources/dependencycheck-base-suppression.xml b/dependency-check-core/src/main/resources/dependencycheck-base-suppression.xml index be710cd9c..71894b128 100644 --- a/dependency-check-core/src/main/resources/dependencycheck-base-suppression.xml +++ b/dependency-check-core/src/main/resources/dependencycheck-base-suppression.xml @@ -64,6 +64,7 @@ 8. Context project is drupal plugin 9. mail_project is ruby library 10. ldap_project is part of type3 written in php + 11. user import project is used in drupal (i.e. php) ]]> .*(\.(dll|jar|ear|war|pom|nupkg|nuspec)|pom\.xml|package.json)$ cpe:/a:sandbox:sandbox @@ -79,6 +80,7 @@ cpe:/a:context_project:context cpe:/a:mail_project:mail cpe:/a:ldap_project:ldap + cpe:/a:user_import_project:user_import Date: Wed, 20 Dec 2017 06:26:03 -0500 Subject: [PATCH 2/5] added a max length to limit query parse issues --- .../dependencycheck/analyzer/CPEAnalyzer.java | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/dependency-check-core/src/main/java/org/owasp/dependencycheck/analyzer/CPEAnalyzer.java b/dependency-check-core/src/main/java/org/owasp/dependencycheck/analyzer/CPEAnalyzer.java index 34685d6bf..4769e5600 100644 --- a/dependency-check-core/src/main/java/org/owasp/dependencycheck/analyzer/CPEAnalyzer.java +++ b/dependency-check-core/src/main/java/org/owasp/dependencycheck/analyzer/CPEAnalyzer.java @@ -251,20 +251,20 @@ public class CPEAnalyzer extends AbstractAnalyzer { * @param evidence an iterable set of evidence to concatenate * @return the new evidence text */ + @SuppressWarnings("null") private String addEvidenceWithoutDuplicateTerms(final String text, final Iterable evidence) { final String txt = (text == null) ? "" : text; - final StringBuilder sb = new StringBuilder(); + final StringBuilder sb = new StringBuilder(text.length() * 2); sb.append(' ').append(txt).append(' '); for (Evidence e : evidence) { - final String value = e.getValue(); - //removed as the URLTokenizingFilter was created - //hack to get around the fact that lucene does a really good job of recognizing domains and not splitting them. -// if (value.startsWith("http://")) { -// value = value.substring(7).replaceAll("\\.", " "); -// } -// if (value.startsWith("https://")) { -// value = value.substring(8).replaceAll("\\.", " "); -// } + String value = e.getValue(); + if (value.length() > 1000) { + value = value.substring(0, 1000); + final int pos = value.lastIndexOf(" "); + if (pos > 0) { + value = value.substring(0, pos); + } + } if (sb.indexOf(" " + value + " ") < 0) { sb.append(value).append(' '); } From dd4d1495c112952b39617468655c0c032aaf115b Mon Sep 17 00:00:00 2001 From: Jeremy Long Date: Thu, 21 Dec 2017 05:58:39 -0500 Subject: [PATCH 3/5] cleaned up lucene query construction and added test cases --- .../dependencycheck/analyzer/CPEAnalyzer.java | 17 +- .../analyzer/CPEAnalyzerIT.java | 8 +- .../analyzer/CPEAnalyzerTest.java | 218 ++++++++++++++++++ 3 files changed, 234 insertions(+), 9 deletions(-) create mode 100644 dependency-check-core/src/test/java/org/owasp/dependencycheck/analyzer/CPEAnalyzerTest.java diff --git a/dependency-check-core/src/main/java/org/owasp/dependencycheck/analyzer/CPEAnalyzer.java b/dependency-check-core/src/main/java/org/owasp/dependencycheck/analyzer/CPEAnalyzer.java index 4769e5600..cde34a4d9 100644 --- a/dependency-check-core/src/main/java/org/owasp/dependencycheck/analyzer/CPEAnalyzer.java +++ b/dependency-check-core/src/main/java/org/owasp/dependencycheck/analyzer/CPEAnalyzer.java @@ -252,7 +252,7 @@ public class CPEAnalyzer extends AbstractAnalyzer { * @return the new evidence text */ @SuppressWarnings("null") - private String addEvidenceWithoutDuplicateTerms(final String text, final Iterable evidence) { + protected String addEvidenceWithoutDuplicateTerms(final String text, final Iterable evidence) { final String txt = (text == null) ? "" : text; final StringBuilder sb = new StringBuilder(text.length() * 2); sb.append(' ').append(txt).append(' '); @@ -373,7 +373,7 @@ public class CPEAnalyzer extends AbstractAnalyzer { * @return if the append was successful. */ private boolean appendWeightedSearch(StringBuilder sb, String field, String searchText, Set weightedText) { - sb.append(' ').append(field).append(":( "); + sb.append(field).append(":("); final String cleanText = cleanseText(searchText); @@ -384,6 +384,7 @@ public class CPEAnalyzer extends AbstractAnalyzer { if (weightedText == null || weightedText.isEmpty()) { LuceneUtils.appendEscapedLuceneQuery(sb, cleanText); } else { + boolean addSpace = false; final StringTokenizer tokens = new StringTokenizer(cleanText); while (tokens.hasMoreElements()) { final String word = tokens.nextToken(); @@ -395,14 +396,20 @@ public class CPEAnalyzer extends AbstractAnalyzer { LuceneUtils.appendEscapedLuceneQuery(temp, word); temp.append(WEIGHTING_BOOST); if (!word.equalsIgnoreCase(weightedStr)) { - temp.append(' '); + if (temp.length() > 0) { + temp.append(' '); + } LuceneUtils.appendEscapedLuceneQuery(temp, weightedStr); temp.append(WEIGHTING_BOOST); } break; } } - sb.append(' '); + if (addSpace) { + sb.append(' '); + } else { + addSpace = true; + } if (temp == null) { LuceneUtils.appendEscapedLuceneQuery(sb, word); } else { @@ -410,7 +417,7 @@ public class CPEAnalyzer extends AbstractAnalyzer { } } } - sb.append(" ) "); + sb.append(")"); return true; } diff --git a/dependency-check-core/src/test/java/org/owasp/dependencycheck/analyzer/CPEAnalyzerIT.java b/dependency-check-core/src/test/java/org/owasp/dependencycheck/analyzer/CPEAnalyzerIT.java index 8bf794121..63d9526f5 100644 --- a/dependency-check-core/src/test/java/org/owasp/dependencycheck/analyzer/CPEAnalyzerIT.java +++ b/dependency-check-core/src/test/java/org/owasp/dependencycheck/analyzer/CPEAnalyzerIT.java @@ -59,19 +59,19 @@ public class CPEAnalyzerIT extends BaseDBTestCase { CPEAnalyzer instance = new CPEAnalyzer(); instance.initialize(getSettings()); String queryText = instance.buildSearch(vendor, product, null, null); - String expResult = " product:( struts 2 core ) AND vendor:( apache software foundation ) "; + String expResult = "product:(struts 2 core) AND vendor:(apache software foundation)"; assertTrue(expResult.equals(queryText)); queryText = instance.buildSearch(vendor, product, null, productWeightings); - expResult = " product:( struts^5 struts2^5 2 core ) AND vendor:( apache software foundation ) "; + expResult = "product:(struts^5 struts2^5 2 core) AND vendor:(apache software foundation)"; assertTrue(expResult.equals(queryText)); queryText = instance.buildSearch(vendor, product, vendorWeightings, null); - expResult = " product:( struts 2 core ) AND vendor:( apache^5 software foundation ) "; + expResult = "product:(struts 2 core) AND vendor:(apache^5 software foundation)"; assertTrue(expResult.equals(queryText)); queryText = instance.buildSearch(vendor, product, vendorWeightings, productWeightings); - expResult = " product:( struts^5 struts2^5 2 core ) AND vendor:( apache^5 software foundation ) "; + expResult = "product:(struts^5 struts2^5 2 core) AND vendor:(apache^5 software foundation)"; assertTrue(expResult.equals(queryText)); instance.close(); } diff --git a/dependency-check-core/src/test/java/org/owasp/dependencycheck/analyzer/CPEAnalyzerTest.java b/dependency-check-core/src/test/java/org/owasp/dependencycheck/analyzer/CPEAnalyzerTest.java new file mode 100644 index 000000000..e4f3492e1 --- /dev/null +++ b/dependency-check-core/src/test/java/org/owasp/dependencycheck/analyzer/CPEAnalyzerTest.java @@ -0,0 +1,218 @@ +/* + * Copyright 2017 OWASP. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.owasp.dependencycheck.analyzer; + +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; +import static org.junit.Assert.*; +import org.owasp.dependencycheck.Engine; +import org.owasp.dependencycheck.data.cpe.IndexEntry; +import org.owasp.dependencycheck.data.nvdcve.CveDB; +import org.owasp.dependencycheck.dependency.Confidence; +import org.owasp.dependencycheck.dependency.Dependency; +import org.owasp.dependencycheck.dependency.Evidence; +import org.owasp.dependencycheck.utils.Settings; + +/** + * + * @author jeremy + */ +public class CPEAnalyzerTest { + + /** + * Test of getName method, of class CPEAnalyzer. + */ + @Test + public void testGetName() { + CPEAnalyzer instance = new CPEAnalyzer(); + String expResult = "CPE Analyzer"; + String result = instance.getName(); + assertEquals(expResult, result); + } + + /** + * Test of getAnalysisPhase method, of class CPEAnalyzer. + */ + @Test + public void testGetAnalysisPhase() { + CPEAnalyzer instance = new CPEAnalyzer(); + AnalysisPhase expResult = AnalysisPhase.IDENTIFIER_ANALYSIS; + AnalysisPhase result = instance.getAnalysisPhase(); + assertEquals(expResult, result); + } + + /** + * Test of getAnalyzerEnabledSettingKey method, of class CPEAnalyzer. + */ + @Test + public void testGetAnalyzerEnabledSettingKey() { + CPEAnalyzer instance = new CPEAnalyzer(); + String expResult = Settings.KEYS.ANALYZER_CPE_ENABLED; + String result = instance.getAnalyzerEnabledSettingKey(); + assertEquals(expResult, result); + } + + /** + * Test of addEvidenceWithoutDuplicateTerms method, of class CPEAnalyzer. + */ + @Test + public void testAddEvidenceWithoutDuplicateTerms() { + String text = ""; + List evidence = new ArrayList<>(); + evidence.add(new Evidence("test case", "value", "test", Confidence.HIGHEST)); + CPEAnalyzer instance = new CPEAnalyzer(); + String expResult = "test"; + String result = instance.addEvidenceWithoutDuplicateTerms(text, evidence); + assertEquals(expResult, result); + + text = "some"; + expResult = "some test"; + result = instance.addEvidenceWithoutDuplicateTerms(text, evidence); + assertEquals(expResult, result); + + text = "test"; + expResult = "test"; + result = instance.addEvidenceWithoutDuplicateTerms(text, evidence); + assertEquals(expResult, result); + + + StringBuilder sb = new StringBuilder(); + StringBuilder expect = new StringBuilder(); + for (int x=0;x<500;x++) { + sb.append("items "); + if (expect.length()+5<1000) { + expect.append("items "); + } + } + evidence.clear(); + evidence.add(new Evidence("test case", "value", sb.toString(), Confidence.HIGHEST)); + text = ""; + expResult = expect.toString().trim(); + result = instance.addEvidenceWithoutDuplicateTerms(text, evidence); + assertEquals(expResult, result); + } + + /** + * Test of buildSearch method, of class CPEAnalyzer. + */ + @Test + public void testBuildSearch() { + String vendor = "apache software foundation"; + String product = "lucene index"; + Set vendorWeighting = null; + Set productWeightings = null; + + CPEAnalyzer instance = new CPEAnalyzer(); + String expResult = "product:(lucene index) AND vendor:(apache software foundation)"; + String result = instance.buildSearch(vendor, product, vendorWeighting, productWeightings); + assertEquals(expResult, result); + + vendorWeighting = new HashSet<>(); + productWeightings = new HashSet<>(); + expResult = "product:(lucene index) AND vendor:(apache software foundation)"; + result = instance.buildSearch(vendor, product, vendorWeighting, productWeightings); + assertEquals(expResult, result); + + vendorWeighting.add("apache"); + expResult = "product:(lucene index) AND vendor:(apache^5 software foundation)"; + result = instance.buildSearch(vendor, product, vendorWeighting, productWeightings); + assertEquals(expResult, result); + + productWeightings.add("lucene"); + expResult = "product:(lucene^5 index) AND vendor:(apache^5 software foundation)"; + result = instance.buildSearch(vendor, product, vendorWeighting, productWeightings); + assertEquals(expResult, result); + + productWeightings.add("ignored"); + expResult = "product:(lucene^5 index) AND vendor:(apache^5 software foundation)"; + result = instance.buildSearch(vendor, product, vendorWeighting, productWeightings); + assertEquals(expResult, result); + + vendorWeighting.clear(); + expResult = "product:(lucene^5 index) AND vendor:(apache software foundation)"; + result = instance.buildSearch(vendor, product, vendorWeighting, productWeightings); + assertEquals(expResult, result); + + vendorWeighting.add("ignored"); + productWeightings.clear(); + expResult = "product:(lucene index) AND vendor:(apache software foundation)"; + result = instance.buildSearch(vendor, product, vendorWeighting, productWeightings); + assertEquals(expResult, result); + } + + /** + * Test of prepareAnalyzer method, of class CPEAnalyzer. + */ + @Test + public void testPrepareAnalyzer() throws Exception { + //Part of the integration tests. + } + + /** + * Test of open method, of class CPEAnalyzer. + */ + @Test + public void testOpen() throws Exception { + //Part of the integration tests. + } + + /** + * Test of closeAnalyzer method, of class CPEAnalyzer. + */ + @Test + public void testCloseAnalyzer() { + //Part of the integration tests. + } + + /** + * Test of determineCPE method, of class CPEAnalyzer. + */ + @Test + public void testDetermineCPE() throws Exception { + //Part of the integration tests. + } + + /** + * Test of searchCPE method, of class CPEAnalyzer. + */ + @Test + public void testSearchCPE() { + //Part of the integration tests. + } + + /** + * Test of analyzeDependency method, of class CPEAnalyzer. + */ + @Test + public void testAnalyzeDependency() throws Exception { + //Part of the integration tests. + } + + /** + * Test of determineIdentifiers method, of class CPEAnalyzer. + */ + @Test + public void testDetermineIdentifiers() throws Exception { + //Part of the integration tests. + } +} From 121972ffd947efbffff886ca1995980759a62445 Mon Sep 17 00:00:00 2001 From: Jeremy Long Date: Fri, 22 Dec 2017 06:10:22 -0500 Subject: [PATCH 4/5] codacy cleanup --- .../analyzer/CPEAnalyzerTest.java | 64 ------------------- 1 file changed, 64 deletions(-) diff --git a/dependency-check-core/src/test/java/org/owasp/dependencycheck/analyzer/CPEAnalyzerTest.java b/dependency-check-core/src/test/java/org/owasp/dependencycheck/analyzer/CPEAnalyzerTest.java index e4f3492e1..d130f2f67 100644 --- a/dependency-check-core/src/test/java/org/owasp/dependencycheck/analyzer/CPEAnalyzerTest.java +++ b/dependency-check-core/src/test/java/org/owasp/dependencycheck/analyzer/CPEAnalyzerTest.java @@ -19,17 +19,9 @@ import java.util.ArrayList; import java.util.HashSet; import java.util.List; import java.util.Set; -import org.junit.After; -import org.junit.AfterClass; -import org.junit.Before; -import org.junit.BeforeClass; import org.junit.Test; import static org.junit.Assert.*; -import org.owasp.dependencycheck.Engine; -import org.owasp.dependencycheck.data.cpe.IndexEntry; -import org.owasp.dependencycheck.data.nvdcve.CveDB; import org.owasp.dependencycheck.dependency.Confidence; -import org.owasp.dependencycheck.dependency.Dependency; import org.owasp.dependencycheck.dependency.Evidence; import org.owasp.dependencycheck.utils.Settings; @@ -159,60 +151,4 @@ public class CPEAnalyzerTest { result = instance.buildSearch(vendor, product, vendorWeighting, productWeightings); assertEquals(expResult, result); } - - /** - * Test of prepareAnalyzer method, of class CPEAnalyzer. - */ - @Test - public void testPrepareAnalyzer() throws Exception { - //Part of the integration tests. - } - - /** - * Test of open method, of class CPEAnalyzer. - */ - @Test - public void testOpen() throws Exception { - //Part of the integration tests. - } - - /** - * Test of closeAnalyzer method, of class CPEAnalyzer. - */ - @Test - public void testCloseAnalyzer() { - //Part of the integration tests. - } - - /** - * Test of determineCPE method, of class CPEAnalyzer. - */ - @Test - public void testDetermineCPE() throws Exception { - //Part of the integration tests. - } - - /** - * Test of searchCPE method, of class CPEAnalyzer. - */ - @Test - public void testSearchCPE() { - //Part of the integration tests. - } - - /** - * Test of analyzeDependency method, of class CPEAnalyzer. - */ - @Test - public void testAnalyzeDependency() throws Exception { - //Part of the integration tests. - } - - /** - * Test of determineIdentifiers method, of class CPEAnalyzer. - */ - @Test - public void testDetermineIdentifiers() throws Exception { - //Part of the integration tests. - } } From 536914c3b7bc7ad9360fd75fca1e79c288b9f680 Mon Sep 17 00:00:00 2001 From: Jeremy Long Date: Fri, 22 Dec 2017 06:32:54 -0500 Subject: [PATCH 5/5] fix possible NPE --- .../java/org/owasp/dependencycheck/analyzer/CPEAnalyzer.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dependency-check-core/src/main/java/org/owasp/dependencycheck/analyzer/CPEAnalyzer.java b/dependency-check-core/src/main/java/org/owasp/dependencycheck/analyzer/CPEAnalyzer.java index cde34a4d9..cb74b6c07 100644 --- a/dependency-check-core/src/main/java/org/owasp/dependencycheck/analyzer/CPEAnalyzer.java +++ b/dependency-check-core/src/main/java/org/owasp/dependencycheck/analyzer/CPEAnalyzer.java @@ -254,7 +254,7 @@ public class CPEAnalyzer extends AbstractAnalyzer { @SuppressWarnings("null") protected String addEvidenceWithoutDuplicateTerms(final String text, final Iterable evidence) { final String txt = (text == null) ? "" : text; - final StringBuilder sb = new StringBuilder(text.length() * 2); + final StringBuilder sb = new StringBuilder(txt.length() * 2); sb.append(' ').append(txt).append(' '); for (Evidence e : evidence) { String value = e.getValue();