converted hint analyzer to use an externalized configuration file to simplify the resolution of issue #522

This commit is contained in:
Jeremy Long
2016-07-04 07:05:31 -04:00
parent cc751aa224
commit 960eeb19af
7 changed files with 714 additions and 5 deletions

View File

@@ -17,15 +17,36 @@
*/
package org.owasp.dependencycheck.analyzer;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
import java.util.Set;
import java.util.regex.Pattern;
import org.owasp.dependencycheck.Engine;
import org.owasp.dependencycheck.analyzer.exception.AnalysisException;
import org.owasp.dependencycheck.dependency.Confidence;
import org.owasp.dependencycheck.dependency.Dependency;
import org.owasp.dependencycheck.dependency.Evidence;
import org.owasp.dependencycheck.suppression.PropertyType;
import org.owasp.dependencycheck.suppression.SuppressionParseException;
import org.owasp.dependencycheck.suppression.SuppressionParser;
import org.owasp.dependencycheck.utils.DownloadFailedException;
import org.owasp.dependencycheck.utils.Downloader;
import org.owasp.dependencycheck.utils.FileUtils;
import org.owasp.dependencycheck.utils.Settings;
import org.owasp.dependencycheck.xml.hints.VendorDuplicatingHintRule;
import org.owasp.dependencycheck.xml.hints.HintParseException;
import org.owasp.dependencycheck.xml.hints.HintParser;
import org.owasp.dependencycheck.xml.hints.HintRule;
import org.owasp.dependencycheck.xml.hints.Hints;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.xml.sax.SAXException;
/**
*
@@ -62,18 +83,93 @@ public class HintAnalyzer extends AbstractAnalyzer implements Analyzer {
public AnalysisPhase getAnalysisPhase() {
return ANALYSIS_PHASE;
}
/**
* The initialize method does nothing for this Analyzer.
*
* @throws Exception thrown if there is an exception
*/
@Override
public void initialize() throws Exception {
super.initialize();
loadHintRules();
}
//</editor-fold>
/**
* The HintAnalyzer uses knowledge about a dependency to add additional information to help in identification of identifiers
* or vulnerabilities.
* The Logger for use throughout the class
*/
private static final Logger LOGGER = LoggerFactory.getLogger(HintAnalyzer.class);
/**
* The name of the hint rule file
*/
private static final String HINT_RULE_FILE_NAME = "dependencycheck-base-hint.xml";
/**
* The collection of hints.
*/
private Hints hints;
/**
* The HintAnalyzer uses knowledge about a dependency to add additional
* information to help in identification of identifiers or vulnerabilities.
*
* @param dependency The dependency being analyzed
* @param engine The scanning engine
* @throws AnalysisException is thrown if there is an exception analyzing the dependency.
* @throws AnalysisException is thrown if there is an exception analyzing
* the dependency.
*/
@Override
public void analyze(Dependency dependency, Engine engine) throws AnalysisException {
for (HintRule hint : hints.getHintRules()) {
boolean shouldAdd = false;
for (Evidence given : hint.getGivenVendor()) {
if (dependency.getVendorEvidence().getEvidence().contains(given)) {
shouldAdd = true;
break;
}
}
if (!shouldAdd) {
for (Evidence given : hint.getGivenProduct()) {
if (dependency.getProductEvidence().getEvidence().contains(given)) {
shouldAdd = true;
break;
}
}
}
if (!shouldAdd) {
for (PropertyType pt : hint.getFilenames()) {
if (pt.matches(dependency.getFileName())) {
shouldAdd = true;
}
}
}
if (shouldAdd) {
for (Evidence e : hint.getAddVendor()) {
dependency.getVendorEvidence().addEvidence(e);
}
for (Evidence e : hint.getAddProduct()) {
dependency.getProductEvidence().addEvidence(e);
}
}
}
final Iterator<Evidence> itr = dependency.getVendorEvidence().iterator();
final List<Evidence> newEntries = new ArrayList<Evidence>();
while (itr.hasNext()) {
final Evidence e = itr.next();
for (VendorDuplicatingHintRule dhr : hints.getVendorDuplicatingHintRules()) {
if (dhr.getValue().equalsIgnoreCase(e.getValue(false))) {
newEntries.add(new Evidence(e.getSource() + " (hint)",
e.getName(), dhr.getDuplicate(), e.getConfidence()));
}
}
}
for (Evidence e : newEntries) {
dependency.getVendorEvidence().addEvidence(e);
}
//<editor-fold defaultstate="collapsed" desc="Old implementation">
/*
final Evidence springTest1 = new Evidence("Manifest",
"Implementation-Title",
"Spring Framework",
@@ -154,7 +250,7 @@ public class HintAnalyzer extends AbstractAnalyzer implements Analyzer {
if (product.contains(zendframeworkProduct)) {
dependency.getProductEvidence().addEvidence("hint analyzer", "vendor", "zend_framework", Confidence.HIGHEST);
}
//sun/oracle problem
final Iterator<Evidence> itr = dependency.getVendorEvidence().iterator();
final List<Evidence> newEntries = new ArrayList<Evidence>();
@@ -171,6 +267,83 @@ public class HintAnalyzer extends AbstractAnalyzer implements Analyzer {
for (Evidence e : newEntries) {
dependency.getVendorEvidence().addEvidence(e);
}
*/
//</editor-fold>
}
/**
* Loads the hint rules file.
*
* @throws SuppressionParseException thrown if the XML cannot be parsed.
*/
private void loadHintRules() throws HintParseException {
final HintParser parser = new HintParser();
File file = null;
try {
hints = parser.parseHints(this.getClass().getClassLoader().getResourceAsStream(HINT_RULE_FILE_NAME));
} catch (HintParseException ex) {
LOGGER.error("Unable to parse the base hint data file");
LOGGER.debug("Unable to parse the base hint data file", ex);
} catch (SAXException ex) {
LOGGER.error("Unable to parse the base hint data file");
LOGGER.debug("Unable to parse the base hint data file", ex);
}
final String filePath = Settings.getString(Settings.KEYS.SUPPRESSION_FILE);
if (filePath == null) {
return;
}
boolean deleteTempFile = false;
try {
final Pattern uriRx = Pattern.compile("^(https?|file)\\:.*", Pattern.CASE_INSENSITIVE);
if (uriRx.matcher(filePath).matches()) {
deleteTempFile = true;
file = FileUtils.getTempFile("hint", "xml");
final URL url = new URL(filePath);
try {
Downloader.fetchFile(url, file, false);
} catch (DownloadFailedException ex) {
Downloader.fetchFile(url, file, true);
}
} else {
file = new File(filePath);
if (!file.exists()) {
final InputStream fromClasspath = this.getClass().getClassLoader().getResourceAsStream(filePath);
if (fromClasspath != null) {
deleteTempFile = true;
file = FileUtils.getTempFile("hint", "xml");
try {
org.apache.commons.io.FileUtils.copyInputStreamToFile(fromClasspath, file);
} catch (IOException ex) {
throw new HintParseException("Unable to locate suppressions file in classpath", ex);
}
}
}
}
if (file != null) {
try {
Hints newHints = parser.parseHints(file);
hints.getHintRules().addAll(newHints.getHintRules());
hints.getVendorDuplicatingHintRules().addAll(newHints.getVendorDuplicatingHintRules());
LOGGER.debug("{} hint rules were loaded.", hints.getHintRules().size());
LOGGER.debug("{} duplicating hint rules were loaded.", hints.getVendorDuplicatingHintRules().size());
} catch (HintParseException ex) {
LOGGER.warn("Unable to parse hint rule xml file '{}'", file.getPath());
LOGGER.warn(ex.getMessage());
LOGGER.debug("", ex);
throw ex;
}
}
} catch (DownloadFailedException ex) {
throw new HintParseException("Unable to fetch the configured hint file", ex);
} catch (MalformedURLException ex) {
throw new HintParseException("Configured hint file has an invalid URL", ex);
} catch (IOException ex) {
throw new HintParseException("Unable to create temp file for hints", ex);
} finally {
if (deleteTempFile && file != null) {
FileUtils.delete(file);
}
}
}
}

View File

@@ -0,0 +1,97 @@
/*
* This file is part of dependency-check-core.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
* Copyright (c) 2016 Jeremy Long. All Rights Reserved.
*/
package org.owasp.dependencycheck.xml.hints;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.xml.sax.ErrorHandler;
import org.xml.sax.SAXException;
import org.xml.sax.SAXParseException;
/**
* An XML parsing error handler.
*
* @author Jeremy Long
*/
public class HintErrorHandler implements ErrorHandler {
/**
* The logger.
*/
private static final Logger LOGGER = LoggerFactory.getLogger(HintErrorHandler.class);
/**
* Builds a prettier exception message.
*
* @param ex the SAXParseException
* @return an easier to read exception message
*/
private String getPrettyParseExceptionInfo(SAXParseException ex) {
final StringBuilder sb = new StringBuilder();
if (ex.getSystemId() != null) {
sb.append("systemId=").append(ex.getSystemId()).append(", ");
}
if (ex.getPublicId() != null) {
sb.append("publicId=").append(ex.getPublicId()).append(", ");
}
if (ex.getLineNumber() > 0) {
sb.append("Line=").append(ex.getLineNumber());
}
if (ex.getColumnNumber() > 0) {
sb.append(", Column=").append(ex.getColumnNumber());
}
sb.append(": ").append(ex.getMessage());
return sb.toString();
}
/**
* Logs warnings.
*
* @param ex the warning to log
* @throws SAXException is never thrown
*/
@Override
public void warning(SAXParseException ex) throws SAXException {
LOGGER.debug("", ex);
}
/**
* Handles errors.
*
* @param ex the error to handle
* @throws SAXException is always thrown
*/
@Override
public void error(SAXParseException ex) throws SAXException {
throw new SAXException(getPrettyParseExceptionInfo(ex));
}
/**
* Handles fatal exceptions.
*
* @param ex a fatal exception
* @throws SAXException is always
*/
@Override
public void fatalError(SAXParseException ex) throws SAXException {
throw new SAXException(getPrettyParseExceptionInfo(ex));
}
}

View File

@@ -0,0 +1,217 @@
/*
* This file is part of dependency-check-core.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
* Copyright (c) 2016 Jeremy Long. All Rights Reserved.
*/
package org.owasp.dependencycheck.xml.hints;
import java.util.ArrayList;
import java.util.List;
import org.owasp.dependencycheck.dependency.Confidence;
import org.owasp.dependencycheck.suppression.PropertyType;
import org.xml.sax.Attributes;
import org.xml.sax.SAXException;
import org.xml.sax.helpers.DefaultHandler;
/**
* A handler to load hint rules.
*
* @author Jeremy Long
*/
public class HintHandler extends DefaultHandler {
//<editor-fold defaultstate="collapsed" desc="Element and attribute names">
/**
* Element name.
*/
private static final String HINT = "hint";
/**
* Element name.
*/
private static final String GIVEN = "given";
/**
* Element name.
*/
private static final String ADD = "add";
/**
* Element name.
*/
private static final String EVIDENCE = "evidence";
/**
* Element name.
*/
private static final String FILE_NAME = "fileName";
/**
* Element name.
*/
private static final String VENDOR_DUPLICATING_RULE = "vendorDuplicatingHint";
/**
* Attribute name.
*/
private static final String DUPLIACE = "dupliace";
/**
* Attribute name.
*/
private static final String VENDOR = "vendor";
/**
* Attribute name.
*/
private static final String CONFIDENCE = "confidence";
/**
* Attribute name.
*/
private static final String VALUE = "value";
/**
* Attribute name.
*/
private static final String NAME = "name";
/**
* Attribute name.
*/
private static final String SOURCE = "source";
/**
* Attribute name.
*/
private static final String TYPE = "type";
/**
* Attribute name.
*/
private static final String CASE_SENSITIVE = "caseSensitive";
/**
* Attribute name.
*/
private static final String REGEX = "regex";
/**
* Attribute name.
*/
private static final String CONTAINS = "contains";
//</editor-fold>
/**
* The list of hint rules.
*/
private final List<HintRule> hintRules = new ArrayList<HintRule>();
/**
* Returns the list of hint rules.
*
* @return the value of hintRules
*/
public List<HintRule> getHintRules() {
return hintRules;
}
/**
* The list of vendor duplicating hint rules.
*/
private final List<VendorDuplicatingHintRule> vendorDuplicatingHintRules = new ArrayList<VendorDuplicatingHintRule>();
/**
* Returns the list of vendor duplicating hint rules.
*
* @return the list of vendor duplicating hint rules
*/
public List<VendorDuplicatingHintRule> getVendorDuplicatingHintRules() {
return vendorDuplicatingHintRules;
}
/**
* The current rule being read.
*/
private HintRule rule;
/**
* The current state of the parent node (to differentiate between 'add' and
* 'given').
*/
private boolean inAddNode = false;
/**
* Handles the start element event.
*
* @param uri the uri of the element being processed
* @param localName the local name of the element being processed
* @param qName the qName of the element being processed
* @param attr the attributes of the element being processed
* @throws SAXException thrown if there is an exception processing
*/
@Override
public void startElement(String uri, String localName, String qName, Attributes attr) throws SAXException {
if (HINT.equals(qName)) {
rule = new HintRule();
} else if (ADD.equals(qName)) {
inAddNode = true;
} else if (GIVEN.equals(qName)) {
inAddNode = false;
} else if (EVIDENCE.equals(qName)) {
final String hintType = attr.getValue(TYPE);
if (VENDOR.equals(hintType)) {
if (inAddNode) {
rule.addAddVendor(attr.getValue(SOURCE),
attr.getValue(NAME),
attr.getValue(VALUE),
Confidence.valueOf(attr.getValue(CONFIDENCE)));
} else {
rule.addGivenVendor(attr.getValue(SOURCE),
attr.getValue(NAME),
attr.getValue(VALUE),
Confidence.valueOf(attr.getValue(CONFIDENCE)));
}
} else if (inAddNode) {
rule.addAddProduct(attr.getValue(SOURCE),
attr.getValue(NAME),
attr.getValue(VALUE),
Confidence.valueOf(attr.getValue(CONFIDENCE)));
} else {
rule.addGivenProduct(attr.getValue(SOURCE),
attr.getValue(NAME),
attr.getValue(VALUE),
Confidence.valueOf(attr.getValue(CONFIDENCE)));
}
} else if (FILE_NAME.equals(qName)) {
final PropertyType pt = new PropertyType();
pt.setValue(attr.getValue(CONTAINS));
if (attr.getLength() > 0) {
final String regex = attr.getValue(REGEX);
if (regex != null) {
pt.setRegex(Boolean.parseBoolean(regex));
}
final String caseSensitive = attr.getValue(CASE_SENSITIVE);
if (caseSensitive != null) {
pt.setCaseSensitive(Boolean.parseBoolean(caseSensitive));
}
}
rule.addFilename(pt);
} else if (VENDOR_DUPLICATING_RULE.equals(qName)) {
vendorDuplicatingHintRules.add(new VendorDuplicatingHintRule(attr.getValue(VALUE), attr.getValue(DUPLIACE)));
}
}
/**
* Handles the end element event.
*
* @param uri the element's uri
* @param localName the local name
* @param qName the qualified name
* @throws SAXException thrown if there is an exception processing the
* element
*/
@Override
public void endElement(String uri, String localName, String qName) throws SAXException {
if (HINT.equals(qName) && rule != null) {
hintRules.add(rule);
rule = null;
}
}
}

View File

@@ -0,0 +1,68 @@
/*
* This file is part of dependency-check-core.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
* Copyright (c) 2016 Jeremy Long. All Rights Reserved.
*/
package org.owasp.dependencycheck.xml.hints;
import java.io.IOException;
/**
* An exception used when parsing a suppression rule file fails.
*
* @author Jeremy Long
*/
public class HintParseException extends IOException {
/**
* The serial version UID for serialization.
*/
private static final long serialVersionUID = 1L;
/**
* Creates a new SuppressionParseException.
*/
public HintParseException() {
super();
}
/**
* Creates a new SuppressionParseException.
*
* @param msg a message for the exception.
*/
public HintParseException(String msg) {
super(msg);
}
/**
* Creates a new SuppressionParseException.
*
* @param ex the cause of the parse exception
*/
public HintParseException(Throwable ex) {
super(ex);
}
/**
* Creates a new SuppressionParseException.
*
* @param msg a message for the exception.
* @param ex the cause of the parse exception
*/
public HintParseException(String msg, Throwable ex) {
super(msg, ex);
}
}

View File

@@ -0,0 +1,146 @@
/*
* This file is part of dependency-check-core.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
* Copyright (c) 2016 Jeremy Long. All Rights Reserved.
*/
package org.owasp.dependencycheck.xml.hints;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.Reader;
import javax.xml.parsers.ParserConfigurationException;
import javax.xml.parsers.SAXParser;
import javax.xml.parsers.SAXParserFactory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.xml.sax.InputSource;
import org.xml.sax.SAXException;
import org.xml.sax.XMLReader;
/**
* A simple validating parser for XML Hint Rules.
*
* @author Jeremy Long
*/
public class HintParser {
/**
* The logger.
*/
private static final Logger LOGGER = LoggerFactory.getLogger(HintParser.class);
/**
* JAXP Schema Language. Source:
* http://docs.oracle.com/javase/tutorial/jaxp/sax/validation.html
*/
public static final String JAXP_SCHEMA_LANGUAGE = "http://java.sun.com/xml/jaxp/properties/schemaLanguage";
/**
* W3C XML Schema. Source:
* http://docs.oracle.com/javase/tutorial/jaxp/sax/validation.html
*/
public static final String W3C_XML_SCHEMA = "http://www.w3.org/2001/XMLSchema";
/**
* JAXP Schema Source. Source:
* http://docs.oracle.com/javase/tutorial/jaxp/sax/validation.html
*/
public static final String JAXP_SCHEMA_SOURCE = "http://java.sun.com/xml/jaxp/properties/schemaSource";
/**
* The schema for the hint XML files.
*/
private static final String HINT_SCHEMA = "schema/dependency-hint.1.0.xsd";
/**
* Parses the given XML file and returns a list of the hints contained.
*
* @param file an XML file containing hints
* @return a list of hint rules
* @throws HintParseException thrown if the XML file cannot be parsed
*/
public Hints parseHints(File file) throws HintParseException {
FileInputStream fis = null;
try {
fis = new FileInputStream(file);
return parseHints(fis);
} catch (IOException ex) {
LOGGER.debug("", ex);
throw new HintParseException(ex);
} catch (SAXException ex) {
throw new HintParseException(ex);
} finally {
if (fis != null) {
try {
fis.close();
} catch (IOException ex) {
LOGGER.debug("Unable to close stream", ex);
}
}
}
}
/**
* Parses the given XML stream and returns a list of the hint rules
* contained.
*
* @param inputStream an InputStream containing hint rules
* @return a list of hint rules
* @throws HintParseException thrown if the XML cannot be parsed
* @throws SAXException thrown if the XML cannot be parsed
*/
public Hints parseHints(InputStream inputStream) throws HintParseException, SAXException {
try {
final InputStream schemaStream = this.getClass().getClassLoader().getResourceAsStream(HINT_SCHEMA);
final HintHandler handler = new HintHandler();
final SAXParserFactory factory = SAXParserFactory.newInstance();
factory.setNamespaceAware(true);
factory.setValidating(true);
final SAXParser saxParser = factory.newSAXParser();
saxParser.setProperty(HintParser.JAXP_SCHEMA_LANGUAGE, HintParser.W3C_XML_SCHEMA);
saxParser.setProperty(HintParser.JAXP_SCHEMA_SOURCE, new InputSource(schemaStream));
final XMLReader xmlReader = saxParser.getXMLReader();
xmlReader.setErrorHandler(new HintErrorHandler());
xmlReader.setContentHandler(handler);
final Reader reader = new InputStreamReader(inputStream, "UTF-8");
final InputSource in = new InputSource(reader);
xmlReader.parse(in);
final Hints hints = new Hints();
hints.setHintRules(handler.getHintRules());
hints.setVendorDuplicatingHintRules(handler.getVendorDuplicatingHintRules());
return hints;
} catch (ParserConfigurationException ex) {
LOGGER.debug("", ex);
throw new HintParseException(ex);
} catch (SAXException ex) {
if (ex.getMessage().contains("Cannot find the declaration of element 'hints'.")) {
throw ex;
} else {
LOGGER.debug("", ex);
throw new HintParseException(ex);
}
} catch (FileNotFoundException ex) {
LOGGER.debug("", ex);
throw new HintParseException(ex);
} catch (IOException ex) {
LOGGER.debug("", ex);
throw new HintParseException(ex);
}
}
}

View File

@@ -0,0 +1,4 @@
/**
* Contains classes used to parse the hints file to add evidence to dependencies.
*/
package org.owasp.dependencycheck.xml.hints;

View File

@@ -189,9 +189,13 @@ public final class Settings {
*/
public static final String MAX_DOWNLOAD_THREAD_POOL_SIZE = "max.download.threads";
/**
* The key for a list of suppression files.
* The key for the suppression file.
*/
public static final String SUPPRESSION_FILE = "suppression.file";
/**
* The key for the hint file.
*/
public static final String HINTS_FILE = "hints.file";
/**
* The properties key for whether the Jar Analyzer is enabled.
*/