github-mirrors/odc-analyzer
1
0
Fork 0
mirror of https://github.com/ysoftdevs/odc-analyzer.git synced 2026-01-15 00:03:59 +01:00
Code Issues 2 Packages Projects Releases Wiki Activity

Add support for newer ODC

Browse Source
This commit is contained in:
Šesták Vít
2020-01-31 00:53:40 +01:00
parent 237f6638a0
commit 52c3228ac3
11 changed files with 194 additions and 40 deletions
Download Patch File Download Diff File Expand all files Collapse all files

105
app/com/ysoft/odc/OdcParser.scala
View File

@@ -207,7 +207,7 @@ object RichBoolean{
@inline implicit def toRichBoolean(value: Boolean) = new RichBoolean(value)
}
final case class Vulnerability(name: String, cweOption: Option[CWE], cvss: CvssRating, description: String, vulnerableSoftware: Seq[VulnerableSoftware], references: Seq[Reference]){
final case class Vulnerability(name: String, /*cweOption: Option[CWE],*/ cvss: CvssRating, description: String, vulnerableSoftware: Seq[VulnerableSoftware], references: Seq[Reference]){
import RichBoolean.toRichBoolean
def cvssScore = cvss.score
def likelyMatchesOnlyWithoutVersion(dependencyIdentifiers: Set[Identifier]) = dependencyIdentifiers.forall { id =>
@@ -236,6 +236,7 @@ final case class Identifier(name: String, confidence: Confidence.Confidence, url
}
object OdcParser {
private val StrictMode = false
private val vulnPool = new ObjectPool()
private val evidencePool = new ObjectPool()
@@ -251,10 +252,12 @@ object OdcParser {
}
def checkElements(node: Node, knownElements: Set[String]) {
val subelementNames = filterWhitespace(node).map(_.label).toSet
val unknownElements = subelementNames -- knownElements
if(unknownElements.nonEmpty){
sys.error("Unknown elements for "+node.label+": "+unknownElements)
if(StrictMode) {
val subelementNames = filterWhitespace(node).map(_.label).toSet
val unknownElements = subelementNames -- knownElements
if (unknownElements.nonEmpty) {
sys.error("Unknown elements for " + node.label + ": " + unknownElements)
}
}
}
@@ -264,17 +267,19 @@ object OdcParser {
}
def checkParams(node: Node, knownParams: Set[String]) {
val paramNames = getAttributes(node.attributes).toSet
val unknownParams = paramNames -- knownParams
if(unknownParams.nonEmpty){
sys.error("Unknown params for "+node.label+": "+unknownParams)
if(StrictMode) {
val paramNames = getAttributes(node.attributes).toSet
val unknownParams = paramNames -- knownParams
if (unknownParams.nonEmpty) {
sys.error("Unknown params for " + node.label + ": " + unknownParams)
}
}
}
def parseVulnerableSoftware(node: Node): VulnerableSoftware = {
checkElements(node, Set("#PCDATA"))
checkParams(node, Set("allPreviousVersion"))
checkParams(node, Set("allPreviousVersion", "versionEndIncluding", "versionEndExcluding", "vulnerabilityIdMatched"))
if(node.label != "software"){
sys.error(s"Unexpected element for vulnerableSoftware: ${node.label}")
}
@@ -298,7 +303,7 @@ object OdcParser {
}
def parseVulnerability(node: Node, expectedLabel: String = "vulnerability"): Vulnerability = {
checkElements(node, Set("name", "severity", "cwe", "cvssScore", "description", "references", "vulnerableSoftware", "cvssAuthenticationr", "cvssAvailabilityImpact", "cvssAccessVector", "cvssIntegrityImpact", "cvssAccessComplexity", "cvssConfidentialImpact", "notes"))
checkElements(node, Set("name", "severity", "cwe", "cwes", "cvssScore", "cvssV2", "cvssV3", "description", "references", "vulnerableSoftware", "cvssAuthenticationr", "cvssAvailabilityImpact", "cvssAccessVector", "cvssIntegrityImpact", "cvssAccessComplexity", "cvssConfidentialImpact", "notes"))
// TODO: notes element is currently ignored
if(node.label != expectedLabel){
sys.error(s"Unexpected element for vuln: ${node.label}")
@@ -319,29 +324,79 @@ object OdcParser {
}
}
}
def cvssScore = {
val cvssV2 = node \ "cvssV2"
if (cvssV2.nonEmpty) {
CvssRating(
score = (cvssV2 \ "score").headOption.map(_.text.toDouble),
authenticationr = t(cvssV2 \ "authenticationr"),
availabilityImpact = t(cvssV2 \ "availabilityImpact"),
accessVector = t(cvssV2 \ "accessVector"),
integrityImpact = t(cvssV2 \ "integrityImpact"),
accessComplexity = t(cvssV2 \ "accessComplexity"),
confidentialImpact = t(cvssV2 \ "confidentialImpact")
)
} else {
CvssRating(
score = (node \ "cvssScore").headOption.map(_.text.toDouble),
authenticationr = t(node \ "cvssAuthenticationr"),
availabilityImpact = t(node \ "cvssAvailabilityImpact"),
accessVector = t(node \ "cvssAccessVector"),
integrityImpact = t(node \ "cvssIntegrityImpact"),
accessComplexity = t(node \ "cvssAccessComplexity"),
confidentialImpact = t(node \ "cvssConfidentialImpact")
)
}
}
vulnPool(Vulnerability(
name = (node \ "name").text,
//severity = (node \ "severity"), <- severity is useless, as it is computed from cvssScore :D
cweOption = (node \ "cwe").headOption.map(_.text).map(CWE.forIdentifierWithDescription),
//cweOption = (node \ "cwe").headOption.map(_.text).map(CWE.forIdentifierWithDescription),
description = (node \ "description").text,
cvss = CvssRating(
score = (node \ "cvssScore").headOption.map(_.text.toDouble),
authenticationr = t(node \ "cvssAuthenticationr"),
availabilityImpact = t(node \ "cvssAvailabilityImpact"),
accessVector = t(node \ "cvssAccessVector"),
integrityImpact = t(node \ "cvssIntegrityImpact"),
accessComplexity = t(node \ "cvssAccessComplexity"),
confidentialImpact = t(node \ "cvssConfidentialImpact")
),
cvss = cvssScore,
references = (node \ "references").flatMap(filterWhitespace).map(parseReference(_)),
vulnerableSoftware = (node \ "vulnerableSoftware").flatMap(filterWhitespace).map(parseVulnerableSoftware)
))
}
def parseIdentifier(node: Node, expectedLabel: String, parseConfidence: Boolean = true): Identifier = {
if(node.label != expectedLabel){
sys.error("Unexpected label for identifier: "+node.label)
// Old ODC produces expectedLabel, new ODC produces package and vulnerabilityIds
node.label match {
case "suppressedIdentifier" if (node \ "id").nonEmpty => parseIdentifierNew(node, parseConfidence, matched=false) // not sure if matched
case `expectedLabel` => parseIdentifierOld(node, parseConfidence)
case "package" => parseIdentifierNew(node, parseConfidence, matched=false)
case "vulnerabilityIds" => parseIdentifierNew(node, parseConfidence, matched=true)
case "suppressedVulnerabilityIds" => parseIdentifierNew(node, parseConfidence, matched=true)
case label => sys.error(s"Expected node name package or vulnerabilityIds or $expectedLabel, got: "+label)
}
}
private val NugetPattern = """^pkg:nuget/([^@]+)@(.*)$""".r
private val MavenPattern = """^pkg:maven/([^/@]+)/([^/@]+)@(.*)$""".r
private val CpePattern = """^cpe:.*""".r
private def parseIdentifierNew(node: Node, parseConfidence: Boolean, matched: Boolean): Identifier = {
checkElements(node, Set("id", "url", "notes"))
// TODO: process currently ignored element “notes”
checkParams(node, Set("type", "confidence"))
val id = (node \ "id").text
val (identifierType, name) = id match {
case NugetPattern(name, version) => ("nuget", s"$name:$version")
case MavenPattern(groupId, artifactId, version) => ("maven", s"$groupId:$artifactId:$version")
case CpePattern() => ("cpe", id)
case _ => ("other", id)
}
identifierPool(Identifier(
name = name,
url = (node \ "url").text,
identifierType = identifierType,
confidence = if(parseConfidence) Confidence.withName(node.attribute("confidence").get.text) else Confidence.Medium
))
}
private def parseIdentifierOld(node: Node, parseConfidence: Boolean): Identifier = {
checkElements(node, Set("name", "url", "notes"))
// TODO: process currently ignored element “notes”
checkParams(node, Set("type", "confidence"))
@@ -352,7 +407,7 @@ object OdcParser {
case text => text // used in new ODC
},
url = (node \ "url").text,
identifierType = node.attribute("type").get.text,
identifierType = try{node.attribute("type").get.text}catch{case e: NoSuchElementException => sys.error(s"No type attribute in $node")},
confidence = if(parseConfidence) Confidence.withName(node.attribute("confidence").get.text) else Confidence.Medium
))
}
@@ -362,7 +417,7 @@ object OdcParser {
// TODO: process projectReferences
checkParams(node, Set("isVirtual"))
val (vulnerabilities: Seq[Node], suppressedVulnerabilities: Seq[Node]) = (node \ "vulnerabilities").headOption.map(filterWhitespace).getOrElse(Seq()).partition(_.label == "vulnerability")
val (identifiers, suppressedIdentifiers) = (node \ "identifiers").headOption.map(filterWhitespace).getOrElse(Seq()).partition(_.label == "identifier")
val (identifiers, suppressedIdentifiers) = (node \ "identifiers").headOption.map(filterWhitespace).getOrElse(Seq()).partition(!_.label.startsWith("suppressed"))
dependencyPool(Dependency(
fileName = (node \ "fileName").text,
filePath = (node \ "filePath").text,

6
app/com/ysoft/odc/statistics/LibDepStatistics.scala
View File

@@ -17,12 +17,12 @@ case class LibDepStatistics(libraries: Set[(Int, Library)], dependencies: Set[Gr
lazy val vulnerableDependencies = dependencies.filter(_.isVulnerable)
lazy val (dependenciesWithCpe, dependenciesWithoutCpe) = dependencies.partition(_.hasCpe)
lazy val cpeRatio = dependenciesWithCpe.size.toDouble / dependencies.size.toDouble
lazy val weaknesses = vulnerabilities.flatMap(_.cweOption)
lazy val weaknessesFrequency = LibDepStatistics.computeWeaknessesFrequency(vulnerabilities)
//lazy val weaknesses = vulnerabilities.flatMap(_.cweOption)
//lazy val weaknessesFrequency = LibDepStatistics.computeWeaknessesFrequency(vulnerabilities)
}
object LibDepStatistics{
private def computeWeaknessesFrequency(vulnerabilities: Set[Vulnerability]) = vulnerabilities.toSeq.map(_.cweOption).groupBy(identity).mapValues(_.size).map(identity).withDefaultValue(0)
//private def computeWeaknessesFrequency(vulnerabilities: Set[Vulnerability]) = vulnerabilities.toSeq.map(_.cweOption).groupBy(identity).mapValues(_.size).map(identity).withDefaultValue(0)
def apply(libraries: Set[(Int, Library)], dependencies: Set[GroupedDependency], parsedReports: Result): LibDepStatistics = LibDepStatistics(
libraries = libraries,
dependencies = dependencies,

8
app/models/vulnerabilityOverviews.scala
View File

@@ -11,7 +11,7 @@ abstract sealed class VulnerabilityOverview {
def descriptionAttempt: String
def isSureAboutDescription: Boolean
def cvssScore: Option[Double]
def cweOption: Option[CWE]
//def cweOption: Option[CWE]
}
object VulnerabilityOverview{
@@ -23,21 +23,21 @@ final class StandardVulnerabilityOverview(vulnerability: Vulnerability) extends
override def descriptionAttempt: String = vulnerability.description
override def isSureAboutDescription = true
override def cvssScore: Option[Double] = vulnerability.cvssScore
override def cweOption = vulnerability.cweOption
//override def cweOption = vulnerability.cweOption
}
private final class UnknownVulnerabilityOverview(override val name: String, link: String) extends VulnerabilityOverview {
override def descriptionAttempt: String = s"Unknown vulnerability. Try looking at the following address for more details: $link"
override def cvssScore: Option[Double] = None
override def isSureAboutDescription = false
override def cweOption = None
//override def cweOption = None
}
private final class TotallyUnknownVulnerabilityOverview(override val name: String) extends VulnerabilityOverview {
override def descriptionAttempt: String = s"Unknown vulnerability. Not even sure where to look for other details. Maybe Googling the identifier will help."
override def cvssScore: Option[Double] = None
override def isSureAboutDescription = false
override def cweOption = None
//override def cweOption = None
}
private object UnknownVulnerabilityOverview {

4
app/services/JiraIssueTrackerService.scala
View File

@@ -110,7 +110,7 @@ class JiraIssueTrackerService @Inject()(absolutizer: Absolutizer, @Named("jira-s
"project" -> Json.obj(
"id" -> projectId.toString
),
"summary" -> s"${vulnerability.name} – ${vulnerability.cweOption.map(_ + ": ").getOrElse("")}${vulnerability.description.take(50).takeWhile(c => c != '\n' && c != '\r')}…"
"summary" -> s"${vulnerability.name} – ${vulnerability.description.take(50).takeWhile(c => c != '\n' && c != '\r')}…"
)
private def extractManagedFields(vulnerability: VulnerabilityOverview, projects: Set[ReportInfo], requiresDescription: Boolean): JsObject = {
@@ -121,7 +121,7 @@ class JiraIssueTrackerService @Inject()(absolutizer: Absolutizer, @Named("jira-s
)
val descriptionObj = if(requiresDescription || vulnerability.isSureAboutDescription) Json.obj("description" -> extractDescription(vulnerability)) else Json.obj()
val additionalFields = Seq[Option[(String, JsValueWrapper)]](
fields.cweId.map(id => id -> vulnerability.cweOption.fold("")(_.brief)),
//fields.cweId.map(id => id -> vulnerability.cweOption.fold("")(_.brief)),
fields.linkId.map(id => id -> link(vulnerability)),
fields.severityId.map(id => id -> vulnerability.cvssScore),
fields.projectsId.map(id => id -> projects.map(friendlyProjectNameString).toSeq.sortBy( x => (x.toLowerCase(), x)).mkString("\n"))

2
app/services/OdcDbService.scala
View File

@@ -48,7 +48,7 @@ class OdcDbService @Inject()(@NamedDatabase("odc") protected val dbConfigProvide
} yield Some(
com.ysoft.odc.Vulnerability(
name = bareVuln.cve,
cweOption = bareVuln.cweOption,
//cweOption = bareVuln.cweOption,
cvss = bareVuln.cvss,
description = bareVuln.description,
vulnerableSoftware = vulnerableSoftware,

6
app/views/statistics/basic.scala.html
View File

@@ -53,7 +53,7 @@
}
<div id="weakness" data-data='@{plotData(lds.weaknessesFrequency)}'></div>
@*<div id="weakness" data-data='@{plotData(lds.weaknessesFrequency)}'></div>*@
<script type="text/javascript">
var WeaknessIdentifier = function(brief, verbose){
@@ -122,7 +122,7 @@
el.attr('data-initialized', 'true');
};
$(document).ready(function(){
initPlot('weakness');
//initPlot('weakness');
var n=0;
$('.stats').click(function(e){
console.log(e);
@@ -174,7 +174,7 @@
</td>
<td class="text-right">
@s.vulnerabilities.size
<button type="button" class="glyphicon glyphicon-signal btn btn-xs stats" @if(s.vulnerabilities.isEmpty){disabled="disabled"} data-data="@plotData(s.weaknessesFrequency)"></button>
@*<button type="button" class="glyphicon glyphicon-signal btn btn-xs stats" @if(s.vulnerabilities.isEmpty){disabled="disabled"} data-data="@plotData(s.weaknessesFrequency)"></button>*@
</td>
<td class="text-right">@s.vulnerableDependencies.size</td>
<td class="text-right">@s.dependencies.size</td>

2
app/views/vulnerability.scala.html
View File

@@ -26,7 +26,7 @@
case Some(score) => {<b>@score</b>}
}
</span></p>
@vuln.cweOption.map{cwe =><p><label>Category:</label> <span class="explained" title="Vulnerability category according to Common Weakness Enumeration" onmouseover="$(this).tooltip({placement: 'right'}).tooltip('show')"><b>@cwe</b></span></p>}
@*@vuln.cweOption.map{cwe =><p><label>Category:</label> <span class="explained" title="Vulnerability category according to Common Weakness Enumeration" onmouseover="$(this).tooltip({placement: 'right'}).tooltip('show')"><b>@cwe</b></span></p>}*@
<label>Description:</label> @vuln.description
@section("vuln-sw", "Vulnerable software"){
<ul id="@idPrefix-details">

60
test/ParserSpec.scala Normal file
View File

@@ -0,0 +1,60 @@
import java.io.{ByteArrayOutputStream, InputStream}
import org.specs2.mutable.Specification
import com.ysoft.odc._
class ParserSpec extends Specification {
private def readStream(in: InputStream): Array[Byte] = {
val buff = new Array[Byte](1024)
val out = new ByteArrayOutputStream()
var n = 0
while({
n = in.read(buff)
n != -1
}){
out.write(buff, 0, n)
}
out.toByteArray
}
private def parseReport(reportResourceName: String) = {
val reportBytes: Array[Byte] = readStream(getClass.getResourceAsStream(reportResourceName))
OdcParser.parseXmlReport(reportBytes)
}
private def findDependency(identifierType: String, name: String)(implicit report: Analysis) = {
val found = report.dependencies.filter(_.identifiers.exists(i => i.identifierType == identifierType && i.name == name))
found.size match {
case 0 => sys.error(s"Dependency $identifierType: $name not found")
case 1 => (found.toSeq)(0)
case _ => sys.error(s"Multiple dependencies $identifierType: $name found: $found")
}
}
private def shouldHaveIdentifier(dep: Dependency, identifierType: String, name: String) = s"should have identifier $identifierType: $name" >> {
(dep.identifiers.exists((i: Identifier) => (i.identifierType == identifierType) && (i.name == name))) should beTrue
}
"Maven report" >> {
implicit val report = parseReport("dependency-check-report-maven.xml")
"groupId" >> {report.groupId shouldEqual "com.ysoft.security"}
println(report.dependencies.map(_.identifiers).mkString("\n\n"))
"commons-collections" >> {
val dep = findDependency("maven", "commons-collections:commons-collections:3.2.1")
dep.vulnerabilities.size shouldEqual 3
//shouldHaveIdentifier(dep, "cpe", "cpe:/a:apache:commons_collections:3.2.1")
}
"commons-cli" >> {
val dep = findDependency("maven", "commons-cli:commons-cli:1.4")
dep.vulnerabilities.size shouldEqual 0
//shouldHaveIdentifier(dep, "cpe", "cpe:/a:cli_project:cli:1.4")
}
"jackson-databind" >> {
val dep = findDependency("maven", "com.fasterxml.jackson.core:jackson-databind:2.9.7")
dep.vulnerabilities.size shouldEqual 15
//shouldHaveIdentifier(dep, "cpe", "cpe:/a:fasterxml:jackson:2.9.7")
//shouldHaveIdentifier(dep, "cpe", "cpe:/a:fasterxml:jackson-databind:2.9.7")
}
}
}

2
test/VulnerabilitySpec.scala
View File

@@ -4,7 +4,7 @@ import org.specs2.mutable.Specification
//noinspection ScalaUnnecessaryParentheses
class VulnerabilitySpec extends Specification {
val vuln = Vulnerability("some-vuln", None, CvssRating(None, None, None, None, None, None, None), "descr", Seq(
val vuln = Vulnerability("some-vuln", /*None,*/ CvssRating(None, None, None, None, None, None, None), "descr", Seq(
VulnerableSoftware(allPreviousVersion = false, "cpe:/a:ftp:ftp"),
VulnerableSoftware(allPreviousVersion = false, "cpe:/a:ssh:ssh:1.0"),
VulnerableSoftware(allPreviousVersion = false, "cpe:/a:asd:asd:1.0")

6
test/resources/dependency-check-report-maven.xml Normal file
View File

File diff suppressed because one or more lines are too long

33
test/resources/pom.xml Normal file
View File

@@ -0,0 +1,33 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>com.ysoft.security</groupId>
<artifactId>java-demo-project</artifactId>
<version>1.0-SNAPSHOT</version>
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
</properties>
<dependencies>
<dependency>
<groupId>commons-collections</groupId>
<artifactId>commons-collections</artifactId>
<version>3.2.1</version>
</dependency>
<dependency>
<groupId>commons-cli</groupId>
<artifactId>commons-cli</artifactId>
<version>1.4</version>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
<version>2.9.7</version>
</dependency>
</dependencies>
</project>