Add support for newer ODC

app/com/ysoft/odc/OdcParser.scala

@@ -207,7 +207,7 @@ object RichBoolean{
   @inline implicit def toRichBoolean(value: Boolean) = new RichBoolean(value)
 }
 
-final case class Vulnerability(name: String, cweOption: Option[CWE], cvss: CvssRating, description: String, vulnerableSoftware: Seq[VulnerableSoftware], references: Seq[Reference]){
+final case class Vulnerability(name: String, /*cweOption: Option[CWE],*/ cvss: CvssRating, description: String, vulnerableSoftware: Seq[VulnerableSoftware], references: Seq[Reference]){
   import RichBoolean.toRichBoolean
   def cvssScore = cvss.score
   def likelyMatchesOnlyWithoutVersion(dependencyIdentifiers: Set[Identifier]) = dependencyIdentifiers.forall { id =>
@@ -236,6 +236,7 @@ final case class Identifier(name: String, confidence: Confidence.Confidence, url
 }
 
 object OdcParser {
+  private val StrictMode = false
 
   private val vulnPool = new ObjectPool()
   private val evidencePool = new ObjectPool()
@@ -251,10 +252,12 @@ object OdcParser {
   }
 
   def checkElements(node: Node, knownElements: Set[String]) {
-    val subelementNames = filterWhitespace(node).map(_.label).toSet
+    if(StrictMode) {
-    val unknownElements = subelementNames -- knownElements
+      val subelementNames = filterWhitespace(node).map(_.label).toSet
-    if(unknownElements.nonEmpty){
+      val unknownElements = subelementNames -- knownElements
-      sys.error("Unknown elements for "+node.label+": "+unknownElements)
+      if (unknownElements.nonEmpty) {
+        sys.error("Unknown elements for " + node.label + ": " + unknownElements)
+      }
     }
   }
 
@@ -264,17 +267,19 @@ object OdcParser {
   }
 
   def checkParams(node: Node, knownParams: Set[String]) {
-    val paramNames = getAttributes(node.attributes).toSet
+    if(StrictMode) {
-    val unknownParams = paramNames -- knownParams
+      val paramNames = getAttributes(node.attributes).toSet
-    if(unknownParams.nonEmpty){
+      val unknownParams = paramNames -- knownParams
-      sys.error("Unknown params for "+node.label+": "+unknownParams)
+      if (unknownParams.nonEmpty) {
+        sys.error("Unknown params for " + node.label + ": " + unknownParams)
+      }
     }
   }
 
 
   def parseVulnerableSoftware(node: Node): VulnerableSoftware = {
     checkElements(node, Set("#PCDATA"))
-    checkParams(node, Set("allPreviousVersion"))
+    checkParams(node, Set("allPreviousVersion", "versionEndIncluding", "versionEndExcluding", "vulnerabilityIdMatched"))
     if(node.label != "software"){
       sys.error(s"Unexpected element for vulnerableSoftware: ${node.label}")
     }
@@ -298,7 +303,7 @@ object OdcParser {
   }
 
   def parseVulnerability(node: Node, expectedLabel: String = "vulnerability"): Vulnerability = {
-    checkElements(node, Set("name", "severity", "cwe", "cvssScore", "description", "references", "vulnerableSoftware", "cvssAuthenticationr", "cvssAvailabilityImpact", "cvssAccessVector", "cvssIntegrityImpact", "cvssAccessComplexity", "cvssConfidentialImpact", "notes"))
+    checkElements(node, Set("name", "severity", "cwe", "cwes", "cvssScore", "cvssV2", "cvssV3", "description", "references", "vulnerableSoftware", "cvssAuthenticationr", "cvssAvailabilityImpact", "cvssAccessVector", "cvssIntegrityImpact", "cvssAccessComplexity", "cvssConfidentialImpact", "notes"))
     // TODO: notes element is currently ignored
     if(node.label != expectedLabel){
       sys.error(s"Unexpected element for vuln: ${node.label}")
@@ -319,29 +324,79 @@ object OdcParser {
           }
       }
     }
+
+    def cvssScore = {
+      val cvssV2 = node \ "cvssV2"
+      if (cvssV2.nonEmpty) {
+        CvssRating(
+          score = (cvssV2 \ "score").headOption.map(_.text.toDouble),
+          authenticationr = t(cvssV2 \ "authenticationr"),
+          availabilityImpact = t(cvssV2 \ "availabilityImpact"),
+          accessVector = t(cvssV2 \ "accessVector"),
+          integrityImpact = t(cvssV2 \ "integrityImpact"),
+          accessComplexity = t(cvssV2 \ "accessComplexity"),
+          confidentialImpact = t(cvssV2 \ "confidentialImpact")
+        )
+      } else {
+        CvssRating(
+          score = (node \ "cvssScore").headOption.map(_.text.toDouble),
+          authenticationr = t(node \ "cvssAuthenticationr"),
+          availabilityImpact = t(node \ "cvssAvailabilityImpact"),
+          accessVector = t(node \ "cvssAccessVector"),
+          integrityImpact = t(node \ "cvssIntegrityImpact"),
+          accessComplexity = t(node \ "cvssAccessComplexity"),
+          confidentialImpact = t(node \ "cvssConfidentialImpact")
+        )
+      }
+    }
+
     vulnPool(Vulnerability(
       name = (node \ "name").text,
       //severity = (node \ "severity"), <- severity is useless, as it is computed from cvssScore :D
-      cweOption = (node \ "cwe").headOption.map(_.text).map(CWE.forIdentifierWithDescription),
+      //cweOption = (node \ "cwe").headOption.map(_.text).map(CWE.forIdentifierWithDescription),
       description = (node \ "description").text,
-      cvss = CvssRating(
+      cvss = cvssScore,
-        score = (node \ "cvssScore").headOption.map(_.text.toDouble),
-        authenticationr = t(node \ "cvssAuthenticationr"),
-        availabilityImpact = t(node \ "cvssAvailabilityImpact"),
-        accessVector = t(node \ "cvssAccessVector"),
-        integrityImpact = t(node \ "cvssIntegrityImpact"),
-        accessComplexity = t(node \ "cvssAccessComplexity"),
-        confidentialImpact = t(node \ "cvssConfidentialImpact")
-      ),
       references = (node \ "references").flatMap(filterWhitespace).map(parseReference(_)),
       vulnerableSoftware = (node \ "vulnerableSoftware").flatMap(filterWhitespace).map(parseVulnerableSoftware)
     ))
   }
 
   def parseIdentifier(node: Node, expectedLabel: String, parseConfidence: Boolean = true): Identifier = {
-    if(node.label != expectedLabel){
+    // Old ODC produces expectedLabel, new ODC produces package and vulnerabilityIds
-      sys.error("Unexpected label for identifier: "+node.label)
+    node.label match {
+      case "suppressedIdentifier" if (node \ "id").nonEmpty => parseIdentifierNew(node, parseConfidence, matched=false) // not sure if matched
+      case `expectedLabel` => parseIdentifierOld(node, parseConfidence)
+      case "package" => parseIdentifierNew(node, parseConfidence, matched=false)
+      case "vulnerabilityIds" => parseIdentifierNew(node, parseConfidence, matched=true)
+      case "suppressedVulnerabilityIds" => parseIdentifierNew(node, parseConfidence, matched=true)
+      case label => sys.error(s"Expected node name package or vulnerabilityIds or $expectedLabel, got: "+label)
     }
+  }
+
+  private val NugetPattern = """^pkg:nuget/([^@]+)@(.*)$""".r
+  private val MavenPattern = """^pkg:maven/([^/@]+)/([^/@]+)@(.*)$""".r
+  private val CpePattern = """^cpe:.*""".r
+
+  private def parseIdentifierNew(node: Node, parseConfidence: Boolean, matched: Boolean): Identifier = {
+    checkElements(node, Set("id", "url", "notes"))
+    // TODO: process currently ignored element “notes”
+    checkParams(node, Set("type", "confidence"))
+    val id = (node \ "id").text
+    val (identifierType, name) = id match {
+      case NugetPattern(name, version) => ("nuget", s"$name:$version")
+      case MavenPattern(groupId, artifactId, version) => ("maven", s"$groupId:$artifactId:$version")
+      case CpePattern() => ("cpe", id)
+      case _ => ("other", id)
+    }
+    identifierPool(Identifier(
+      name = name,
+      url = (node \ "url").text,
+      identifierType = identifierType,
+      confidence = if(parseConfidence) Confidence.withName(node.attribute("confidence").get.text) else Confidence.Medium
+    ))
+  }
+
+  private def parseIdentifierOld(node: Node, parseConfidence: Boolean): Identifier = {
     checkElements(node, Set("name", "url", "notes"))
     // TODO: process currently ignored element “notes”
     checkParams(node, Set("type", "confidence"))
@@ -352,7 +407,7 @@ object OdcParser {
         case text => text // used in new ODC
       },
       url = (node \ "url").text,
-      identifierType = node.attribute("type").get.text,
+      identifierType = try{node.attribute("type").get.text}catch{case e: NoSuchElementException => sys.error(s"No type attribute in $node")},
       confidence = if(parseConfidence) Confidence.withName(node.attribute("confidence").get.text) else Confidence.Medium
     ))
   }
@@ -362,7 +417,7 @@ object OdcParser {
     // TODO: process projectReferences
     checkParams(node, Set("isVirtual"))
     val (vulnerabilities: Seq[Node], suppressedVulnerabilities: Seq[Node]) = (node \ "vulnerabilities").headOption.map(filterWhitespace).getOrElse(Seq()).partition(_.label == "vulnerability")
-    val (identifiers, suppressedIdentifiers) = (node \ "identifiers").headOption.map(filterWhitespace).getOrElse(Seq()).partition(_.label == "identifier")
+    val (identifiers, suppressedIdentifiers) = (node \ "identifiers").headOption.map(filterWhitespace).getOrElse(Seq()).partition(!_.label.startsWith("suppressed"))
     dependencyPool(Dependency(
       fileName = (node \ "fileName").text,
       filePath = (node \ "filePath").text,

app/com/ysoft/odc/statistics/LibDepStatistics.scala

@@ -17,12 +17,12 @@ case class LibDepStatistics(libraries: Set[(Int, Library)], dependencies: Set[Gr
   lazy val vulnerableDependencies = dependencies.filter(_.isVulnerable)
   lazy val (dependenciesWithCpe, dependenciesWithoutCpe) = dependencies.partition(_.hasCpe)
   lazy val cpeRatio = dependenciesWithCpe.size.toDouble / dependencies.size.toDouble
-  lazy val weaknesses = vulnerabilities.flatMap(_.cweOption)
+  //lazy val weaknesses = vulnerabilities.flatMap(_.cweOption)
-  lazy val weaknessesFrequency = LibDepStatistics.computeWeaknessesFrequency(vulnerabilities)
+  //lazy val weaknessesFrequency = LibDepStatistics.computeWeaknessesFrequency(vulnerabilities)
 }
 
 object LibDepStatistics{
-  private def computeWeaknessesFrequency(vulnerabilities: Set[Vulnerability]) = vulnerabilities.toSeq.map(_.cweOption).groupBy(identity).mapValues(_.size).map(identity).withDefaultValue(0)
+  //private def computeWeaknessesFrequency(vulnerabilities: Set[Vulnerability]) = vulnerabilities.toSeq.map(_.cweOption).groupBy(identity).mapValues(_.size).map(identity).withDefaultValue(0)
   def apply(libraries: Set[(Int, Library)], dependencies: Set[GroupedDependency], parsedReports: Result): LibDepStatistics = LibDepStatistics(
     libraries = libraries,
     dependencies = dependencies,

app/models/vulnerabilityOverviews.scala

@@ -11,7 +11,7 @@ abstract sealed class VulnerabilityOverview {
   def descriptionAttempt: String
   def isSureAboutDescription: Boolean
   def cvssScore: Option[Double]
-  def cweOption: Option[CWE]
+  //def cweOption: Option[CWE]
 }
 
 object VulnerabilityOverview{
@@ -23,21 +23,21 @@ final class StandardVulnerabilityOverview(vulnerability: Vulnerability) extends
   override def descriptionAttempt: String = vulnerability.description
   override def isSureAboutDescription = true
   override def cvssScore: Option[Double] = vulnerability.cvssScore
-  override def cweOption = vulnerability.cweOption
+  //override def cweOption = vulnerability.cweOption
 }
 
 private final class UnknownVulnerabilityOverview(override val name: String, link: String) extends VulnerabilityOverview {
   override def descriptionAttempt: String = s"Unknown vulnerability. Try looking at the following address for more details: $link"
   override def cvssScore: Option[Double] = None
   override def isSureAboutDescription = false
-  override def cweOption = None
+  //override def cweOption = None
 }
 
 private final class TotallyUnknownVulnerabilityOverview(override val name: String) extends VulnerabilityOverview {
   override def descriptionAttempt: String = s"Unknown vulnerability. Not even sure where to look for other details. Maybe Googling the identifier will help."
   override def cvssScore: Option[Double] = None
   override def isSureAboutDescription = false
-  override def cweOption = None
+  //override def cweOption = None
 }
 
 private object UnknownVulnerabilityOverview {

app/services/JiraIssueTrackerService.scala

@@ -110,7 +110,7 @@ class JiraIssueTrackerService @Inject()(absolutizer: Absolutizer, @Named("jira-s
     "project" -> Json.obj(
       "id" -> projectId.toString
     ),
-    "summary" -> s"${vulnerability.name} – ${vulnerability.cweOption.map(_ + ": ").getOrElse("")}${vulnerability.description.take(50).takeWhile(c => c != '\n' && c != '\r')}…"
+    "summary" -> s"${vulnerability.name} – ${vulnerability.description.take(50).takeWhile(c => c != '\n' && c != '\r')}…"
   )
 
   private def extractManagedFields(vulnerability: VulnerabilityOverview, projects: Set[ReportInfo], requiresDescription: Boolean): JsObject = {
@@ -121,7 +121,7 @@ class JiraIssueTrackerService @Inject()(absolutizer: Absolutizer, @Named("jira-s
     )
     val descriptionObj = if(requiresDescription || vulnerability.isSureAboutDescription) Json.obj("description" -> extractDescription(vulnerability)) else Json.obj()
     val additionalFields = Seq[Option[(String, JsValueWrapper)]](
-      fields.cweId.map(id => id -> vulnerability.cweOption.fold("")(_.brief)),
+      //fields.cweId.map(id => id -> vulnerability.cweOption.fold("")(_.brief)),
       fields.linkId.map(id => id -> link(vulnerability)),
       fields.severityId.map(id => id -> vulnerability.cvssScore),
       fields.projectsId.map(id => id -> projects.map(friendlyProjectNameString).toSeq.sortBy( x => (x.toLowerCase(), x)).mkString("\n"))

app/services/OdcDbService.scala

@@ -48,7 +48,7 @@ class OdcDbService @Inject()(@NamedDatabase("odc") protected val dbConfigProvide
         } yield Some(
           com.ysoft.odc.Vulnerability(
             name = bareVuln.cve,
-            cweOption = bareVuln.cweOption,
+            //cweOption = bareVuln.cweOption,
             cvss = bareVuln.cvss,
             description = bareVuln.description,
             vulnerableSoftware = vulnerableSoftware,

app/views/statistics/basic.scala.html

@@ -53,7 +53,7 @@
     }
 
 
-    <div id="weakness" data-data='@{plotData(lds.weaknessesFrequency)}'></div>
+    @*<div id="weakness" data-data='@{plotData(lds.weaknessesFrequency)}'></div>*@
     <script type="text/javascript">
 
     var WeaknessIdentifier = function(brief, verbose){
@@ -122,7 +122,7 @@
         el.attr('data-initialized', 'true');
     };
     $(document).ready(function(){
-        initPlot('weakness');
+        //initPlot('weakness');
         var n=0;
         $('.stats').click(function(e){
             console.log(e);
@@ -174,7 +174,7 @@
                     </td>
                     <td class="text-right">
                         @s.vulnerabilities.size
-                        <button type="button" class="glyphicon glyphicon-signal btn btn-xs stats" @if(s.vulnerabilities.isEmpty){disabled="disabled"} data-data="@plotData(s.weaknessesFrequency)"></button>
+                        @*<button type="button" class="glyphicon glyphicon-signal btn btn-xs stats" @if(s.vulnerabilities.isEmpty){disabled="disabled"} data-data="@plotData(s.weaknessesFrequency)"></button>*@
                     </td>
                     <td class="text-right">@s.vulnerableDependencies.size</td>
                     <td class="text-right">@s.dependencies.size</td>

app/views/vulnerability.scala.html

@@ -26,7 +26,7 @@
         case Some(score) => {<b>@score</b>}
     }
 </span></p>
-@vuln.cweOption.map{cwe =><p><label>Category:</label> <span class="explained" title="Vulnerability category according to Common Weakness Enumeration" onmouseover="$(this).tooltip({placement: 'right'}).tooltip('show')"><b>@cwe</b></span></p>}
+@*@vuln.cweOption.map{cwe =><p><label>Category:</label> <span class="explained" title="Vulnerability category according to Common Weakness Enumeration" onmouseover="$(this).tooltip({placement: 'right'}).tooltip('show')"><b>@cwe</b></span></p>}*@
 <label>Description:</label> @vuln.description
 @section("vuln-sw", "Vulnerable software"){
     <ul id="@idPrefix-details">

test/ParserSpec.scala

@@ -0,0 +1,60 @@
+import java.io.{ByteArrayOutputStream, InputStream}
+
+import org.specs2.mutable.Specification
+import com.ysoft.odc._
+
+class ParserSpec  extends Specification {
+  private def readStream(in: InputStream): Array[Byte] = {
+    val buff = new Array[Byte](1024)
+    val out = new ByteArrayOutputStream()
+    var n = 0
+    while({
+      n = in.read(buff)
+      n != -1
+    }){
+      out.write(buff, 0, n)
+    }
+    out.toByteArray
+  }
+
+  private def parseReport(reportResourceName: String) = {
+    val reportBytes: Array[Byte] = readStream(getClass.getResourceAsStream(reportResourceName))
+    OdcParser.parseXmlReport(reportBytes)
+  }
+
+  private def findDependency(identifierType: String, name: String)(implicit report: Analysis) = {
+    val found = report.dependencies.filter(_.identifiers.exists(i => i.identifierType == identifierType && i.name == name))
+    found.size match {
+      case 0 => sys.error(s"Dependency $identifierType: $name not found")
+      case 1 => (found.toSeq)(0)
+      case _ => sys.error(s"Multiple dependencies $identifierType: $name found: $found")
+    }
+  }
+
+  private def shouldHaveIdentifier(dep: Dependency, identifierType: String, name: String) = s"should have identifier $identifierType: $name" >> {
+    (dep.identifiers.exists((i: Identifier) => (i.identifierType == identifierType) && (i.name == name))) should beTrue
+  }
+
+
+  "Maven report" >> {
+    implicit val report = parseReport("dependency-check-report-maven.xml")
+    "groupId" >> {report.groupId shouldEqual "com.ysoft.security"}
+    println(report.dependencies.map(_.identifiers).mkString("\n\n"))
+    "commons-collections" >> {
+      val dep = findDependency("maven", "commons-collections:commons-collections:3.2.1")
+      dep.vulnerabilities.size shouldEqual 3
+      //shouldHaveIdentifier(dep, "cpe", "cpe:/a:apache:commons_collections:3.2.1")
+    }
+    "commons-cli" >> {
+      val dep = findDependency("maven", "commons-cli:commons-cli:1.4")
+      dep.vulnerabilities.size shouldEqual 0
+      //shouldHaveIdentifier(dep, "cpe", "cpe:/a:cli_project:cli:1.4")
+    }
+    "jackson-databind" >> {
+      val dep = findDependency("maven", "com.fasterxml.jackson.core:jackson-databind:2.9.7")
+      dep.vulnerabilities.size shouldEqual 15
+      //shouldHaveIdentifier(dep, "cpe", "cpe:/a:fasterxml:jackson:2.9.7")
+      //shouldHaveIdentifier(dep, "cpe", "cpe:/a:fasterxml:jackson-databind:2.9.7")
+    }
+  }
+}

test/VulnerabilitySpec.scala

@@ -4,7 +4,7 @@ import org.specs2.mutable.Specification
 //noinspection ScalaUnnecessaryParentheses
 class VulnerabilitySpec extends Specification {
 
-  val vuln = Vulnerability("some-vuln", None, CvssRating(None, None, None, None, None, None, None), "descr", Seq(
+  val vuln = Vulnerability("some-vuln", /*None,*/ CvssRating(None, None, None, None, None, None, None), "descr", Seq(
     VulnerableSoftware(allPreviousVersion = false, "cpe:/a:ftp:ftp"),
     VulnerableSoftware(allPreviousVersion = false, "cpe:/a:ssh:ssh:1.0"),
     VulnerableSoftware(allPreviousVersion = false, "cpe:/a:asd:asd:1.0")

test/resources/pom.xml

@@ -0,0 +1,33 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+
+    <groupId>com.ysoft.security</groupId>
+    <artifactId>java-demo-project</artifactId>
+    <version>1.0-SNAPSHOT</version>
+
+    <properties>
+        <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+    </properties>
+
+    <dependencies>
+		<dependency>
+			<groupId>commons-collections</groupId>
+			<artifactId>commons-collections</artifactId>
+			<version>3.2.1</version>
+		</dependency>
+        <dependency>
+            <groupId>commons-cli</groupId>
+            <artifactId>commons-cli</artifactId>
+            <version>1.4</version>
+        </dependency>
+        <dependency>
+            <groupId>com.fasterxml.jackson.core</groupId>
+            <artifactId>jackson-databind</artifactId>
+            <version>2.9.7</version>
+        </dependency>
+    </dependencies>
+
+</project>