Add support for newer ODC

2026-05-30 19:30:37 +02:00 · 2020-01-31 00:53:40 +01:00
parent 237f6638a0
commit 52c3228ac3
11 changed files with 194 additions and 40 deletions
@@ -207,7 +207,7 @@ object RichBoolean{
  @inline implicit def toRichBoolean(value: Boolean) = new RichBoolean(value)
 }

-final case class Vulnerability(name: String, cweOption: Option[CWE], cvss: CvssRating, description: String, vulnerableSoftware: Seq[VulnerableSoftware], references: Seq[Reference]){
+final case class Vulnerability(name: String, /*cweOption: Option[CWE],*/ cvss: CvssRating, description: String, vulnerableSoftware: Seq[VulnerableSoftware], references: Seq[Reference]){
  import RichBoolean.toRichBoolean
  def cvssScore = cvss.score
  def likelyMatchesOnlyWithoutVersion(dependencyIdentifiers: Set[Identifier]) = dependencyIdentifiers.forall { id =>
@@ -236,6 +236,7 @@ final case class Identifier(name: String, confidence: Confidence.Confidence, url
 }

 object OdcParser {
+  private val StrictMode = false

  private val vulnPool = new ObjectPool()
  private val evidencePool = new ObjectPool()
@@ -251,10 +252,12 @@ object OdcParser {
  }

  def checkElements(node: Node, knownElements: Set[String]) {
-    val subelementNames = filterWhitespace(node).map(_.label).toSet
-    val unknownElements = subelementNames -- knownElements
-    if(unknownElements.nonEmpty){
-      sys.error("Unknown elements for "+node.label+": "+unknownElements)
+    if(StrictMode) {
+      val subelementNames = filterWhitespace(node).map(_.label).toSet
+      val unknownElements = subelementNames -- knownElements
+      if (unknownElements.nonEmpty) {
+        sys.error("Unknown elements for " + node.label + ": " + unknownElements)
+      }
    }
  }

@@ -264,17 +267,19 @@ object OdcParser {
  }

  def checkParams(node: Node, knownParams: Set[String]) {
-    val paramNames = getAttributes(node.attributes).toSet
-    val unknownParams = paramNames -- knownParams
-    if(unknownParams.nonEmpty){
-      sys.error("Unknown params for "+node.label+": "+unknownParams)
+    if(StrictMode) {
+      val paramNames = getAttributes(node.attributes).toSet
+      val unknownParams = paramNames -- knownParams
+      if (unknownParams.nonEmpty) {
+        sys.error("Unknown params for " + node.label + ": " + unknownParams)
+      }
    }
  }


  def parseVulnerableSoftware(node: Node): VulnerableSoftware = {
    checkElements(node, Set("#PCDATA"))
-    checkParams(node, Set("allPreviousVersion"))
+    checkParams(node, Set("allPreviousVersion", "versionEndIncluding", "versionEndExcluding", "vulnerabilityIdMatched"))
    if(node.label != "software"){
      sys.error(s"Unexpected element for vulnerableSoftware: ${node.label}")
    }
@@ -298,7 +303,7 @@ object OdcParser {
  }

  def parseVulnerability(node: Node, expectedLabel: String = "vulnerability"): Vulnerability = {
-    checkElements(node, Set("name", "severity", "cwe", "cvssScore", "description", "references", "vulnerableSoftware", "cvssAuthenticationr", "cvssAvailabilityImpact", "cvssAccessVector", "cvssIntegrityImpact", "cvssAccessComplexity", "cvssConfidentialImpact", "notes"))
+    checkElements(node, Set("name", "severity", "cwe", "cwes", "cvssScore", "cvssV2", "cvssV3", "description", "references", "vulnerableSoftware", "cvssAuthenticationr", "cvssAvailabilityImpact", "cvssAccessVector", "cvssIntegrityImpact", "cvssAccessComplexity", "cvssConfidentialImpact", "notes"))
    // TODO: notes element is currently ignored
    if(node.label != expectedLabel){
      sys.error(s"Unexpected element for vuln: ${node.label}")
@@ -319,29 +324,79 @@ object OdcParser {
          }
      }
    }
+
+    def cvssScore = {
+      val cvssV2 = node \ "cvssV2"
+      if (cvssV2.nonEmpty) {
+        CvssRating(
+          score = (cvssV2 \ "score").headOption.map(_.text.toDouble),
+          authenticationr = t(cvssV2 \ "authenticationr"),
+          availabilityImpact = t(cvssV2 \ "availabilityImpact"),
+          accessVector = t(cvssV2 \ "accessVector"),
+          integrityImpact = t(cvssV2 \ "integrityImpact"),
+          accessComplexity = t(cvssV2 \ "accessComplexity"),
+          confidentialImpact = t(cvssV2 \ "confidentialImpact")
+        )
+      } else {
+        CvssRating(
+          score = (node \ "cvssScore").headOption.map(_.text.toDouble),
+          authenticationr = t(node \ "cvssAuthenticationr"),
+          availabilityImpact = t(node \ "cvssAvailabilityImpact"),
+          accessVector = t(node \ "cvssAccessVector"),
+          integrityImpact = t(node \ "cvssIntegrityImpact"),
+          accessComplexity = t(node \ "cvssAccessComplexity"),
+          confidentialImpact = t(node \ "cvssConfidentialImpact")
+        )
+      }
+    }
+
    vulnPool(Vulnerability(
      name = (node \ "name").text,
      //severity = (node \ "severity"), <- severity is useless, as it is computed from cvssScore :D
-      cweOption = (node \ "cwe").headOption.map(_.text).map(CWE.forIdentifierWithDescription),
+      //cweOption = (node \ "cwe").headOption.map(_.text).map(CWE.forIdentifierWithDescription),
      description = (node \ "description").text,
-      cvss = CvssRating(
-        score = (node \ "cvssScore").headOption.map(_.text.toDouble),
-        authenticationr = t(node \ "cvssAuthenticationr"),
-        availabilityImpact = t(node \ "cvssAvailabilityImpact"),
-        accessVector = t(node \ "cvssAccessVector"),
-        integrityImpact = t(node \ "cvssIntegrityImpact"),
-        accessComplexity = t(node \ "cvssAccessComplexity"),
-        confidentialImpact = t(node \ "cvssConfidentialImpact")
-      ),
+      cvss = cvssScore,
      references = (node \ "references").flatMap(filterWhitespace).map(parseReference(_)),
      vulnerableSoftware = (node \ "vulnerableSoftware").flatMap(filterWhitespace).map(parseVulnerableSoftware)
    ))
  }

  def parseIdentifier(node: Node, expectedLabel: String, parseConfidence: Boolean = true): Identifier = {
-    if(node.label != expectedLabel){
-      sys.error("Unexpected label for identifier: "+node.label)
+    // Old ODC produces expectedLabel, new ODC produces package and vulnerabilityIds
+    node.label match {
+      case "suppressedIdentifier" if (node \ "id").nonEmpty => parseIdentifierNew(node, parseConfidence, matched=false) // not sure if matched
+      case `expectedLabel` => parseIdentifierOld(node, parseConfidence)
+      case "package" => parseIdentifierNew(node, parseConfidence, matched=false)
+      case "vulnerabilityIds" => parseIdentifierNew(node, parseConfidence, matched=true)
+      case "suppressedVulnerabilityIds" => parseIdentifierNew(node, parseConfidence, matched=true)
+      case label => sys.error(s"Expected node name package or vulnerabilityIds or $expectedLabel, got: "+label)
    }
+  }
+
+  private val NugetPattern = """^pkg:nuget/([^@]+)@(.*)$""".r
+  private val MavenPattern = """^pkg:maven/([^/@]+)/([^/@]+)@(.*)$""".r
+  private val CpePattern = """^cpe:.*""".r
+
+  private def parseIdentifierNew(node: Node, parseConfidence: Boolean, matched: Boolean): Identifier = {
+    checkElements(node, Set("id", "url", "notes"))
+    // TODO: process currently ignored element “notes”
+    checkParams(node, Set("type", "confidence"))
+    val id = (node \ "id").text
+    val (identifierType, name) = id match {
+      case NugetPattern(name, version) => ("nuget", s"$name:$version")
+      case MavenPattern(groupId, artifactId, version) => ("maven", s"$groupId:$artifactId:$version")
+      case CpePattern() => ("cpe", id)
+      case _ => ("other", id)
+    }
+    identifierPool(Identifier(
+      name = name,
+      url = (node \ "url").text,
+      identifierType = identifierType,
+      confidence = if(parseConfidence) Confidence.withName(node.attribute("confidence").get.text) else Confidence.Medium
+    ))
+  }
+
+  private def parseIdentifierOld(node: Node, parseConfidence: Boolean): Identifier = {
    checkElements(node, Set("name", "url", "notes"))
    // TODO: process currently ignored element “notes”
    checkParams(node, Set("type", "confidence"))
@@ -352,7 +407,7 @@ object OdcParser {
        case text => text // used in new ODC
      },
      url = (node \ "url").text,
-      identifierType = node.attribute("type").get.text,
+      identifierType = try{node.attribute("type").get.text}catch{case e: NoSuchElementException => sys.error(s"No type attribute in $node")},
      confidence = if(parseConfidence) Confidence.withName(node.attribute("confidence").get.text) else Confidence.Medium
    ))
  }
@@ -362,7 +417,7 @@ object OdcParser {
    // TODO: process projectReferences
    checkParams(node, Set("isVirtual"))
    val (vulnerabilities: Seq[Node], suppressedVulnerabilities: Seq[Node]) = (node \ "vulnerabilities").headOption.map(filterWhitespace).getOrElse(Seq()).partition(_.label == "vulnerability")
-    val (identifiers, suppressedIdentifiers) = (node \ "identifiers").headOption.map(filterWhitespace).getOrElse(Seq()).partition(_.label == "identifier")
+    val (identifiers, suppressedIdentifiers) = (node \ "identifiers").headOption.map(filterWhitespace).getOrElse(Seq()).partition(!_.label.startsWith("suppressed"))
    dependencyPool(Dependency(
      fileName = (node \ "fileName").text,
      filePath = (node \ "filePath").text,
@@ -17,12 +17,12 @@ case class LibDepStatistics(libraries: Set[(Int, Library)], dependencies: Set[Gr
  lazy val vulnerableDependencies = dependencies.filter(_.isVulnerable)
  lazy val (dependenciesWithCpe, dependenciesWithoutCpe) = dependencies.partition(_.hasCpe)
  lazy val cpeRatio = dependenciesWithCpe.size.toDouble / dependencies.size.toDouble
-  lazy val weaknesses = vulnerabilities.flatMap(_.cweOption)
-  lazy val weaknessesFrequency = LibDepStatistics.computeWeaknessesFrequency(vulnerabilities)
+  //lazy val weaknesses = vulnerabilities.flatMap(_.cweOption)
+  //lazy val weaknessesFrequency = LibDepStatistics.computeWeaknessesFrequency(vulnerabilities)
 }

 object LibDepStatistics{
-  private def computeWeaknessesFrequency(vulnerabilities: Set[Vulnerability]) = vulnerabilities.toSeq.map(_.cweOption).groupBy(identity).mapValues(_.size).map(identity).withDefaultValue(0)
+  //private def computeWeaknessesFrequency(vulnerabilities: Set[Vulnerability]) = vulnerabilities.toSeq.map(_.cweOption).groupBy(identity).mapValues(_.size).map(identity).withDefaultValue(0)
  def apply(libraries: Set[(Int, Library)], dependencies: Set[GroupedDependency], parsedReports: Result): LibDepStatistics = LibDepStatistics(
    libraries = libraries,
    dependencies = dependencies,
@@ -11,7 +11,7 @@ abstract sealed class VulnerabilityOverview {
  def descriptionAttempt: String
  def isSureAboutDescription: Boolean
  def cvssScore: Option[Double]
-  def cweOption: Option[CWE]
+  //def cweOption: Option[CWE]
 }

 object VulnerabilityOverview{
@@ -23,21 +23,21 @@ final class StandardVulnerabilityOverview(vulnerability: Vulnerability) extends
  override def descriptionAttempt: String = vulnerability.description
  override def isSureAboutDescription = true
  override def cvssScore: Option[Double] = vulnerability.cvssScore
-  override def cweOption = vulnerability.cweOption
+  //override def cweOption = vulnerability.cweOption
 }

 private final class UnknownVulnerabilityOverview(override val name: String, link: String) extends VulnerabilityOverview {
  override def descriptionAttempt: String = s"Unknown vulnerability. Try looking at the following address for more details: $link"
  override def cvssScore: Option[Double] = None
  override def isSureAboutDescription = false
-  override def cweOption = None
+  //override def cweOption = None
 }

 private final class TotallyUnknownVulnerabilityOverview(override val name: String) extends VulnerabilityOverview {
  override def descriptionAttempt: String = s"Unknown vulnerability. Not even sure where to look for other details. Maybe Googling the identifier will help."
  override def cvssScore: Option[Double] = None
  override def isSureAboutDescription = false
-  override def cweOption = None
+  //override def cweOption = None
 }

 private object UnknownVulnerabilityOverview {
@@ -110,7 +110,7 @@ class JiraIssueTrackerService @Inject()(absolutizer: Absolutizer, @Named("jira-s
    "project" -> Json.obj(
      "id" -> projectId.toString
    ),
-    "summary" -> s"${vulnerability.name} – ${vulnerability.cweOption.map(_ + ": ").getOrElse("")}${vulnerability.description.take(50).takeWhile(c => c != '\n' && c != '\r')}…"
+    "summary" -> s"${vulnerability.name} – ${vulnerability.description.take(50).takeWhile(c => c != '\n' && c != '\r')}…"
  )

  private def extractManagedFields(vulnerability: VulnerabilityOverview, projects: Set[ReportInfo], requiresDescription: Boolean): JsObject = {
@@ -121,7 +121,7 @@ class JiraIssueTrackerService @Inject()(absolutizer: Absolutizer, @Named("jira-s
    )
    val descriptionObj = if(requiresDescription || vulnerability.isSureAboutDescription) Json.obj("description" -> extractDescription(vulnerability)) else Json.obj()
    val additionalFields = Seq[Option[(String, JsValueWrapper)]](
-      fields.cweId.map(id => id -> vulnerability.cweOption.fold("")(_.brief)),
+      //fields.cweId.map(id => id -> vulnerability.cweOption.fold("")(_.brief)),
      fields.linkId.map(id => id -> link(vulnerability)),
      fields.severityId.map(id => id -> vulnerability.cvssScore),
      fields.projectsId.map(id => id -> projects.map(friendlyProjectNameString).toSeq.sortBy( x => (x.toLowerCase(), x)).mkString("\n"))
@@ -48,7 +48,7 @@ class OdcDbService @Inject()(@NamedDatabase("odc") protected val dbConfigProvide
        } yield Some(
          com.ysoft.odc.Vulnerability(
            name = bareVuln.cve,
-            cweOption = bareVuln.cweOption,
+            //cweOption = bareVuln.cweOption,
            cvss = bareVuln.cvss,
            description = bareVuln.description,
            vulnerableSoftware = vulnerableSoftware,
@@ -53,7 +53,7 @@
    }


-    <div id="weakness" data-data='@{plotData(lds.weaknessesFrequency)}'></div>
+    @*<div id="weakness" data-data='@{plotData(lds.weaknessesFrequency)}'></div>*@
    <script type="text/javascript">

    var WeaknessIdentifier = function(brief, verbose){
@@ -122,7 +122,7 @@
        el.attr('data-initialized', 'true');
    };
    $(document).ready(function(){
-        initPlot('weakness');
+        //initPlot('weakness');
        var n=0;
        $('.stats').click(function(e){
            console.log(e);
@@ -174,7 +174,7 @@
                    </td>
                    <td class="text-right">
                        @s.vulnerabilities.size
-                        <button type="button" class="glyphicon glyphicon-signal btn btn-xs stats" @if(s.vulnerabilities.isEmpty){disabled="disabled"} data-data="@plotData(s.weaknessesFrequency)"></button>
+                        @*<button type="button" class="glyphicon glyphicon-signal btn btn-xs stats" @if(s.vulnerabilities.isEmpty){disabled="disabled"} data-data="@plotData(s.weaknessesFrequency)"></button>*@
                    </td>
                    <td class="text-right">@s.vulnerableDependencies.size</td>
                    <td class="text-right">@s.dependencies.size</td>
@@ -26,7 +26,7 @@
        case Some(score) => {<b>@score</b>}
    }
 </span></p>
-@vuln.cweOption.map{cwe =><p><label>Category:</label> <span class="explained" title="Vulnerability category according to Common Weakness Enumeration" onmouseover="$(this).tooltip({placement: 'right'}).tooltip('show')"><b>@cwe</b></span></p>}
+@*@vuln.cweOption.map{cwe =><p><label>Category:</label> <span class="explained" title="Vulnerability category according to Common Weakness Enumeration" onmouseover="$(this).tooltip({placement: 'right'}).tooltip('show')"><b>@cwe</b></span></p>}*@
 <label>Description:</label> @vuln.description
@section("vuln-sw", "Vulnerable software"){
    <ul id="@idPrefix-details">