Adapt for new version of ODC database

2026-01-11 14:30:50 +01:00 · 2020-01-31 02:07:08 +01:00
parent 52c3228ac3
commit c537a5c5c5
11 changed files with 108 additions and 107 deletions
--- a/app/com/ysoft/odc/OdcParser.scala
+++ b/app/com/ysoft/odc/OdcParser.scala
@@ -177,7 +177,7 @@ object Confidence extends Enumeration {

 final case class Reference(source: String, url: String, name: String)

-final case class VulnerableSoftware(allPreviousVersion: Boolean, name: String){
+final case class VulnerableSoftware(/*allPreviousVersion: Boolean,*/ name: String){
  def containsVersion: Boolean = name.count(_==':') >= 4
  def isCpe: Boolean = name.startsWith("cpe:")
  def isVersionless: Boolean = isCpe && !containsVersion
@@ -207,16 +207,16 @@ object RichBoolean{
  @inline implicit def toRichBoolean(value: Boolean) = new RichBoolean(value)
 }

-final case class Vulnerability(name: String, /*cweOption: Option[CWE],*/ cvss: CvssRating, description: String, vulnerableSoftware: Seq[VulnerableSoftware], references: Seq[Reference]){
+final case class Vulnerability(name: String, /*cweOption: Option[CWE],*/ cvss: CvssRating, description: String, /*vulnerableSoftware: Seq[VulnerableSoftware],*/ references: Seq[Reference]){
  import RichBoolean.toRichBoolean
  def cvssScore = cvss.score
-  def likelyMatchesOnlyWithoutVersion(dependencyIdentifiers: Set[Identifier]) = dependencyIdentifiers.forall { id =>
-    // Rather a quick hack. Maybe it would be better to do this check in ODC.
-    val versionlessCpeIdentifierOption = id.toCpeIdentifierOption.map(_.split(':').take(4).mkString(":"))
-    versionlessCpeIdentifierOption.fold(true){ versionlessCpeIdentifier =>
-      vulnerableSoftware.forall(vs => vs.name.startsWith(versionlessCpeIdentifier) ==> vs.isVersionless)
-    }
-  }
+//  def likelyMatchesOnlyWithoutVersion(dependencyIdentifiers: Set[Identifier]) = dependencyIdentifiers.forall { id =>
+//    // Rather a quick hack. Maybe it would be better to do this check in ODC.
+//    val versionlessCpeIdentifierOption = id.toCpeIdentifierOption.map(_.split(':').take(4).mkString(":"))
+//    versionlessCpeIdentifierOption.fold(true){ versionlessCpeIdentifier =>
+//      vulnerableSoftware.forall(vs => vs.name.startsWith(versionlessCpeIdentifier) ==> vs.isVersionless)
+//    }
+//  }
 }

 final case class Identifier(name: String, confidence: Confidence.Confidence, url: String, identifierType: String) {
@@ -284,8 +284,8 @@ object OdcParser {
      sys.error(s"Unexpected element for vulnerableSoftware: ${node.label}")
    }
    vulnerableSoftwarePool(VulnerableSoftware(
-      name = node.text,
-      allPreviousVersion = node.boolAttribute("allPreviousVersion").getOrElse(false)
+      name = node.text
+      //allPreviousVersion = node.boolAttribute("allPreviousVersion").getOrElse(false)
    ))
  }

@@ -356,8 +356,8 @@ object OdcParser {
      //cweOption = (node \ "cwe").headOption.map(_.text).map(CWE.forIdentifierWithDescription),
      description = (node \ "description").text,
      cvss = cvssScore,
-      references = (node \ "references").flatMap(filterWhitespace).map(parseReference(_)),
-      vulnerableSoftware = (node \ "vulnerableSoftware").flatMap(filterWhitespace).map(parseVulnerableSoftware)
+      references = (node \ "references").flatMap(filterWhitespace).map(parseReference(_))
+      //vulnerableSoftware = (node \ "vulnerableSoftware").flatMap(filterWhitespace).map(parseVulnerableSoftware)
    ))
  }

--- a/app/controllers/Statistics.scala
+++ b/app/controllers/Statistics.scala
@@ -129,35 +129,35 @@ class Statistics @Inject()(
  private def select(allResults: (Map[String, (Build, ArtifactItem, ArtifactFile)], Map[String, Throwable]), selectorOption: Option[String]): Option[ResultWithSelection] = select(allResults._1, allResults._2, selectorOption)
  private def select(successfulResults: Map[String, (Build, ArtifactItem, ArtifactFile)], failedResults: Map[String, Throwable], selectorOption: Option[String]): Option[ResultWithSelection] = dependencyCheckReportsParser.parseReports(successfulResults, failedResults).selection(selectorOption)

-  def searchVulnerableSoftware(versionlessCpes: Seq[String], versionOption: Option[String]) = ReadAction.async{ implicit req =>
-    if(versionlessCpes.isEmpty){
-      Future.successful(notFound())
-    }else{
-      val now = DateTime.now()
-      val oldDataThreshold = 2.days
-      val lastDbUpdateFuture = odcDbService.loadLastDbUpdate()
-      val isOldFuture = lastDbUpdateFuture.map{ lastUpdate => now - oldDataThreshold > lastUpdate}
-      versionOption match {
-        case Some(version) =>
-          for {
-            res1 <- Future.traverse(versionlessCpes) { versionlessCpe => odcDbService.findRelevantCpes(versionlessCpe, version) }
-            vulnIds = res1.flatten.map(_.vulnerabilityId).toSet
-            vulns <- Future.traverse(vulnIds)(id => odcDbService.getVulnerabilityDetails(id).map(_.get))
-            isOld <- isOldFuture
-          } yield Ok(views.html.statistics.vulnerabilitiesForLibrary(
-            vulnsAndVersionOption = Some((vulns, version)),
-            cpes = versionlessCpes,
-            isDbOld = isOld
-          ))
-        case None =>
-          for(isOld <- isOldFuture) yield Ok(views.html.statistics.vulnerabilitiesForLibrary(
-            vulnsAndVersionOption = None,
-            cpes = versionlessCpes,
-            isDbOld = isOld
-          ))
-      }
-    }
-  }
+//  def searchVulnerableSoftware(versionlessCpes: Seq[String], versionOption: Option[String]) = ReadAction.async{ implicit req =>
+//    if(versionlessCpes.isEmpty){
+//      Future.successful(notFound())
+//    }else{
+//      val now = DateTime.now()
+//      val oldDataThreshold = 2.days
+//      val lastDbUpdateFuture = odcDbService.loadLastDbUpdate()
+//      val isOldFuture = lastDbUpdateFuture.map{ lastUpdate => now - oldDataThreshold > lastUpdate}
+//      versionOption match {
+//        case Some(version) =>
+//          for {
+//            res1 <- Future.traverse(versionlessCpes) { versionlessCpe => odcDbService.findRelevantCpes(versionlessCpe, version) }
+//            vulnIds = res1.flatten.map(_.vulnerabilityId).toSet
+//            vulns <- Future.traverse(vulnIds)(id => odcDbService.getVulnerabilityDetails(id).map(_.get))
+//            isOld <- isOldFuture
+//          } yield Ok(views.html.statistics.vulnerabilitiesForLibrary(
+//            vulnsAndVersionOption = Some((vulns, version)),
+//            cpes = versionlessCpes,
+//            isDbOld = isOld
+//          ))
+//        case None =>
+//          for(isOld <- isOldFuture) yield Ok(views.html.statistics.vulnerabilitiesForLibrary(
+//            vulnsAndVersionOption = None,
+//            cpes = versionlessCpes,
+//            isDbOld = isOld
+//          ))
+//      }
+//    }
+//  }

  def basic(selectorOption: Option[String]) = ReadAction.async{ implicit req =>
    val (lastRefreshTime, resultsFuture) = projectReportsProvider.resultsForVersions(versions)
--- a/app/controllers/package.scala
+++ b/app/controllers/package.scala
@@ -47,11 +47,12 @@ package object controllers {
  def vulnerableSoftwareSearches(groupedDependency: GroupedDependency): Seq[(Call, String)] = {
    val legacySearchOption = groupedDependency.cpeIdentifiers match {
      case Seq() => None
-      case cpeIds => Some(
-        routes.Statistics.searchVulnerableSoftware(
-          cpeIds.map(_.name.split(':').take(4).mkString(":")).toSeq, None
-        ) -> "Search by CPE (legacy option)"
-      )
+      case cpeIds => None
+//        Some(
+//        routes.Statistics.searchVulnerableSoftware(
+//          cpeIds.map(_.name.split(':').take(4).mkString(":")).toSeq, None
+//        ) -> "Search by CPE (legacy option)"
+//      )
    }
    val mavenSearches = groupedDependency.mavenIdentifiers.map(_.name).toSeq.sorted.map{mavenIdentifier =>
      val Array(groupId, artifactId, version) = mavenIdentifier.split(":", 3)
--- a/app/models/odc/SoftwareVulnerability.scala
+++ b/app/models/odc/SoftwareVulnerability.scala
@@ -9,12 +9,12 @@ import slick.jdbc.JdbcType
 import scala.reflect.ClassTag

 // TODO: consider renaming to CpeEntryVulnerability or something like that
-final case class SoftwareVulnerability (vulnerabilityId: Int, cpeEntryId: Int, includesAllPreviousVersionsRaw: Option[String]){
-  def includesAllPreviousVersions: Boolean = includesAllPreviousVersionsRaw match {
-    case Some("1") => true
-    case None => false
-    case _ => sys.error("Unexpected value from ODC database")
-  }
+final case class SoftwareVulnerability (vulnerabilityId: Int, cpeEntryId: Int/*, includesAllPreviousVersionsRaw: Option[String]*/){
+//  def includesAllPreviousVersions: Boolean = includesAllPreviousVersionsRaw match {
+//    case Some("1") => true
+//    case None => false
+//    case _ => sys.error("Unexpected value from ODC database")
+//  }
 }

 /*private class OdcBooleanType(implicit t: JdbcType[Option[String]]) extends MappedJdbcType[Boolean, Option[String]] {
@@ -35,6 +35,6 @@ class SoftwareVulnerabilities(tag: Tag) extends Table[SoftwareVulnerability](tag
  def cpeEntryId = column[Int]("cpeentryid")
  //private val bt = new OdcBooleanType()(jdbcTypeFor(implicitly[BaseColumnType[String]].optionType).asInstanceOf[JdbcType[Option[String]]])
  //MappedJdbcType.base[Boolean, Option[String]](???, ???)(implicitly[ClassTag[Boolean]], )
-  def includesAllPreviousVersionsRaw = column[String]("previousversion").?
-  def * = (vulnerabilityId, cpeEntryId, includesAllPreviousVersionsRaw) <> (SoftwareVulnerability.tupled, SoftwareVulnerability.unapply)
+  //def includesAllPreviousVersionsRaw = column[String]("previousversion").?
+  def * = (vulnerabilityId, cpeEntryId/*, includesAllPreviousVersionsRaw*/) <> (SoftwareVulnerability.tupled, SoftwareVulnerability.unapply)
 }
--- a/app/models/odc/Vulnerability.scala
+++ b/app/models/odc/Vulnerability.scala
@@ -4,23 +4,23 @@ import com.ysoft.odc.{CvssRating, CWE}
 import models.odc.profile.api._
 import slick.lifted.Tag

-case class Vulnerability (cve: String, description: String, cweOption: Option[CWE], cvss: CvssRating)
+case class Vulnerability (cve: String, description: String, /*cweOption: Option[CWE],*/ cvss: CvssRating)
 class Vulnerabilities(tag: Tag) extends Table[(Int, Vulnerability)](tag, "vulnerability") {
  def id = column[Int]("id")
  def cve = column[String]("cve")
  def description = column[String]("description")
-  def cweOption = column[String]("cwe").?
-  def cvssScore = column[Double]("cvssscore").?
-  def authentication = column[String]("cvssauthentication").?
-  def availabilityImpact = column[String]("cvssavailabilityimpact").?
-  def accessVector = column[String]("cvssaccessvector").?
-  def integrityImpact = column[String]("cvssintegrityimpact").?
-  def cvssAccessComplexity = column[String]("cvssaccesscomplexity").?
-  def cvssConfidentialityImpact = column[String]("cvssconfidentialityimpact").?
+  //def cweOption = column[String]("cwe").?
+  def cvssScore = column[Double]("cvssv2score").?
+  def authentication = column[String]("cvssv2authentication").?
+  def availabilityImpact = column[String]("cvssv2availabilityimpact").?
+  def accessVector = column[String]("cvssv2accessvector").?
+  def integrityImpact = column[String]("cvssv2integrityimpact").?
+  def cvssAccessComplexity = column[String]("cvssv2accesscomplexity").?
+  def cvssConfidentialityImpact = column[String]("cvssv2confidentialityimpact").?

  def cvssRating = (cvssScore, authentication, availabilityImpact, accessVector, integrityImpact, cvssAccessComplexity, cvssConfidentialityImpact) <> (CvssRating.tupled, CvssRating.unapply)
-  def cweOptionMapped = cweOption <> ((_: Option[String]).map(CWE.forIdentifierWithDescription), (_: Option[CWE]).map(CWE.unapply))
-  def base = (cve, description, cweOptionMapped, cvssRating) <> (Vulnerability.tupled, Vulnerability.unapply)
+  //def cweOptionMapped = cweOption <> ((_: Option[String]).map(CWE.forIdentifierWithDescription), (_: Option[CWE]).map(CWE.unapply))
+  def base = (cve, description, cvssRating) <> (Vulnerability.tupled, Vulnerability.unapply)

  def * = (id, base)
 }
--- a/app/services/OdcDbService.scala
+++ b/app/services/OdcDbService.scala
@@ -23,13 +23,13 @@ class OdcDbService @Inject()(@NamedDatabase("odc") protected val dbConfigProvide

  import dbConfig.driver.api._

-  private def getVulnerableSoftware(id: Int): Future[Seq[com.ysoft.odc.VulnerableSoftware]] = {
-    db.run(softwareVulnerabilities.joinLeft(cpeEntries).on((sv, ce) => sv.cpeEntryId === ce.id).filter{case (sv, ceo) => sv.vulnerabilityId === id}.result).map{rawRefs =>
-      rawRefs.map{
-        case (softVuln, Some((_, cpeEntry))) => com.ysoft.odc.VulnerableSoftware(allPreviousVersion = softVuln.includesAllPreviousVersions, name=cpeEntry.cpe)
-      }
-    }
-  }
+//  private def getVulnerableSoftware(id: Int): Future[Seq[com.ysoft.odc.VulnerableSoftware]] = {
+//    db.run(softwareVulnerabilities.joinLeft(cpeEntries).on((sv, ce) => sv.cpeEntryId === ce.id).filter{case (sv, ceo) => sv.vulnerabilityId === id}.result).map{rawRefs =>
+//      rawRefs.map{
+//        case (softVuln, Some((_, cpeEntry))) => com.ysoft.odc.VulnerableSoftware(/*allPreviousVersion = softVuln.includesAllPreviousVersions, */name=cpeEntry.cpe)
+//      }
+//    }
+//  }

  private def getReferences(id: Int): Future[Seq[com.ysoft.odc.Reference]] = db.run(references.filter(_.cveId === id).map(_.base).result)

@@ -43,7 +43,7 @@ class OdcDbService @Inject()(@NamedDatabase("odc") protected val dbConfigProvide
    db.run(vulnerabilities.filter(cond).result).map(_.headOption) flatMap { bareVulnOption =>
      bareVulnOption.fold[Future[Option[com.ysoft.odc.Vulnerability]]](Future.successful(None)) { case (id, bareVuln) =>
        for {
-          vulnerableSoftware <- getVulnerableSoftware(id)
+//          vulnerableSoftware <- getVulnerableSoftware(id)
          references <- getReferences(id)
        } yield Some(
          com.ysoft.odc.Vulnerability(
@@ -51,7 +51,7 @@ class OdcDbService @Inject()(@NamedDatabase("odc") protected val dbConfigProvide
            //cweOption = bareVuln.cweOption,
            cvss = bareVuln.cvss,
            description = bareVuln.description,
-            vulnerableSoftware = vulnerableSoftware,
+            //vulnerableSoftware = vulnerableSoftware,
            references = references
          )
        )
@@ -69,34 +69,34 @@ class OdcDbService @Inject()(@NamedDatabase("odc") protected val dbConfigProvide
    DependencyVersionUtil.parseVersion(version)
  }

-  def findRelevantCpes(versionlessCpe: String, version: String) = {
-    println(s"versionlessCpe: $versionlessCpe")
-    val Seq("cpe", "/a", vendor, product, rest @ _*) = versionlessCpe.split(':').toSeq
-    val cpesFuture = db.run(
-      cpeEntries.filter(c =>
-        c.vendor === vendor && c.product === product
-      ).result
-    )
-    for(cpes <- cpesFuture){println(s"cpes: $cpes")}
-    val cpesMapFuture = cpesFuture.map(_.toMap)
-    val cpeIdsFuture = cpesFuture.map(_.map(_._1))
-    val parsedVersion = parseVersion(version)
-    val res = for{
-      cpeIds <- cpeIdsFuture
-      relevantVulnerabilities <- db.run(
-        softwareVulnerabilities.join(vulnerabilities).on( (sv, v) => sv.vulnerabilityId === v.id)
-          .filter{case (sv, v) => sv.cpeEntryId inSet cpeIds}.map{case (sv, v) ⇒ sv}.result
-      ).map(_.groupBy(_.vulnerabilityId).mapValues(_.toSet))
-      cpesMap <- cpesMapFuture
-      //relevantVulnerabilities <- db.run(vulnerabilities.filter(_.id inSet relevantVulnerabilityIds).result)
-    } yield relevantVulnerabilities.filter{case (vulnId, sv) => Option(CveDbHelper.matchSofware(
-      vulnerableSoftware = sv.map(sv => cpesMap(sv.cpeEntryId).cpe -> sv.includesAllPreviousVersions).toMap,
-      vendor = vendor,
-      product = product,
-      identifiedVersion = parsedVersion
-    )).isDefined}
-    res.map(_.values.toSet.flatten)
-  }
+//  def findRelevantCpes(versionlessCpe: String, version: String) = {
+//    println(s"versionlessCpe: $versionlessCpe")
+//    val Seq("cpe", "/a", vendor, product, rest @ _*) = versionlessCpe.split(':').toSeq
+//    val cpesFuture = db.run(
+//      cpeEntries.filter(c =>
+//        c.vendor === vendor && c.product === product
+//      ).result
+//    )
+//    for(cpes <- cpesFuture){println(s"cpes: $cpes")}
+//    val cpesMapFuture = cpesFuture.map(_.toMap)
+//    val cpeIdsFuture = cpesFuture.map(_.map(_._1))
+//    val parsedVersion = parseVersion(version)
+//    val res = for{
+//      cpeIds <- cpeIdsFuture
+//      relevantVulnerabilities <- db.run(
+//        softwareVulnerabilities.join(vulnerabilities).on( (sv, v) => sv.vulnerabilityId === v.id)
+//          .filter{case (sv, v) => sv.cpeEntryId inSet cpeIds}.map{case (sv, v) ⇒ sv}.result
+//      ).map(_.groupBy(_.vulnerabilityId).mapValues(_.toSet))
+//      cpesMap <- cpesMapFuture
+//      //relevantVulnerabilities <- db.run(vulnerabilities.filter(_.id inSet relevantVulnerabilityIds).result)
+//    } yield relevantVulnerabilities.filter{case (vulnId, sv) => Option(CveDbHelper.matchSofware(
+//      vulnerableSoftware = sv.map(sv => cpesMap(sv.cpeEntryId).cpe -> sv.includesAllPreviousVersions).toMap,
+//      vendor = vendor,
+//      product = product,
+//      identifiedVersion = parsedVersion
+//    )).isDefined}
+//    res.map(_.values.toSet.flatten)
+//  }

  private def loadUpdateProperties(): Future[Map[String, Long]] = db.run(properties.filter(_.id like "NVD CVE%").result).map(_.map{case OdcProperty(id, value) => (id, value.toLong)}.toMap)

--- a/app/views/dependencyDetailsInner.scala.html
+++ b/app/views/dependencyDetailsInner.scala.html
@@ -97,7 +97,7 @@
            <h5 data-toggle="collapse" class="expandable@if(!expandVulnerabilities){ collapsed}" data-target="#@vulnPrefix-details">
                @vuln.name
                <a href="@routes.Statistics.vulnerability(vuln.name, selectorOption)" target="_blank" onclick="event.stopPropagation();"><span class="glyphicon glyphicon-new-window"></span></a>
-                @if(vuln.likelyMatchesOnlyWithoutVersion(dep.identifiers)){<span class="warning-expandable" title="Heuristics suspect false positive. Double check <b>what version</b> does this vulnerability apply to, please. It seems that the vulnerability database does not provide enough information to check it automatically." onmouseover="$(this).tooltip({placement: 'right', html:true}).tooltip('show');"></span>}
+                @*@if(vuln.likelyMatchesOnlyWithoutVersion(dep.identifiers)){<span class="warning-expandable" title="Heuristics suspect false positive. Double check <b>what version</b> does this vulnerability apply to, please. It seems that the vulnerability database does not provide enough information to check it automatically." onmouseover="$(this).tooltip({placement: 'right', html:true}).tooltip('show');"></span>}*@
            </h5>
            <div id="@vulnPrefix-details" class="collapse vulnerability-expandable@if(expandVulnerabilities){ in}">
                @vulnerability("h6", depPrefix+"-"+vuln.name, vuln)
--- a/app/views/statistics/vulnerabilitiesForLibrary.scala.html.old
+++ b/app/views/statistics/vulnerabilitiesForLibrary.scala.html.old
--- a/app/views/vulnerability.scala.html
+++ b/app/views/vulnerability.scala.html
@@ -28,13 +28,13 @@
 </span></p>
@*@vuln.cweOption.map{cwe =><p><label>Category:</label> <span class="explained" title="Vulnerability category according to Common Weakness Enumeration" onmouseover="$(this).tooltip({placement: 'right'}).tooltip('show')"><b>@cwe</b></span></p>}*@
 <label>Description:</label> @vuln.description
-@section("vuln-sw", "Vulnerable software"){
+@*@section("vuln-sw", "Vulnerable software"){
    <ul id="@idPrefix-details">
    @for(sw <- vuln.vulnerableSoftware){
        <li>@sw.name@if(sw.allPreviousVersion){ and all previous versions}@if(sw.isVersionless){<span class="warning-expandable" title="This identifier does not contain version. It will match <b>any</b> version, which might cause false positives." onmouseover="$(this).tooltip({placement: 'right', html:true}).tooltip('show');"></span>}</li>
    }
    </ul>
-}
+}*@
@section("references", "References"){
    <ul>
    @for(reference <- vuln.references){
--- a/conf/routes
+++ b/conf/routes
@@ -45,7 +45,7 @@ POST        /notifications/watch              controllers.Notifications.watch(pr
 POST        /notifications/unwatch            controllers.Notifications.unwatch(project: String, filter: Option[String])
 GET         /notifications/cron/:key          controllers.Notifications.cron(key: String, purgeCache: Boolean ?= true)

-GET         /libraries/vulnerabilities        controllers.Statistics.searchVulnerableSoftware(versionlessCpes: Seq[String], versionOption: Option[String])
+#GET         /libraries/vulnerabilities        controllers.Statistics.searchVulnerableSoftware(versionlessCpes: Seq[String], versionOption: Option[String])

 GET         /vulnerability/:name              controllers.Statistics.vulnerability(name, selector: Option[String])

--- a/test/VulnerabilitySpec.scala.old
+++ b/test/VulnerabilitySpec.scala.old