Adapt for new version of ODC database

app/com/ysoft/odc/OdcParser.scala

@@ -177,7 +177,7 @@ object Confidence extends Enumeration {
 
 final case class Reference(source: String, url: String, name: String)
 
-final case class VulnerableSoftware(allPreviousVersion: Boolean, name: String){
+final case class VulnerableSoftware(/*allPreviousVersion: Boolean,*/ name: String){
   def containsVersion: Boolean = name.count(_==':') >= 4
   def isCpe: Boolean = name.startsWith("cpe:")
   def isVersionless: Boolean = isCpe && !containsVersion
@@ -207,16 +207,16 @@ object RichBoolean{
   @inline implicit def toRichBoolean(value: Boolean) = new RichBoolean(value)
 }
 
-final case class Vulnerability(name: String, /*cweOption: Option[CWE],*/ cvss: CvssRating, description: String, vulnerableSoftware: Seq[VulnerableSoftware], references: Seq[Reference]){
+final case class Vulnerability(name: String, /*cweOption: Option[CWE],*/ cvss: CvssRating, description: String, /*vulnerableSoftware: Seq[VulnerableSoftware],*/ references: Seq[Reference]){
   import RichBoolean.toRichBoolean
   def cvssScore = cvss.score
-  def likelyMatchesOnlyWithoutVersion(dependencyIdentifiers: Set[Identifier]) = dependencyIdentifiers.forall { id =>
-    // Rather a quick hack. Maybe it would be better to do this check in ODC.
-    val versionlessCpeIdentifierOption = id.toCpeIdentifierOption.map(_.split(':').take(4).mkString(":"))
-    versionlessCpeIdentifierOption.fold(true){ versionlessCpeIdentifier =>
-      vulnerableSoftware.forall(vs => vs.name.startsWith(versionlessCpeIdentifier) ==> vs.isVersionless)
-    }
-  }
+//  def likelyMatchesOnlyWithoutVersion(dependencyIdentifiers: Set[Identifier]) = dependencyIdentifiers.forall { id =>
+//    // Rather a quick hack. Maybe it would be better to do this check in ODC.
+//    val versionlessCpeIdentifierOption = id.toCpeIdentifierOption.map(_.split(':').take(4).mkString(":"))
+//    versionlessCpeIdentifierOption.fold(true){ versionlessCpeIdentifier =>
+//      vulnerableSoftware.forall(vs => vs.name.startsWith(versionlessCpeIdentifier) ==> vs.isVersionless)
+//    }
+//  }
 }
 
 final case class Identifier(name: String, confidence: Confidence.Confidence, url: String, identifierType: String) {
@@ -284,8 +284,8 @@ object OdcParser {
       sys.error(s"Unexpected element for vulnerableSoftware: ${node.label}")
     }
     vulnerableSoftwarePool(VulnerableSoftware(
-      name = node.text,
-      allPreviousVersion = node.boolAttribute("allPreviousVersion").getOrElse(false)
+      name = node.text
+      //allPreviousVersion = node.boolAttribute("allPreviousVersion").getOrElse(false)
     ))
   }
 
@@ -356,8 +356,8 @@ object OdcParser {
       //cweOption = (node \ "cwe").headOption.map(_.text).map(CWE.forIdentifierWithDescription),
       description = (node \ "description").text,
       cvss = cvssScore,
-      references = (node \ "references").flatMap(filterWhitespace).map(parseReference(_)),
-      vulnerableSoftware = (node \ "vulnerableSoftware").flatMap(filterWhitespace).map(parseVulnerableSoftware)
+      references = (node \ "references").flatMap(filterWhitespace).map(parseReference(_))
+      //vulnerableSoftware = (node \ "vulnerableSoftware").flatMap(filterWhitespace).map(parseVulnerableSoftware)
     ))
   }
 

app/controllers/Statistics.scala

@@ -129,35 +129,35 @@ class Statistics @Inject()(
   private def select(allResults: (Map[String, (Build, ArtifactItem, ArtifactFile)], Map[String, Throwable]), selectorOption: Option[String]): Option[ResultWithSelection] = select(allResults._1, allResults._2, selectorOption)
   private def select(successfulResults: Map[String, (Build, ArtifactItem, ArtifactFile)], failedResults: Map[String, Throwable], selectorOption: Option[String]): Option[ResultWithSelection] = dependencyCheckReportsParser.parseReports(successfulResults, failedResults).selection(selectorOption)
 
-  def searchVulnerableSoftware(versionlessCpes: Seq[String], versionOption: Option[String]) = ReadAction.async{ implicit req =>
-    if(versionlessCpes.isEmpty){
-      Future.successful(notFound())
-    }else{
-      val now = DateTime.now()
-      val oldDataThreshold = 2.days
-      val lastDbUpdateFuture = odcDbService.loadLastDbUpdate()
-      val isOldFuture = lastDbUpdateFuture.map{ lastUpdate => now - oldDataThreshold > lastUpdate}
-      versionOption match {
-        case Some(version) =>
-          for {
-            res1 <- Future.traverse(versionlessCpes) { versionlessCpe => odcDbService.findRelevantCpes(versionlessCpe, version) }
-            vulnIds = res1.flatten.map(_.vulnerabilityId).toSet
-            vulns <- Future.traverse(vulnIds)(id => odcDbService.getVulnerabilityDetails(id).map(_.get))
-            isOld <- isOldFuture
-          } yield Ok(views.html.statistics.vulnerabilitiesForLibrary(
-            vulnsAndVersionOption = Some((vulns, version)),
-            cpes = versionlessCpes,
-            isDbOld = isOld
-          ))
-        case None =>
-          for(isOld <- isOldFuture) yield Ok(views.html.statistics.vulnerabilitiesForLibrary(
-            vulnsAndVersionOption = None,
-            cpes = versionlessCpes,
-            isDbOld = isOld
-          ))
-      }
-    }
-  }
+//  def searchVulnerableSoftware(versionlessCpes: Seq[String], versionOption: Option[String]) = ReadAction.async{ implicit req =>
+//    if(versionlessCpes.isEmpty){
+//      Future.successful(notFound())
+//    }else{
+//      val now = DateTime.now()
+//      val oldDataThreshold = 2.days
+//      val lastDbUpdateFuture = odcDbService.loadLastDbUpdate()
+//      val isOldFuture = lastDbUpdateFuture.map{ lastUpdate => now - oldDataThreshold > lastUpdate}
+//      versionOption match {
+//        case Some(version) =>
+//          for {
+//            res1 <- Future.traverse(versionlessCpes) { versionlessCpe => odcDbService.findRelevantCpes(versionlessCpe, version) }
+//            vulnIds = res1.flatten.map(_.vulnerabilityId).toSet
+//            vulns <- Future.traverse(vulnIds)(id => odcDbService.getVulnerabilityDetails(id).map(_.get))
+//            isOld <- isOldFuture
+//          } yield Ok(views.html.statistics.vulnerabilitiesForLibrary(
+//            vulnsAndVersionOption = Some((vulns, version)),
+//            cpes = versionlessCpes,
+//            isDbOld = isOld
+//          ))
+//        case None =>
+//          for(isOld <- isOldFuture) yield Ok(views.html.statistics.vulnerabilitiesForLibrary(
+//            vulnsAndVersionOption = None,
+//            cpes = versionlessCpes,
+//            isDbOld = isOld
+//          ))
+//      }
+//    }
+//  }
 
   def basic(selectorOption: Option[String]) = ReadAction.async{ implicit req =>
     val (lastRefreshTime, resultsFuture) = projectReportsProvider.resultsForVersions(versions)

app/controllers/package.scala

@@ -47,11 +47,12 @@ package object controllers {
   def vulnerableSoftwareSearches(groupedDependency: GroupedDependency): Seq[(Call, String)] = {
     val legacySearchOption = groupedDependency.cpeIdentifiers match {
       case Seq() => None
-      case cpeIds => Some(
-        routes.Statistics.searchVulnerableSoftware(
-          cpeIds.map(_.name.split(':').take(4).mkString(":")).toSeq, None
-        ) -> "Search by CPE (legacy option)"
-      )
+      case cpeIds => None
+//        Some(
+//        routes.Statistics.searchVulnerableSoftware(
+//          cpeIds.map(_.name.split(':').take(4).mkString(":")).toSeq, None
+//        ) -> "Search by CPE (legacy option)"
+//      )
     }
     val mavenSearches = groupedDependency.mavenIdentifiers.map(_.name).toSeq.sorted.map{mavenIdentifier =>
       val Array(groupId, artifactId, version) = mavenIdentifier.split(":", 3)

app/models/odc/SoftwareVulnerability.scala

@@ -9,12 +9,12 @@ import slick.jdbc.JdbcType
 import scala.reflect.ClassTag
 
 // TODO: consider renaming to CpeEntryVulnerability or something like that
-final case class SoftwareVulnerability (vulnerabilityId: Int, cpeEntryId: Int, includesAllPreviousVersionsRaw: Option[String]){
-  def includesAllPreviousVersions: Boolean = includesAllPreviousVersionsRaw match {
-    case Some("1") => true
-    case None => false
-    case _ => sys.error("Unexpected value from ODC database")
-  }
+final case class SoftwareVulnerability (vulnerabilityId: Int, cpeEntryId: Int/*, includesAllPreviousVersionsRaw: Option[String]*/){
+//  def includesAllPreviousVersions: Boolean = includesAllPreviousVersionsRaw match {
+//    case Some("1") => true
+//    case None => false
+//    case _ => sys.error("Unexpected value from ODC database")
+//  }
 }
 
 /*private class OdcBooleanType(implicit t: JdbcType[Option[String]]) extends MappedJdbcType[Boolean, Option[String]] {
@@ -35,6 +35,6 @@ class SoftwareVulnerabilities(tag: Tag) extends Table[SoftwareVulnerability](tag
   def cpeEntryId = column[Int]("cpeentryid")
   //private val bt = new OdcBooleanType()(jdbcTypeFor(implicitly[BaseColumnType[String]].optionType).asInstanceOf[JdbcType[Option[String]]])
   //MappedJdbcType.base[Boolean, Option[String]](???, ???)(implicitly[ClassTag[Boolean]], )
-  def includesAllPreviousVersionsRaw = column[String]("previousversion").?
-  def * = (vulnerabilityId, cpeEntryId, includesAllPreviousVersionsRaw) <> (SoftwareVulnerability.tupled, SoftwareVulnerability.unapply)
+  //def includesAllPreviousVersionsRaw = column[String]("previousversion").?
+  def * = (vulnerabilityId, cpeEntryId/*, includesAllPreviousVersionsRaw*/) <> (SoftwareVulnerability.tupled, SoftwareVulnerability.unapply)
 }

app/models/odc/Vulnerability.scala

@@ -4,23 +4,23 @@ import com.ysoft.odc.{CvssRating, CWE}
 import models.odc.profile.api._
 import slick.lifted.Tag
 
-case class Vulnerability (cve: String, description: String, cweOption: Option[CWE], cvss: CvssRating)
+case class Vulnerability (cve: String, description: String, /*cweOption: Option[CWE],*/ cvss: CvssRating)
 class Vulnerabilities(tag: Tag) extends Table[(Int, Vulnerability)](tag, "vulnerability") {
   def id = column[Int]("id")
   def cve = column[String]("cve")
   def description = column[String]("description")
-  def cweOption = column[String]("cwe").?
-  def cvssScore = column[Double]("cvssscore").?
-  def authentication = column[String]("cvssauthentication").?
-  def availabilityImpact = column[String]("cvssavailabilityimpact").?
-  def accessVector = column[String]("cvssaccessvector").?
-  def integrityImpact = column[String]("cvssintegrityimpact").?
-  def cvssAccessComplexity = column[String]("cvssaccesscomplexity").?
-  def cvssConfidentialityImpact = column[String]("cvssconfidentialityimpact").?
+  //def cweOption = column[String]("cwe").?
+  def cvssScore = column[Double]("cvssv2score").?
+  def authentication = column[String]("cvssv2authentication").?
+  def availabilityImpact = column[String]("cvssv2availabilityimpact").?
+  def accessVector = column[String]("cvssv2accessvector").?
+  def integrityImpact = column[String]("cvssv2integrityimpact").?
+  def cvssAccessComplexity = column[String]("cvssv2accesscomplexity").?
+  def cvssConfidentialityImpact = column[String]("cvssv2confidentialityimpact").?
 
   def cvssRating = (cvssScore, authentication, availabilityImpact, accessVector, integrityImpact, cvssAccessComplexity, cvssConfidentialityImpact) <> (CvssRating.tupled, CvssRating.unapply)
-  def cweOptionMapped = cweOption <> ((_: Option[String]).map(CWE.forIdentifierWithDescription), (_: Option[CWE]).map(CWE.unapply))
-  def base = (cve, description, cweOptionMapped, cvssRating) <> (Vulnerability.tupled, Vulnerability.unapply)
+  //def cweOptionMapped = cweOption <> ((_: Option[String]).map(CWE.forIdentifierWithDescription), (_: Option[CWE]).map(CWE.unapply))
+  def base = (cve, description, cvssRating) <> (Vulnerability.tupled, Vulnerability.unapply)
 
   def * = (id, base)
 }

app/services/OdcDbService.scala

@@ -23,13 +23,13 @@ class OdcDbService @Inject()(@NamedDatabase("odc") protected val dbConfigProvide
 
   import dbConfig.driver.api._
 
-  private def getVulnerableSoftware(id: Int): Future[Seq[com.ysoft.odc.VulnerableSoftware]] = {
-    db.run(softwareVulnerabilities.joinLeft(cpeEntries).on((sv, ce) => sv.cpeEntryId === ce.id).filter{case (sv, ceo) => sv.vulnerabilityId === id}.result).map{rawRefs =>
-      rawRefs.map{
-        case (softVuln, Some((_, cpeEntry))) => com.ysoft.odc.VulnerableSoftware(allPreviousVersion = softVuln.includesAllPreviousVersions, name=cpeEntry.cpe)
-      }
-    }
-  }
+//  private def getVulnerableSoftware(id: Int): Future[Seq[com.ysoft.odc.VulnerableSoftware]] = {
+//    db.run(softwareVulnerabilities.joinLeft(cpeEntries).on((sv, ce) => sv.cpeEntryId === ce.id).filter{case (sv, ceo) => sv.vulnerabilityId === id}.result).map{rawRefs =>
+//      rawRefs.map{
+//        case (softVuln, Some((_, cpeEntry))) => com.ysoft.odc.VulnerableSoftware(/*allPreviousVersion = softVuln.includesAllPreviousVersions, */name=cpeEntry.cpe)
+//      }
+//    }
+//  }
 
   private def getReferences(id: Int): Future[Seq[com.ysoft.odc.Reference]] = db.run(references.filter(_.cveId === id).map(_.base).result)
 
@@ -43,7 +43,7 @@ class OdcDbService @Inject()(@NamedDatabase("odc") protected val dbConfigProvide
     db.run(vulnerabilities.filter(cond).result).map(_.headOption) flatMap { bareVulnOption =>
       bareVulnOption.fold[Future[Option[com.ysoft.odc.Vulnerability]]](Future.successful(None)) { case (id, bareVuln) =>
         for {
-          vulnerableSoftware <- getVulnerableSoftware(id)
+//          vulnerableSoftware <- getVulnerableSoftware(id)
           references <- getReferences(id)
         } yield Some(
           com.ysoft.odc.Vulnerability(
@@ -51,7 +51,7 @@ class OdcDbService @Inject()(@NamedDatabase("odc") protected val dbConfigProvide
             //cweOption = bareVuln.cweOption,
             cvss = bareVuln.cvss,
             description = bareVuln.description,
-            vulnerableSoftware = vulnerableSoftware,
+            //vulnerableSoftware = vulnerableSoftware,
             references = references
           )
         )
@@ -69,34 +69,34 @@ class OdcDbService @Inject()(@NamedDatabase("odc") protected val dbConfigProvide
     DependencyVersionUtil.parseVersion(version)
   }
 
-  def findRelevantCpes(versionlessCpe: String, version: String) = {
-    println(s"versionlessCpe: $versionlessCpe")
-    val Seq("cpe", "/a", vendor, product, rest @ _*) = versionlessCpe.split(':').toSeq
-    val cpesFuture = db.run(
-      cpeEntries.filter(c =>
-        c.vendor === vendor && c.product === product
-      ).result
-    )
-    for(cpes <- cpesFuture){println(s"cpes: $cpes")}
-    val cpesMapFuture = cpesFuture.map(_.toMap)
-    val cpeIdsFuture = cpesFuture.map(_.map(_._1))
-    val parsedVersion = parseVersion(version)
-    val res = for{
-      cpeIds <- cpeIdsFuture
-      relevantVulnerabilities <- db.run(
-        softwareVulnerabilities.join(vulnerabilities).on( (sv, v) => sv.vulnerabilityId === v.id)
-          .filter{case (sv, v) => sv.cpeEntryId inSet cpeIds}.map{case (sv, v) ⇒ sv}.result
-      ).map(_.groupBy(_.vulnerabilityId).mapValues(_.toSet))
-      cpesMap <- cpesMapFuture
-      //relevantVulnerabilities <- db.run(vulnerabilities.filter(_.id inSet relevantVulnerabilityIds).result)
-    } yield relevantVulnerabilities.filter{case (vulnId, sv) => Option(CveDbHelper.matchSofware(
-      vulnerableSoftware = sv.map(sv => cpesMap(sv.cpeEntryId).cpe -> sv.includesAllPreviousVersions).toMap,
-      vendor = vendor,
-      product = product,
-      identifiedVersion = parsedVersion
-    )).isDefined}
-    res.map(_.values.toSet.flatten)
-  }
+//  def findRelevantCpes(versionlessCpe: String, version: String) = {
+//    println(s"versionlessCpe: $versionlessCpe")
+//    val Seq("cpe", "/a", vendor, product, rest @ _*) = versionlessCpe.split(':').toSeq
+//    val cpesFuture = db.run(
+//      cpeEntries.filter(c =>
+//        c.vendor === vendor && c.product === product
+//      ).result
+//    )
+//    for(cpes <- cpesFuture){println(s"cpes: $cpes")}
+//    val cpesMapFuture = cpesFuture.map(_.toMap)
+//    val cpeIdsFuture = cpesFuture.map(_.map(_._1))
+//    val parsedVersion = parseVersion(version)
+//    val res = for{
+//      cpeIds <- cpeIdsFuture
+//      relevantVulnerabilities <- db.run(
+//        softwareVulnerabilities.join(vulnerabilities).on( (sv, v) => sv.vulnerabilityId === v.id)
+//          .filter{case (sv, v) => sv.cpeEntryId inSet cpeIds}.map{case (sv, v) ⇒ sv}.result
+//      ).map(_.groupBy(_.vulnerabilityId).mapValues(_.toSet))
+//      cpesMap <- cpesMapFuture
+//      //relevantVulnerabilities <- db.run(vulnerabilities.filter(_.id inSet relevantVulnerabilityIds).result)
+//    } yield relevantVulnerabilities.filter{case (vulnId, sv) => Option(CveDbHelper.matchSofware(
+//      vulnerableSoftware = sv.map(sv => cpesMap(sv.cpeEntryId).cpe -> sv.includesAllPreviousVersions).toMap,
+//      vendor = vendor,
+//      product = product,
+//      identifiedVersion = parsedVersion
+//    )).isDefined}
+//    res.map(_.values.toSet.flatten)
+//  }
 
   private def loadUpdateProperties(): Future[Map[String, Long]] = db.run(properties.filter(_.id like "NVD CVE%").result).map(_.map{case OdcProperty(id, value) => (id, value.toLong)}.toMap)
 

app/views/dependencyDetailsInner.scala.html

@@ -97,7 +97,7 @@
             <h5 data-toggle="collapse" class="expandable@if(!expandVulnerabilities){ collapsed}" data-target="#@vulnPrefix-details">
                 @vuln.name
                 <a href="@routes.Statistics.vulnerability(vuln.name, selectorOption)" target="_blank" onclick="event.stopPropagation();"><span class="glyphicon glyphicon-new-window"></span></a>
-                @if(vuln.likelyMatchesOnlyWithoutVersion(dep.identifiers)){<span class="warning-expandable" title="Heuristics suspect false positive. Double check <b>what version</b> does this vulnerability apply to, please. It seems that the vulnerability database does not provide enough information to check it automatically." onmouseover="$(this).tooltip({placement: 'right', html:true}).tooltip('show');"></span>}
+                @*@if(vuln.likelyMatchesOnlyWithoutVersion(dep.identifiers)){<span class="warning-expandable" title="Heuristics suspect false positive. Double check <b>what version</b> does this vulnerability apply to, please. It seems that the vulnerability database does not provide enough information to check it automatically." onmouseover="$(this).tooltip({placement: 'right', html:true}).tooltip('show');"></span>}*@
             </h5>
             <div id="@vulnPrefix-details" class="collapse vulnerability-expandable@if(expandVulnerabilities){ in}">
                 @vulnerability("h6", depPrefix+"-"+vuln.name, vuln)

app/views/vulnerability.scala.html

@@ -28,13 +28,13 @@
 </span></p>
 @*@vuln.cweOption.map{cwe =><p><label>Category:</label> <span class="explained" title="Vulnerability category according to Common Weakness Enumeration" onmouseover="$(this).tooltip({placement: 'right'}).tooltip('show')"><b>@cwe</b></span></p>}*@
 <label>Description:</label> @vuln.description
-@section("vuln-sw", "Vulnerable software"){
+@*@section("vuln-sw", "Vulnerable software"){
     <ul id="@idPrefix-details">
     @for(sw <- vuln.vulnerableSoftware){
         <li>@sw.name@if(sw.allPreviousVersion){ and all previous versions}@if(sw.isVersionless){<span class="warning-expandable" title="This identifier does not contain version. It will match <b>any</b> version, which might cause false positives." onmouseover="$(this).tooltip({placement: 'right', html:true}).tooltip('show');"></span>}</li>
     }
     </ul>
-}
+}*@
 @section("references", "References"){
     <ul>
     @for(reference <- vuln.references){

conf/routes

@@ -45,7 +45,7 @@ POST        /notifications/watch              controllers.Notifications.watch(pr
 POST        /notifications/unwatch            controllers.Notifications.unwatch(project: String, filter: Option[String])
 GET         /notifications/cron/:key          controllers.Notifications.cron(key: String, purgeCache: Boolean ?= true)
 
-GET         /libraries/vulnerabilities        controllers.Statistics.searchVulnerableSoftware(versionlessCpes: Seq[String], versionOption: Option[String])
+#GET         /libraries/vulnerabilities        controllers.Statistics.searchVulnerableSoftware(versionlessCpes: Seq[String], versionOption: Option[String])
 
 GET         /vulnerability/:name              controllers.Statistics.vulnerability(name, selector: Option[String])