Fixed issues not exporting when they disappear

2026-01-15 16:23:52 +01:00 · 2016-12-13 23:37:18 +01:00
parent 2d3e1cea0a
commit 5680eec2d5
7 changed files with 58 additions and 27 deletions
--- a/app/controllers/Notifications.scala
+++ b/app/controllers/Notifications.scala
@@ -25,6 +25,7 @@ class Notifications @Inject()(
  dependencyCheckReportsParser: DependencyCheckReportsParser,
  issueTrackerServiceOption: Option[IssueTrackerService],
  emailExportServiceOption: Option[EmailExportService],
+  odcService: OdcService,
  absolutizer: Absolutizer,
  val env: AuthEnv
 )(implicit val messagesApi: MessagesApi, executionContext: ExecutionContext) extends AuthenticatedController {
@@ -59,23 +60,26 @@ class Notifications @Inject()(
  ) = {
    val vulnerabilitiesByName = lds.vulnerabilitiesToDependencies.map{case (v, deps) => (v.name, (v, deps))}
    for{
-      tickets <- ep.ticketsForVulnerabilities(lds.vulnerabilityNames)
+      tickets <- ep.loadUnfinishedTickets().map(_.map{case rec @ (id, ticket) => ticket.vulnerabilityName->rec}.toMap)
      // Check existing tickets
      existingTicketsIds = tickets.values.map(_._1).toSet
-      ticketsById = tickets.values.map{case (id, ev) => id -> ev}.toMap
+      ticketsById = tickets.values.toMap
      existingTicketsProjects <- ep.projectsForTickets(existingTicketsIds)
      projectUpdates <- Future.traverse(existingTicketsIds){ ticketId =>  // If we traversed over existingTicketsProjects, we would skip vulns with no projects
        val oldProjectIdsSet = existingTicketsProjects(ticketId)
        val exportedVulnerability = ticketsById(ticketId)
        val vulnerabilityName = exportedVulnerability.vulnerabilityName
        val failedOldProjects = oldProjectIdsSet.filter(failedProjects.isFailed)
-        val newKnownProjectIdsSet = vulnerabilitiesByName(vulnerabilityName)._2.flatMap(_.projects).map(_.fullId)
+        val newKnownProjectIdsSet = vulnerabilitiesByName.get(vulnerabilityName).fold(Set[String]())(_._2.flatMap(_.projects).map(_.fullId))
        val allNewProjectIdsSet = newKnownProjectIdsSet ++ failedOldProjects  //If build for a project currently fails and it used to be affected, consider it as still affected. This prevents sudden switching these two states.
        val diff = new SetDiff(oldSet = oldProjectIdsSet, newSet = allNewProjectIdsSet)
        if(diff.nonEmpty) {
-          reportChangedProjectsForVulnerability(lds.vulnerabilitiesByName(vulnerabilityName), diff, exportedVulnerability.ticket).flatMap { _ =>
-            ep.changeProjects(ticketId, diff, projects)
-          }.map( _ => Some(diff))
+          for{
+            // Try to load vuln from memory; If the vuln has disappeared, we have to load it from DB.
+            vulnerability <- lds.vulnerabilitiesByName.get(vulnerabilityName).fold(odcService.getVulnerabilityDetails(vulnerabilityName).map(_.get))(Future(_))
+            (_: Unit) <- reportChangedProjectsForVulnerability(vulnerability, diff, exportedVulnerability.ticket)
+            (_: Unit) <- ep.changeProjects(ticketId, diff, projects)
+          } yield Some(diff)
        } else {
          Fut(None)
        }
@@ -88,7 +92,7 @@ class Notifications @Inject()(
          ep.addTicket(ticket, dependencies.flatMap(_.projects)).map(_ => ticket.ticket)
        }
      }
-    } yield (missingTickets, newTicketIds, projectUpdates.toSet: Set[Any])
+    } yield (missingTickets, newTicketIds, projectUpdates.toSet: Set[Option[Any]])
  }

  private def exportFailedReports(lds: LibDepStatistics, failed: FailedProjects): Future[Unit] = {
@@ -138,11 +142,11 @@ class Notifications @Inject()(
    }
  }

-  private def forService[S, T](serviceOption: Option[S])(f: S => Future[(Set[String], Set[T], Set[Any])]) = serviceOption.fold(Fut((Set[String](), Set[T](), Set[Any]())))(f)
+  private def forService[S, T](serviceOption: Option[S])(f: S => Future[(Set[String], Set[T], Set[Option[Any]])]) = serviceOption.fold(Fut((Set[String](), Set[T](), Set[Option[Any]]())))(f)

  private def exportToEmail(lds: LibDepStatistics, failedProjects: FailedProjects, p: ProjectsWithReports) = forService(emailExportServiceOption){ emailExportService =>
    notifyVulnerabilities[EmailMessageId](lds, failedProjects, notificationService.mailExport, p) { (vulnerability, dependencies) =>
-      emailExportService.mailForVulnerability(vulnerability, dependencies).flatMap(emailExportService.sendEmail).map(id => ExportedVulnerability(vulnerability.name, EmailMessageId(id), 0))
+      emailExportService.mailForVulnerability(vulnerability, dependencies).flatMap(emailExportService.sendEmail).map(id => ExportedVulnerability(vulnerability.name, EmailMessageId(id), 0, done = false))
    }{ (vuln, diff, msgid) =>
      emailExportService.mailForVulnerabilityProjectsChange(vuln, msgid, diff, p).flatMap(emailExportService.sendEmail).map(_ => ())
    }
@@ -166,7 +170,7 @@ class Notifications @Inject()(
      val affectedProjects = dependencies.flatMap(_.projects)
      val diff = new SetDiff(Set(), affectedProjects)
      notificationService.changeAffectedProjects(vulnerability.name, diff.map(_.fullId)).map{_ =>
-        ExportedVulnerability[String](vulnerabilityName = vulnerability.name, ticket = vulnerability.name, ticketFormatVersion = 0)
+        ExportedVulnerability[String](vulnerabilityName = vulnerability.name, ticket = vulnerability.name, ticketFormatVersion = 0, done = false)
      }
    }{ (vulnerability, diff, id) =>
      notificationService.changeAffectedProjects(vulnerability.name, diff)
--- a/app/models/ExportPlatformTables.scala
+++ b/app/models/ExportPlatformTables.scala
@@ -5,5 +5,5 @@ import models.profile.api._
 abstract class ExportPlatformTables[T, U] private[models] () {
  val tickets: TableQuery[_ <: ExportedVulnerabilities[T, U]]
  val projects: TableQuery[_ <: ExportedVulnerabilityProjects]
-  def schema = tickets.schema ++ projects.schema
+  def schema: models.profile.DDL = tickets.schema ++ projects.schema
 }
--- a/app/models/ExportedVulnerability.scala
+++ b/app/models/ExportedVulnerability.scala
@@ -4,16 +4,19 @@ import models.profile.api._
 import slick.lifted.{MappedProjection, Tag}


-case class ExportedVulnerability[T] (vulnerabilityName: String, ticket: T, ticketFormatVersion: Int/*, maintainedAutomatically: Boolean*/)
+case class ExportedVulnerability[T] (vulnerabilityName: String, ticket: T, ticketFormatVersion: Int/*, maintainedAutomatically: Boolean*/, done: Boolean)

+//noinspection TypeAnnotation
 abstract class ExportedVulnerabilities[T, U](tag: Tag, tableNamePart: String) extends Table[(Int, ExportedVulnerability[T])](tag, s"exported_${tableNamePart}_vulnerabilities"){
  def id = column[Int]("id", O.PrimaryKey, O.AutoInc)
  def vulnerabilityName = column[String]("vulnerability_name")
  def ticketFormatVersion = column[Int]("ticket_format_version")
+  def done = column[Boolean]("done")
  //def maintainedAutomatically = column[Boolean]("maintained_automatically")

  def base: MappedProjection[ExportedVulnerability[T], U]// = (vulnerabilityName, ticket, ticketFormatVersion) <> ((ExportedVulnerability.apply[T] _).tupled, ExportedVulnerability.unapply[T])
  def * = (id, base)

  def idx_vulnerabilityName = index(s"idx_${tableName}_vulnerabilityName", vulnerabilityName, unique = true)
+  def idx_done = index(s"idx_${tableName}_done", done, unique = false)
 }
--- a/app/models/package.scala
+++ b/app/models/package.scala
@@ -23,24 +23,24 @@ package object models {
    val changelog = TableQuery[Changes]
    val notificationDigestStatuses = TableQuery[NotificationDigestStatuses]

-    val issueTrackerExportTables = new ExportPlatformTables[String, (String, String, Int)](){
+    val issueTrackerExportTables = new ExportPlatformTables[String, (String, String, Int, Boolean)](){
      val tableNamePart = "issue_tracker"
-      class IssueTrackerVulnerabilities(tag: Tag) extends ExportedVulnerabilities[String, (String, String, Int)](tag, tableNamePart){
+      class IssueTrackerVulnerabilities(tag: Tag) extends ExportedVulnerabilities[String, (String, String, Int, Boolean)](tag, tableNamePart){
        def ticket = column[String]("ticket")
-        override def base = (vulnerabilityName, ticket, ticketFormatVersion) <> ((ExportedVulnerability.apply[String] _).tupled, ExportedVulnerability.unapply[String])
+        override def base = (vulnerabilityName, ticket, ticketFormatVersion, done) <> ((ExportedVulnerability.apply[String] _).tupled, ExportedVulnerability.unapply[String])
        def idx_ticket = index("idx_ticket", ticket, unique = true)
      }
      class IssueTrackerVulnerabilityProject(tag: Tag) extends ExportedVulnerabilityProjects(tag, tableNamePart)
      override val tickets = TableQuery[IssueTrackerVulnerabilities]
      override val projects: profile.api.TableQuery[_ <: ExportedVulnerabilityProjects] = TableQuery[IssueTrackerVulnerabilityProject]
    }
-    type EmailExportedVulnerabilitiesShape = (String, EmailMessageId, Int)
+    type EmailExportedVulnerabilitiesShape = (String, EmailMessageId, Int, Boolean)
    val mailExportTables = new ExportPlatformTables[EmailMessageId, EmailExportedVulnerabilitiesShape](){
      val tableNamePart = "email"
      class EmailExportedVulnerabilities(tag: Tag) extends ExportedVulnerabilities[EmailMessageId, EmailExportedVulnerabilitiesShape](tag, tableNamePart){
        private implicit val mmiMapper = MappedJdbcType.base[EmailMessageId, String](_.messageId, EmailMessageId)
        def messageId = column[EmailMessageId]("message_id")  // Unlike ticket, message id is not required to be unique in order to handle some edge cases like play.mailer.mock = true
-        override def base = (vulnerabilityName, messageId, ticketFormatVersion) <> ( (ExportedVulnerability.apply[EmailMessageId] _).tupled, ExportedVulnerability.unapply[EmailMessageId])
+        override def base = (vulnerabilityName, messageId, ticketFormatVersion, done) <> ( (ExportedVulnerability.apply[EmailMessageId] _).tupled, ExportedVulnerability.unapply[EmailMessageId])
      }
      class EmailVulnerabilityProject(tag: Tag) extends ExportedVulnerabilityProjects(tag, tableNamePart)

@@ -48,12 +48,12 @@ package object models {
      override val tickets = TableQuery[EmailExportedVulnerabilities]
    }

-    val diffDbExportTables = new ExportPlatformTables[String, (String, Int)] {
+    val diffDbExportTables = new ExportPlatformTables[String, (String, Int, Boolean)] {
      val tableNamePart = "diff_db"
-      class DiffDbVulnerabilities(tag: Tag) extends ExportedVulnerabilities[String, (String, Int)](tag, tableNamePart){
-        override def base: MappedProjection[ExportedVulnerability[String], (String, Int)] = (vulnerabilityName, ticketFormatVersion) <> (
-          ((n: String, v: Int) => ExportedVulnerability[String](n, n, v)).tupled,
-          obj => ExportedVulnerability.unapply[String](obj).map{case (n, _, v) => (n, v)}
+      class DiffDbVulnerabilities(tag: Tag) extends ExportedVulnerabilities[String, (String, Int, Boolean)](tag, tableNamePart){
+        override def base: MappedProjection[ExportedVulnerability[String], (String, Int, Boolean)] = (vulnerabilityName, ticketFormatVersion, done) <> (
+          ((n: String, v: Int, d: Boolean) => ExportedVulnerability[String](n, n, v, d)).tupled,
+          obj => ExportedVulnerability.unapply[String](obj).map{case (n, _, v, d) => (n, v, d)}
        )
      }
      class DiffDbVulnerabilityProject(tag: Tag) extends ExportedVulnerabilityProjects(tag, tableNamePart)
@@ -65,7 +65,8 @@ package object models {
    /*{
      import profile.SchemaDescription
      val schema = Seq[Any{def schema: SchemaDescription}](
-        notificationDigestStatuses
+        //notificationDigestStatuses
+        //diffDbExportTables, mailExportTables, issueTrackerExportTables
      ).map(_.schema).foldLeft(profile.DDL(Seq(), Seq()))(_ ++ _)

      val sql = Seq(
@@ -76,7 +77,7 @@ package object models {
        schema.dropStatements.toSeq.map(_+";").mkString("\n").dropWhile(_ == "\n"),
        "\n"
      ).mkString("\n")
-      Files.write(Paths.get("conf/evolutions/default/8.sql"), sql.getBytes("utf-8"))
+      Files.write(Paths.get("conf/evolutions/default/10.sql"), sql.getBytes("utf-8"))
    }*/

  }
--- a/app/services/JiraIssueTrackerService.scala
+++ b/app/services/JiraIssueTrackerService.scala
@@ -53,7 +53,7 @@ class JiraIssueTrackerService @Inject()(absolutizer: Absolutizer, @Named("jira-s
  )).map(response => // returns responses like {"id":"1234","key":"PROJ-6","self":"https://…/rest/api/2/issue/1234"}
    try{
      val issueInfo = Json.reads[JiraNewIssueResponse].reads(response.json).get
-      ExportedVulnerability(vulnerabilityName = vulnerability.name, ticket = issueInfo.key, ticketFormatVersion = ticketFormatVersion)
+      ExportedVulnerability(vulnerabilityName = vulnerability.name, ticket = issueInfo.key, ticketFormatVersion = ticketFormatVersion, done = false)
    }catch{
      case e:Throwable=>sys.error("bad data: "+response.body)
    }
--- a/app/services/VulnerabilityNotificationService.scala
+++ b/app/services/VulnerabilityNotificationService.scala
@@ -63,8 +63,14 @@ class VulnerabilityNotificationService @Inject() (protected val dbConfigProvider
  }

  class ExportPlatform[T, U] private[VulnerabilityNotificationService] (ept: ExportPlatformTables[T, U]) {
-    def changeProjects(ticketId: Int, diff: SetDiff[String], projects: ProjectsWithReports) = db.run(
+
+    def loadUnfinishedTickets(): Future[Seq[(Int, ExportedVulnerability[T])]] = db.run(
+      ept.tickets.filter(_.done === false).result
+    )
+
+    def changeProjects(ticketId: Int, diff: SetDiff[String], projects: ProjectsWithReports): Future[Unit] = db.run(
      DBIO.seq(
+        ept.tickets.filter(_.id === ticketId).map(_.done).update(diff.newSet.isEmpty),
        ept.projects.filter(_.exportedVulnerabilityId === ticketId).delete,
        ept.projects ++= diff.newSet.map(fullId => ExportedVulnerabilityProject(ticketId, fullId)).toSet
      ).transactionally
@@ -74,7 +80,7 @@ class VulnerabilityNotificationService @Inject() (protected val dbConfigProvider
      ept.projects.filter(_.exportedVulnerabilityId inSet ticketsIds).result
    ).map{_.groupBy(_.exportedVulnerabilityId).mapValues(_.map(_.projectFullId).toSet).map(identity).withDefaultValue(Set())}

-    def ticketsForVulnerabilities(vulnerabilities: Traversable[String]) = db.run(
+    def ticketsForVulnerabilities(vulnerabilities: Traversable[String]): Future[Map[String, (Int, ExportedVulnerability[T])]] = db.run(
      ept.tickets.filter(_.vulnerabilityName inSet vulnerabilities).result
    ).map(_.map{ rec =>
        rec._2.vulnerabilityName -> rec
--- a/conf/evolutions/default/9.sql
+++ b/conf/evolutions/default/9.sql
@@ -0,0 +1,17 @@
+# --- !Ups
+ALTER TABLE "exported_diff_db_vulnerabilities" ADD "done" BOOLEAN NOT NULL DEFAULT FALSE;
+ALTER TABLE "exported_email_vulnerabilities" ADD "done" BOOLEAN NOT NULL DEFAULT FALSE;
+ALTER TABLE "exported_issue_tracker_vulnerabilities" ADD "done" BOOLEAN NOT NULL DEFAULT FALSE;
+
+CREATE INDEX "idx_exported_diff_db_vulnerabilities_done" ON "exported_diff_db_vulnerabilities" ("done");
+CREATE INDEX "idx_exported_email_vulnerabilities_done" ON "exported_email_vulnerabilities" ("done");
+CREATE INDEX "idx_exported_issue_tracker_vulnerabilities_done" ON "exported_issue_tracker_vulnerabilities" ("done");
+
+# --- !Downs
+DROP INDEX "idx_exported_diff_db_vulnerabilities_done";
+DROP INDEX "idx_exported_email_vulnerabilities_done";
+DROP INDEX "idx_exported_issue_tracker_vulnerabilities_done";
+
+ALTER TABLE "exported_diff_db_vulnerabilities" DROP COLUMN "done";
+ALTER TABLE "exported_email_vulnerabilities" DROP COLUMN "done";
+ALTER TABLE "exported_issue_tracker_vulnerabilities" DROP COLUMN "done";