Fixed issues not exporting when they disappear

app/controllers/Notifications.scala

@@ -25,6 +25,7 @@ class Notifications @Inject()(
   dependencyCheckReportsParser: DependencyCheckReportsParser,
   issueTrackerServiceOption: Option[IssueTrackerService],
   emailExportServiceOption: Option[EmailExportService],
+  odcService: OdcService,
   absolutizer: Absolutizer,
   val env: AuthEnv
 )(implicit val messagesApi: MessagesApi, executionContext: ExecutionContext) extends AuthenticatedController {
@@ -59,23 +60,26 @@ class Notifications @Inject()(
   ) = {
     val vulnerabilitiesByName = lds.vulnerabilitiesToDependencies.map{case (v, deps) => (v.name, (v, deps))}
     for{
-      tickets <- ep.ticketsForVulnerabilities(lds.vulnerabilityNames)
+      tickets <- ep.loadUnfinishedTickets().map(_.map{case rec @ (id, ticket) => ticket.vulnerabilityName->rec}.toMap)
       // Check existing tickets
       existingTicketsIds = tickets.values.map(_._1).toSet
-      ticketsById = tickets.values.map{case (id, ev) => id -> ev}.toMap
+      ticketsById = tickets.values.toMap
       existingTicketsProjects <- ep.projectsForTickets(existingTicketsIds)
       projectUpdates <- Future.traverse(existingTicketsIds){ ticketId =>  // If we traversed over existingTicketsProjects, we would skip vulns with no projects
         val oldProjectIdsSet = existingTicketsProjects(ticketId)
         val exportedVulnerability = ticketsById(ticketId)
         val vulnerabilityName = exportedVulnerability.vulnerabilityName
         val failedOldProjects = oldProjectIdsSet.filter(failedProjects.isFailed)
-        val newKnownProjectIdsSet = vulnerabilitiesByName(vulnerabilityName)._2.flatMap(_.projects).map(_.fullId)
+        val newKnownProjectIdsSet = vulnerabilitiesByName.get(vulnerabilityName).fold(Set[String]())(_._2.flatMap(_.projects).map(_.fullId))
         val allNewProjectIdsSet = newKnownProjectIdsSet ++ failedOldProjects  //If build for a project currently fails and it used to be affected, consider it as still affected. This prevents sudden switching these two states.
         val diff = new SetDiff(oldSet = oldProjectIdsSet, newSet = allNewProjectIdsSet)
         if(diff.nonEmpty) {
-          reportChangedProjectsForVulnerability(lds.vulnerabilitiesByName(vulnerabilityName), diff, exportedVulnerability.ticket).flatMap { _ =>
+          for{
-            ep.changeProjects(ticketId, diff, projects)
+            // Try to load vuln from memory; If the vuln has disappeared, we have to load it from DB.
-          }.map( _ => Some(diff))
+            vulnerability <- lds.vulnerabilitiesByName.get(vulnerabilityName).fold(odcService.getVulnerabilityDetails(vulnerabilityName).map(_.get))(Future(_))
+            (_: Unit) <- reportChangedProjectsForVulnerability(vulnerability, diff, exportedVulnerability.ticket)
+            (_: Unit) <- ep.changeProjects(ticketId, diff, projects)
+          } yield Some(diff)
         } else {
           Fut(None)
         }
@@ -88,7 +92,7 @@ class Notifications @Inject()(
           ep.addTicket(ticket, dependencies.flatMap(_.projects)).map(_ => ticket.ticket)
         }
       }
-    } yield (missingTickets, newTicketIds, projectUpdates.toSet: Set[Any])
+    } yield (missingTickets, newTicketIds, projectUpdates.toSet: Set[Option[Any]])
   }
 
   private def exportFailedReports(lds: LibDepStatistics, failed: FailedProjects): Future[Unit] = {
@@ -138,11 +142,11 @@ class Notifications @Inject()(
     }
   }
 
-  private def forService[S, T](serviceOption: Option[S])(f: S => Future[(Set[String], Set[T], Set[Any])]) = serviceOption.fold(Fut((Set[String](), Set[T](), Set[Any]())))(f)
+  private def forService[S, T](serviceOption: Option[S])(f: S => Future[(Set[String], Set[T], Set[Option[Any]])]) = serviceOption.fold(Fut((Set[String](), Set[T](), Set[Option[Any]]())))(f)
 
   private def exportToEmail(lds: LibDepStatistics, failedProjects: FailedProjects, p: ProjectsWithReports) = forService(emailExportServiceOption){ emailExportService =>
     notifyVulnerabilities[EmailMessageId](lds, failedProjects, notificationService.mailExport, p) { (vulnerability, dependencies) =>
-      emailExportService.mailForVulnerability(vulnerability, dependencies).flatMap(emailExportService.sendEmail).map(id => ExportedVulnerability(vulnerability.name, EmailMessageId(id), 0))
+      emailExportService.mailForVulnerability(vulnerability, dependencies).flatMap(emailExportService.sendEmail).map(id => ExportedVulnerability(vulnerability.name, EmailMessageId(id), 0, done = false))
     }{ (vuln, diff, msgid) =>
       emailExportService.mailForVulnerabilityProjectsChange(vuln, msgid, diff, p).flatMap(emailExportService.sendEmail).map(_ => ())
     }
@@ -166,7 +170,7 @@ class Notifications @Inject()(
       val affectedProjects = dependencies.flatMap(_.projects)
       val diff = new SetDiff(Set(), affectedProjects)
       notificationService.changeAffectedProjects(vulnerability.name, diff.map(_.fullId)).map{_ =>
-        ExportedVulnerability[String](vulnerabilityName = vulnerability.name, ticket = vulnerability.name, ticketFormatVersion = 0)
+        ExportedVulnerability[String](vulnerabilityName = vulnerability.name, ticket = vulnerability.name, ticketFormatVersion = 0, done = false)
       }
     }{ (vulnerability, diff, id) =>
       notificationService.changeAffectedProjects(vulnerability.name, diff)

app/models/ExportPlatformTables.scala

@@ -5,5 +5,5 @@ import models.profile.api._
 abstract class ExportPlatformTables[T, U] private[models] () {
   val tickets: TableQuery[_ <: ExportedVulnerabilities[T, U]]
   val projects: TableQuery[_ <: ExportedVulnerabilityProjects]
-  def schema = tickets.schema ++ projects.schema
+  def schema: models.profile.DDL = tickets.schema ++ projects.schema
 }

app/models/ExportedVulnerability.scala

@@ -4,16 +4,19 @@ import models.profile.api._
 import slick.lifted.{MappedProjection, Tag}
 
 
-case class ExportedVulnerability[T] (vulnerabilityName: String, ticket: T, ticketFormatVersion: Int/*, maintainedAutomatically: Boolean*/)
+case class ExportedVulnerability[T] (vulnerabilityName: String, ticket: T, ticketFormatVersion: Int/*, maintainedAutomatically: Boolean*/, done: Boolean)
 
+//noinspection TypeAnnotation
 abstract class ExportedVulnerabilities[T, U](tag: Tag, tableNamePart: String) extends Table[(Int, ExportedVulnerability[T])](tag, s"exported_${tableNamePart}_vulnerabilities"){
   def id = column[Int]("id", O.PrimaryKey, O.AutoInc)
   def vulnerabilityName = column[String]("vulnerability_name")
   def ticketFormatVersion = column[Int]("ticket_format_version")
+  def done = column[Boolean]("done")
   //def maintainedAutomatically = column[Boolean]("maintained_automatically")
 
   def base: MappedProjection[ExportedVulnerability[T], U]// = (vulnerabilityName, ticket, ticketFormatVersion) <> ((ExportedVulnerability.apply[T] _).tupled, ExportedVulnerability.unapply[T])
   def * = (id, base)
 
   def idx_vulnerabilityName = index(s"idx_${tableName}_vulnerabilityName", vulnerabilityName, unique = true)
+  def idx_done = index(s"idx_${tableName}_done", done, unique = false)
 }

app/models/package.scala

@@ -23,24 +23,24 @@ package object models {
     val changelog = TableQuery[Changes]
     val notificationDigestStatuses = TableQuery[NotificationDigestStatuses]
 
-    val issueTrackerExportTables = new ExportPlatformTables[String, (String, String, Int)](){
+    val issueTrackerExportTables = new ExportPlatformTables[String, (String, String, Int, Boolean)](){
       val tableNamePart = "issue_tracker"
-      class IssueTrackerVulnerabilities(tag: Tag) extends ExportedVulnerabilities[String, (String, String, Int)](tag, tableNamePart){
+      class IssueTrackerVulnerabilities(tag: Tag) extends ExportedVulnerabilities[String, (String, String, Int, Boolean)](tag, tableNamePart){
         def ticket = column[String]("ticket")
-        override def base = (vulnerabilityName, ticket, ticketFormatVersion) <> ((ExportedVulnerability.apply[String] _).tupled, ExportedVulnerability.unapply[String])
+        override def base = (vulnerabilityName, ticket, ticketFormatVersion, done) <> ((ExportedVulnerability.apply[String] _).tupled, ExportedVulnerability.unapply[String])
         def idx_ticket = index("idx_ticket", ticket, unique = true)
       }
       class IssueTrackerVulnerabilityProject(tag: Tag) extends ExportedVulnerabilityProjects(tag, tableNamePart)
       override val tickets = TableQuery[IssueTrackerVulnerabilities]
       override val projects: profile.api.TableQuery[_ <: ExportedVulnerabilityProjects] = TableQuery[IssueTrackerVulnerabilityProject]
     }
-    type EmailExportedVulnerabilitiesShape = (String, EmailMessageId, Int)
+    type EmailExportedVulnerabilitiesShape = (String, EmailMessageId, Int, Boolean)
     val mailExportTables = new ExportPlatformTables[EmailMessageId, EmailExportedVulnerabilitiesShape](){
       val tableNamePart = "email"
       class EmailExportedVulnerabilities(tag: Tag) extends ExportedVulnerabilities[EmailMessageId, EmailExportedVulnerabilitiesShape](tag, tableNamePart){
         private implicit val mmiMapper = MappedJdbcType.base[EmailMessageId, String](_.messageId, EmailMessageId)
         def messageId = column[EmailMessageId]("message_id")  // Unlike ticket, message id is not required to be unique in order to handle some edge cases like play.mailer.mock = true
-        override def base = (vulnerabilityName, messageId, ticketFormatVersion) <> ( (ExportedVulnerability.apply[EmailMessageId] _).tupled, ExportedVulnerability.unapply[EmailMessageId])
+        override def base = (vulnerabilityName, messageId, ticketFormatVersion, done) <> ( (ExportedVulnerability.apply[EmailMessageId] _).tupled, ExportedVulnerability.unapply[EmailMessageId])
       }
       class EmailVulnerabilityProject(tag: Tag) extends ExportedVulnerabilityProjects(tag, tableNamePart)
 
@@ -48,12 +48,12 @@ package object models {
       override val tickets = TableQuery[EmailExportedVulnerabilities]
     }
 
-    val diffDbExportTables = new ExportPlatformTables[String, (String, Int)] {
+    val diffDbExportTables = new ExportPlatformTables[String, (String, Int, Boolean)] {
       val tableNamePart = "diff_db"
-      class DiffDbVulnerabilities(tag: Tag) extends ExportedVulnerabilities[String, (String, Int)](tag, tableNamePart){
+      class DiffDbVulnerabilities(tag: Tag) extends ExportedVulnerabilities[String, (String, Int, Boolean)](tag, tableNamePart){
-        override def base: MappedProjection[ExportedVulnerability[String], (String, Int)] = (vulnerabilityName, ticketFormatVersion) <> (
+        override def base: MappedProjection[ExportedVulnerability[String], (String, Int, Boolean)] = (vulnerabilityName, ticketFormatVersion, done) <> (
-          ((n: String, v: Int) => ExportedVulnerability[String](n, n, v)).tupled,
+          ((n: String, v: Int, d: Boolean) => ExportedVulnerability[String](n, n, v, d)).tupled,
-          obj => ExportedVulnerability.unapply[String](obj).map{case (n, _, v) => (n, v)}
+          obj => ExportedVulnerability.unapply[String](obj).map{case (n, _, v, d) => (n, v, d)}
         )
       }
       class DiffDbVulnerabilityProject(tag: Tag) extends ExportedVulnerabilityProjects(tag, tableNamePart)
@@ -65,7 +65,8 @@ package object models {
     /*{
       import profile.SchemaDescription
       val schema = Seq[Any{def schema: SchemaDescription}](
-        notificationDigestStatuses
+        //notificationDigestStatuses
+        //diffDbExportTables, mailExportTables, issueTrackerExportTables
       ).map(_.schema).foldLeft(profile.DDL(Seq(), Seq()))(_ ++ _)
 
       val sql = Seq(
@@ -76,7 +77,7 @@ package object models {
         schema.dropStatements.toSeq.map(_+";").mkString("\n").dropWhile(_ == "\n"),
         "\n"
       ).mkString("\n")
-      Files.write(Paths.get("conf/evolutions/default/8.sql"), sql.getBytes("utf-8"))
+      Files.write(Paths.get("conf/evolutions/default/10.sql"), sql.getBytes("utf-8"))
     }*/
 
   }

app/services/JiraIssueTrackerService.scala

@@ -53,7 +53,7 @@ class JiraIssueTrackerService @Inject()(absolutizer: Absolutizer, @Named("jira-s
   )).map(response => // returns responses like {"id":"1234","key":"PROJ-6","self":"https://…/rest/api/2/issue/1234"}
     try{
       val issueInfo = Json.reads[JiraNewIssueResponse].reads(response.json).get
-      ExportedVulnerability(vulnerabilityName = vulnerability.name, ticket = issueInfo.key, ticketFormatVersion = ticketFormatVersion)
+      ExportedVulnerability(vulnerabilityName = vulnerability.name, ticket = issueInfo.key, ticketFormatVersion = ticketFormatVersion, done = false)
     }catch{
       case e:Throwable=>sys.error("bad data: "+response.body)
     }

app/services/VulnerabilityNotificationService.scala

@@ -63,8 +63,14 @@ class VulnerabilityNotificationService @Inject() (protected val dbConfigProvider
   }
 
   class ExportPlatform[T, U] private[VulnerabilityNotificationService] (ept: ExportPlatformTables[T, U]) {
-    def changeProjects(ticketId: Int, diff: SetDiff[String], projects: ProjectsWithReports) = db.run(
+
+    def loadUnfinishedTickets(): Future[Seq[(Int, ExportedVulnerability[T])]] = db.run(
+      ept.tickets.filter(_.done === false).result
+    )
+
+    def changeProjects(ticketId: Int, diff: SetDiff[String], projects: ProjectsWithReports): Future[Unit] = db.run(
       DBIO.seq(
+        ept.tickets.filter(_.id === ticketId).map(_.done).update(diff.newSet.isEmpty),
         ept.projects.filter(_.exportedVulnerabilityId === ticketId).delete,
         ept.projects ++= diff.newSet.map(fullId => ExportedVulnerabilityProject(ticketId, fullId)).toSet
       ).transactionally
@@ -74,7 +80,7 @@ class VulnerabilityNotificationService @Inject() (protected val dbConfigProvider
       ept.projects.filter(_.exportedVulnerabilityId inSet ticketsIds).result
     ).map{_.groupBy(_.exportedVulnerabilityId).mapValues(_.map(_.projectFullId).toSet).map(identity).withDefaultValue(Set())}
 
-    def ticketsForVulnerabilities(vulnerabilities: Traversable[String]) = db.run(
+    def ticketsForVulnerabilities(vulnerabilities: Traversable[String]): Future[Map[String, (Int, ExportedVulnerability[T])]] = db.run(
       ept.tickets.filter(_.vulnerabilityName inSet vulnerabilities).result
     ).map(_.map{ rec =>
         rec._2.vulnerabilityName -> rec

conf/evolutions/default/9.sql

@@ -0,0 +1,17 @@
+# --- !Ups
+ALTER TABLE "exported_diff_db_vulnerabilities" ADD "done" BOOLEAN NOT NULL DEFAULT FALSE;
+ALTER TABLE "exported_email_vulnerabilities" ADD "done" BOOLEAN NOT NULL DEFAULT FALSE;
+ALTER TABLE "exported_issue_tracker_vulnerabilities" ADD "done" BOOLEAN NOT NULL DEFAULT FALSE;
+
+CREATE INDEX "idx_exported_diff_db_vulnerabilities_done" ON "exported_diff_db_vulnerabilities" ("done");
+CREATE INDEX "idx_exported_email_vulnerabilities_done" ON "exported_email_vulnerabilities" ("done");
+CREATE INDEX "idx_exported_issue_tracker_vulnerabilities_done" ON "exported_issue_tracker_vulnerabilities" ("done");
+
+# --- !Downs
+DROP INDEX "idx_exported_diff_db_vulnerabilities_done";
+DROP INDEX "idx_exported_email_vulnerabilities_done";
+DROP INDEX "idx_exported_issue_tracker_vulnerabilities_done";
+
+ALTER TABLE "exported_diff_db_vulnerabilities" DROP COLUMN "done";
+ALTER TABLE "exported_email_vulnerabilities" DROP COLUMN "done";
+ALTER TABLE "exported_issue_tracker_vulnerabilities" DROP COLUMN "done";