diff --git a/app/controllers/Notifications.scala b/app/controllers/Notifications.scala index 27c6d81..950e5c8 100644 --- a/app/controllers/Notifications.scala +++ b/app/controllers/Notifications.scala @@ -25,6 +25,7 @@ class Notifications @Inject()( dependencyCheckReportsParser: DependencyCheckReportsParser, issueTrackerServiceOption: Option[IssueTrackerService], emailExportServiceOption: Option[EmailExportService], + odcService: OdcService, absolutizer: Absolutizer, val env: AuthEnv )(implicit val messagesApi: MessagesApi, executionContext: ExecutionContext) extends AuthenticatedController { @@ -59,23 +60,26 @@ class Notifications @Inject()( ) = { val vulnerabilitiesByName = lds.vulnerabilitiesToDependencies.map{case (v, deps) => (v.name, (v, deps))} for{ - tickets <- ep.ticketsForVulnerabilities(lds.vulnerabilityNames) + tickets <- ep.loadUnfinishedTickets().map(_.map{case rec @ (id, ticket) => ticket.vulnerabilityName->rec}.toMap) // Check existing tickets existingTicketsIds = tickets.values.map(_._1).toSet - ticketsById = tickets.values.map{case (id, ev) => id -> ev}.toMap + ticketsById = tickets.values.toMap existingTicketsProjects <- ep.projectsForTickets(existingTicketsIds) projectUpdates <- Future.traverse(existingTicketsIds){ ticketId => // If we traversed over existingTicketsProjects, we would skip vulns with no projects val oldProjectIdsSet = existingTicketsProjects(ticketId) val exportedVulnerability = ticketsById(ticketId) val vulnerabilityName = exportedVulnerability.vulnerabilityName val failedOldProjects = oldProjectIdsSet.filter(failedProjects.isFailed) - val newKnownProjectIdsSet = vulnerabilitiesByName(vulnerabilityName)._2.flatMap(_.projects).map(_.fullId) + val newKnownProjectIdsSet = vulnerabilitiesByName.get(vulnerabilityName).fold(Set[String]())(_._2.flatMap(_.projects).map(_.fullId)) val allNewProjectIdsSet = newKnownProjectIdsSet ++ failedOldProjects //If build for a project currently fails and it used to be affected, consider it as still affected. This prevents sudden switching these two states. val diff = new SetDiff(oldSet = oldProjectIdsSet, newSet = allNewProjectIdsSet) if(diff.nonEmpty) { - reportChangedProjectsForVulnerability(lds.vulnerabilitiesByName(vulnerabilityName), diff, exportedVulnerability.ticket).flatMap { _ => - ep.changeProjects(ticketId, diff, projects) - }.map( _ => Some(diff)) + for{ + // Try to load vuln from memory; If the vuln has disappeared, we have to load it from DB. + vulnerability <- lds.vulnerabilitiesByName.get(vulnerabilityName).fold(odcService.getVulnerabilityDetails(vulnerabilityName).map(_.get))(Future(_)) + (_: Unit) <- reportChangedProjectsForVulnerability(vulnerability, diff, exportedVulnerability.ticket) + (_: Unit) <- ep.changeProjects(ticketId, diff, projects) + } yield Some(diff) } else { Fut(None) } @@ -88,7 +92,7 @@ class Notifications @Inject()( ep.addTicket(ticket, dependencies.flatMap(_.projects)).map(_ => ticket.ticket) } } - } yield (missingTickets, newTicketIds, projectUpdates.toSet: Set[Any]) + } yield (missingTickets, newTicketIds, projectUpdates.toSet: Set[Option[Any]]) } private def exportFailedReports(lds: LibDepStatistics, failed: FailedProjects): Future[Unit] = { @@ -138,11 +142,11 @@ class Notifications @Inject()( } } - private def forService[S, T](serviceOption: Option[S])(f: S => Future[(Set[String], Set[T], Set[Any])]) = serviceOption.fold(Fut((Set[String](), Set[T](), Set[Any]())))(f) + private def forService[S, T](serviceOption: Option[S])(f: S => Future[(Set[String], Set[T], Set[Option[Any]])]) = serviceOption.fold(Fut((Set[String](), Set[T](), Set[Option[Any]]())))(f) private def exportToEmail(lds: LibDepStatistics, failedProjects: FailedProjects, p: ProjectsWithReports) = forService(emailExportServiceOption){ emailExportService => notifyVulnerabilities[EmailMessageId](lds, failedProjects, notificationService.mailExport, p) { (vulnerability, dependencies) => - emailExportService.mailForVulnerability(vulnerability, dependencies).flatMap(emailExportService.sendEmail).map(id => ExportedVulnerability(vulnerability.name, EmailMessageId(id), 0)) + emailExportService.mailForVulnerability(vulnerability, dependencies).flatMap(emailExportService.sendEmail).map(id => ExportedVulnerability(vulnerability.name, EmailMessageId(id), 0, done = false)) }{ (vuln, diff, msgid) => emailExportService.mailForVulnerabilityProjectsChange(vuln, msgid, diff, p).flatMap(emailExportService.sendEmail).map(_ => ()) } @@ -166,7 +170,7 @@ class Notifications @Inject()( val affectedProjects = dependencies.flatMap(_.projects) val diff = new SetDiff(Set(), affectedProjects) notificationService.changeAffectedProjects(vulnerability.name, diff.map(_.fullId)).map{_ => - ExportedVulnerability[String](vulnerabilityName = vulnerability.name, ticket = vulnerability.name, ticketFormatVersion = 0) + ExportedVulnerability[String](vulnerabilityName = vulnerability.name, ticket = vulnerability.name, ticketFormatVersion = 0, done = false) } }{ (vulnerability, diff, id) => notificationService.changeAffectedProjects(vulnerability.name, diff) diff --git a/app/models/ExportPlatformTables.scala b/app/models/ExportPlatformTables.scala index 342d719..13e75ec 100644 --- a/app/models/ExportPlatformTables.scala +++ b/app/models/ExportPlatformTables.scala @@ -5,5 +5,5 @@ import models.profile.api._ abstract class ExportPlatformTables[T, U] private[models] () { val tickets: TableQuery[_ <: ExportedVulnerabilities[T, U]] val projects: TableQuery[_ <: ExportedVulnerabilityProjects] - def schema = tickets.schema ++ projects.schema + def schema: models.profile.DDL = tickets.schema ++ projects.schema } diff --git a/app/models/ExportedVulnerability.scala b/app/models/ExportedVulnerability.scala index 317c924..95f09a5 100644 --- a/app/models/ExportedVulnerability.scala +++ b/app/models/ExportedVulnerability.scala @@ -4,16 +4,19 @@ import models.profile.api._ import slick.lifted.{MappedProjection, Tag} -case class ExportedVulnerability[T] (vulnerabilityName: String, ticket: T, ticketFormatVersion: Int/*, maintainedAutomatically: Boolean*/) +case class ExportedVulnerability[T] (vulnerabilityName: String, ticket: T, ticketFormatVersion: Int/*, maintainedAutomatically: Boolean*/, done: Boolean) +//noinspection TypeAnnotation abstract class ExportedVulnerabilities[T, U](tag: Tag, tableNamePart: String) extends Table[(Int, ExportedVulnerability[T])](tag, s"exported_${tableNamePart}_vulnerabilities"){ def id = column[Int]("id", O.PrimaryKey, O.AutoInc) def vulnerabilityName = column[String]("vulnerability_name") def ticketFormatVersion = column[Int]("ticket_format_version") + def done = column[Boolean]("done") //def maintainedAutomatically = column[Boolean]("maintained_automatically") def base: MappedProjection[ExportedVulnerability[T], U]// = (vulnerabilityName, ticket, ticketFormatVersion) <> ((ExportedVulnerability.apply[T] _).tupled, ExportedVulnerability.unapply[T]) def * = (id, base) def idx_vulnerabilityName = index(s"idx_${tableName}_vulnerabilityName", vulnerabilityName, unique = true) + def idx_done = index(s"idx_${tableName}_done", done, unique = false) } \ No newline at end of file diff --git a/app/models/package.scala b/app/models/package.scala index dc659d2..f50fe48 100644 --- a/app/models/package.scala +++ b/app/models/package.scala @@ -23,24 +23,24 @@ package object models { val changelog = TableQuery[Changes] val notificationDigestStatuses = TableQuery[NotificationDigestStatuses] - val issueTrackerExportTables = new ExportPlatformTables[String, (String, String, Int)](){ + val issueTrackerExportTables = new ExportPlatformTables[String, (String, String, Int, Boolean)](){ val tableNamePart = "issue_tracker" - class IssueTrackerVulnerabilities(tag: Tag) extends ExportedVulnerabilities[String, (String, String, Int)](tag, tableNamePart){ + class IssueTrackerVulnerabilities(tag: Tag) extends ExportedVulnerabilities[String, (String, String, Int, Boolean)](tag, tableNamePart){ def ticket = column[String]("ticket") - override def base = (vulnerabilityName, ticket, ticketFormatVersion) <> ((ExportedVulnerability.apply[String] _).tupled, ExportedVulnerability.unapply[String]) + override def base = (vulnerabilityName, ticket, ticketFormatVersion, done) <> ((ExportedVulnerability.apply[String] _).tupled, ExportedVulnerability.unapply[String]) def idx_ticket = index("idx_ticket", ticket, unique = true) } class IssueTrackerVulnerabilityProject(tag: Tag) extends ExportedVulnerabilityProjects(tag, tableNamePart) override val tickets = TableQuery[IssueTrackerVulnerabilities] override val projects: profile.api.TableQuery[_ <: ExportedVulnerabilityProjects] = TableQuery[IssueTrackerVulnerabilityProject] } - type EmailExportedVulnerabilitiesShape = (String, EmailMessageId, Int) + type EmailExportedVulnerabilitiesShape = (String, EmailMessageId, Int, Boolean) val mailExportTables = new ExportPlatformTables[EmailMessageId, EmailExportedVulnerabilitiesShape](){ val tableNamePart = "email" class EmailExportedVulnerabilities(tag: Tag) extends ExportedVulnerabilities[EmailMessageId, EmailExportedVulnerabilitiesShape](tag, tableNamePart){ private implicit val mmiMapper = MappedJdbcType.base[EmailMessageId, String](_.messageId, EmailMessageId) def messageId = column[EmailMessageId]("message_id") // Unlike ticket, message id is not required to be unique in order to handle some edge cases like play.mailer.mock = true - override def base = (vulnerabilityName, messageId, ticketFormatVersion) <> ( (ExportedVulnerability.apply[EmailMessageId] _).tupled, ExportedVulnerability.unapply[EmailMessageId]) + override def base = (vulnerabilityName, messageId, ticketFormatVersion, done) <> ( (ExportedVulnerability.apply[EmailMessageId] _).tupled, ExportedVulnerability.unapply[EmailMessageId]) } class EmailVulnerabilityProject(tag: Tag) extends ExportedVulnerabilityProjects(tag, tableNamePart) @@ -48,12 +48,12 @@ package object models { override val tickets = TableQuery[EmailExportedVulnerabilities] } - val diffDbExportTables = new ExportPlatformTables[String, (String, Int)] { + val diffDbExportTables = new ExportPlatformTables[String, (String, Int, Boolean)] { val tableNamePart = "diff_db" - class DiffDbVulnerabilities(tag: Tag) extends ExportedVulnerabilities[String, (String, Int)](tag, tableNamePart){ - override def base: MappedProjection[ExportedVulnerability[String], (String, Int)] = (vulnerabilityName, ticketFormatVersion) <> ( - ((n: String, v: Int) => ExportedVulnerability[String](n, n, v)).tupled, - obj => ExportedVulnerability.unapply[String](obj).map{case (n, _, v) => (n, v)} + class DiffDbVulnerabilities(tag: Tag) extends ExportedVulnerabilities[String, (String, Int, Boolean)](tag, tableNamePart){ + override def base: MappedProjection[ExportedVulnerability[String], (String, Int, Boolean)] = (vulnerabilityName, ticketFormatVersion, done) <> ( + ((n: String, v: Int, d: Boolean) => ExportedVulnerability[String](n, n, v, d)).tupled, + obj => ExportedVulnerability.unapply[String](obj).map{case (n, _, v, d) => (n, v, d)} ) } class DiffDbVulnerabilityProject(tag: Tag) extends ExportedVulnerabilityProjects(tag, tableNamePart) @@ -65,7 +65,8 @@ package object models { /*{ import profile.SchemaDescription val schema = Seq[Any{def schema: SchemaDescription}]( - notificationDigestStatuses + //notificationDigestStatuses + //diffDbExportTables, mailExportTables, issueTrackerExportTables ).map(_.schema).foldLeft(profile.DDL(Seq(), Seq()))(_ ++ _) val sql = Seq( @@ -76,7 +77,7 @@ package object models { schema.dropStatements.toSeq.map(_+";").mkString("\n").dropWhile(_ == "\n"), "\n" ).mkString("\n") - Files.write(Paths.get("conf/evolutions/default/8.sql"), sql.getBytes("utf-8")) + Files.write(Paths.get("conf/evolutions/default/10.sql"), sql.getBytes("utf-8")) }*/ } diff --git a/app/services/JiraIssueTrackerService.scala b/app/services/JiraIssueTrackerService.scala index b1717d6..25c8afa 100644 --- a/app/services/JiraIssueTrackerService.scala +++ b/app/services/JiraIssueTrackerService.scala @@ -53,7 +53,7 @@ class JiraIssueTrackerService @Inject()(absolutizer: Absolutizer, @Named("jira-s )).map(response => // returns responses like {"id":"1234","key":"PROJ-6","self":"https://…/rest/api/2/issue/1234"} try{ val issueInfo = Json.reads[JiraNewIssueResponse].reads(response.json).get - ExportedVulnerability(vulnerabilityName = vulnerability.name, ticket = issueInfo.key, ticketFormatVersion = ticketFormatVersion) + ExportedVulnerability(vulnerabilityName = vulnerability.name, ticket = issueInfo.key, ticketFormatVersion = ticketFormatVersion, done = false) }catch{ case e:Throwable=>sys.error("bad data: "+response.body) } diff --git a/app/services/VulnerabilityNotificationService.scala b/app/services/VulnerabilityNotificationService.scala index e23eb55..c12414c 100644 --- a/app/services/VulnerabilityNotificationService.scala +++ b/app/services/VulnerabilityNotificationService.scala @@ -63,8 +63,14 @@ class VulnerabilityNotificationService @Inject() (protected val dbConfigProvider } class ExportPlatform[T, U] private[VulnerabilityNotificationService] (ept: ExportPlatformTables[T, U]) { - def changeProjects(ticketId: Int, diff: SetDiff[String], projects: ProjectsWithReports) = db.run( + + def loadUnfinishedTickets(): Future[Seq[(Int, ExportedVulnerability[T])]] = db.run( + ept.tickets.filter(_.done === false).result + ) + + def changeProjects(ticketId: Int, diff: SetDiff[String], projects: ProjectsWithReports): Future[Unit] = db.run( DBIO.seq( + ept.tickets.filter(_.id === ticketId).map(_.done).update(diff.newSet.isEmpty), ept.projects.filter(_.exportedVulnerabilityId === ticketId).delete, ept.projects ++= diff.newSet.map(fullId => ExportedVulnerabilityProject(ticketId, fullId)).toSet ).transactionally @@ -74,7 +80,7 @@ class VulnerabilityNotificationService @Inject() (protected val dbConfigProvider ept.projects.filter(_.exportedVulnerabilityId inSet ticketsIds).result ).map{_.groupBy(_.exportedVulnerabilityId).mapValues(_.map(_.projectFullId).toSet).map(identity).withDefaultValue(Set())} - def ticketsForVulnerabilities(vulnerabilities: Traversable[String]) = db.run( + def ticketsForVulnerabilities(vulnerabilities: Traversable[String]): Future[Map[String, (Int, ExportedVulnerability[T])]] = db.run( ept.tickets.filter(_.vulnerabilityName inSet vulnerabilities).result ).map(_.map{ rec => rec._2.vulnerabilityName -> rec diff --git a/conf/evolutions/default/9.sql b/conf/evolutions/default/9.sql new file mode 100644 index 0000000..4c44883 --- /dev/null +++ b/conf/evolutions/default/9.sql @@ -0,0 +1,17 @@ +# --- !Ups +ALTER TABLE "exported_diff_db_vulnerabilities" ADD "done" BOOLEAN NOT NULL DEFAULT FALSE; +ALTER TABLE "exported_email_vulnerabilities" ADD "done" BOOLEAN NOT NULL DEFAULT FALSE; +ALTER TABLE "exported_issue_tracker_vulnerabilities" ADD "done" BOOLEAN NOT NULL DEFAULT FALSE; + +CREATE INDEX "idx_exported_diff_db_vulnerabilities_done" ON "exported_diff_db_vulnerabilities" ("done"); +CREATE INDEX "idx_exported_email_vulnerabilities_done" ON "exported_email_vulnerabilities" ("done"); +CREATE INDEX "idx_exported_issue_tracker_vulnerabilities_done" ON "exported_issue_tracker_vulnerabilities" ("done"); + +# --- !Downs +DROP INDEX "idx_exported_diff_db_vulnerabilities_done"; +DROP INDEX "idx_exported_email_vulnerabilities_done"; +DROP INDEX "idx_exported_issue_tracker_vulnerabilities_done"; + +ALTER TABLE "exported_diff_db_vulnerabilities" DROP COLUMN "done"; +ALTER TABLE "exported_email_vulnerabilities" DROP COLUMN "done"; +ALTER TABLE "exported_issue_tracker_vulnerabilities" DROP COLUMN "done";