Added support for changelog

app/com/ysoft/odc/SetDiff.scala

@@ -5,4 +5,10 @@ class SetDiff[T](val oldSet: Set[T], val newSet: Set[T]) {
   lazy val removed = oldSet -- newSet
   lazy val isEmpty = newSet == oldSet
   def nonEmpty = !isEmpty
+
+  def map[U](f: T => U): SetDiff[U] = new SetDiff[U](
+    oldSet = oldSet.map(f),
+    newSet = newSet.map(f)
+  )
+
 }

app/controllers/Notifications.scala

@@ -104,9 +104,12 @@ class Notifications @Inject()(
           parsedReports = dependencyCheckReportsParser.parseReports(successfulReports)
           lds = LibDepStatistics(dependencies = parsedReports.groupedDependencies.toSet, libraries = libraries.toSet)
           issuesExportResultFuture = exportToIssueTracker(lds, parsedReports.projectsReportInfo)
+          diffDbExportResultFuture = exportToDiffDb(lds, parsedReports.projectsReportInfo)
+          //mailExportResultFuture = diffDbExportResultFuture.flatMap(_ => exportToEmailDigest(lds, parsedReports.projectsReportInfo))
           mailExportResultFuture = exportToEmail(lds, parsedReports.projectsReportInfo)
           (missingTickets, newTicketIds, updatedTickets) <- issuesExportResultFuture
           (missingEmails, newMessageIds, updatedEmails) <- mailExportResultFuture
+          (missingVulns, newVulnIds, updatedVulns) <- diffDbExportResultFuture
         } yield Ok(
           missingTickets.mkString("\n") + "\n\n" + newTicketIds.mkString("\n") + updatedTickets.toString +
             "\n\n" +
@@ -130,6 +133,7 @@ class Notifications @Inject()(
     }
   }
 
+  // FIXME: In case of crash during export, one change might be exported multiple times. This can't be solved in e-mail exports, but it might be solved in issueTracker and diffDb exports.
   private def exportToIssueTracker(lds: LibDepStatistics, p: ProjectsWithReports) = forService(issueTrackerServiceOption){ issueTrackerService =>
     notifyVulnerabilities[String](lds, notificationService.issueTrackerExport, p) { (vulnerability, dependencies) =>
       issueTrackerService.reportVulnerability(vulnerability)
@@ -138,6 +142,21 @@ class Notifications @Inject()(
     }
   }
 
+  private def exportToDiffDb(lds: LibDepStatistics, p: ProjectsWithReports) = {
+    notifyVulnerabilities[String](lds, notificationService.diffDbExport, p){ (vulnerability, dependencies) =>
+      //?save_new_vulnerability
+      val affectedProjects = dependencies.flatMap(_.projects)
+      val diff = new SetDiff(Set(), affectedProjects)
+      notificationService.changeAffectedProjects(vulnerability.name, diff.map(_.fullId)).map{_ =>
+        ExportedVulnerability[String](vulnerabilityName = vulnerability.name, ticket = vulnerability.name, ticketFormatVersion = 0)
+      }
+    }{ (vulnerability, diff, id) =>
+      notificationService.changeAffectedProjects(vulnerability.name, diff).map{_ =>
+        ()
+      }
+    }
+  }
+
   // Redirection to a specific position does not look intuituve now, so it has been disabled for now.
   private def redirectToProject(project: String)(implicit th: DefaultRequest) = Redirect(routes.Notifications.listProjects()/*.withFragment("project-" + URLEncoder.encode(project, "utf-8")).absoluteURL()*/)
 

app/models/Change.scala

@@ -0,0 +1,38 @@
+package models
+
+import java.time.LocalTime
+
+import models.profile.MappedJdbcType
+import models.profile.api._
+import models.jodaSupport._
+import models.profile.api._
+import org.joda.time.{DateTime, LocalDate}
+import play.api.data.Form
+import slick.lifted.{ProvenShape, Tag}
+
+
+object Change {
+  abstract sealed class Direction private[Change] (private[Change] val description: String)
+  object Direction{
+    object Added extends Direction("added")
+    object Removed extends Direction("removed")
+    val All = Set(Added, Removed)
+    val ByName = All.map(x => x.description -> x).toMap
+    implicit val TypeMapper = MappedJdbcType.base[Direction, String](_.description, ByName)
+  }
+
+}
+
+case class Change (time: DateTime, vulnerabilityName: String, projectName: String, direction: Change.Direction)
+
+class Changes(tag: Tag) extends Table[(Int, Change)](tag, "change"){
+  def id = column[Int]("id", O.PrimaryKey, O.AutoInc)
+  import Change.Direction.TypeMapper
+  def time = column[DateTime]("time")
+  def vulnerabilityName = column[String]("vulnerability_name")
+  def projectName = column[String]("project_name")
+  def direction = column[Change.Direction]("direction")
+
+  def base = (time, vulnerabilityName, projectName, direction) <> ((Change.apply _).tupled, Change.unapply)
+  override def * = (id, base)
+}

app/models/package.scala

@@ -1,6 +1,8 @@
 
 import java.nio.file.{Paths, Files}
 
+import slick.lifted.MappedProjection
+
 import scala.language.reflectiveCalls
 
 package object models {
@@ -18,6 +20,7 @@ package object models {
     val snoozesTable = TableQuery[Snoozes]
     val authTokens = TableQuery[CookieAuthenticators]
     val vulnerabilitySubscriptions = TableQuery[VulnerabilitySubscriptions]
+    val changelog = TableQuery[Changes]
 
     val issueTrackerExportTables = new ExportPlatformTables[String, (String, String, Int)](){
       val tableNamePart = "issue_tracker"
@@ -44,10 +47,24 @@ package object models {
       override val tickets = TableQuery[EmailExportedVulnerabilities]
     }
 
+    val diffDbExportTables = new ExportPlatformTables[String, (String, Int)] {
+      val tableNamePart = "diff_db"
+      class DiffDbVulnerabilities(tag: Tag) extends ExportedVulnerabilities[String, (String, Int)](tag, tableNamePart){
+        override def base: MappedProjection[ExportedVulnerability[String], (String, Int)] = (vulnerabilityName, ticketFormatVersion) <> (
+          ((n: String, v: Int) => ExportedVulnerability[String](n, n, v)).tupled,
+          obj => ExportedVulnerability.unapply[String](obj).map{case (n, _, v) => (n, v)}
+        )
+      }
+      class DiffDbVulnerabilityProject(tag: Tag) extends ExportedVulnerabilityProjects(tag, tableNamePart)
+
+      override val projects = TableQuery[DiffDbVulnerabilityProject]
+      override val tickets = TableQuery[DiffDbVulnerabilities]
+    }
+
     /*{
       import profile.SchemaDescription
       val schema = Seq[Any{def schema: SchemaDescription}](
-        vulnerabilitySubscriptions, issueTrackerExportTables, mailExportTables
+        diffDbExportTables, changelog
       ).map(_.schema).foldLeft(profile.DDL(Seq(), Seq()))(_ ++ _)
 
       val sql = Seq(
@@ -58,7 +75,7 @@ package object models {
         schema.dropStatements.toSeq.map(_+";").mkString("\n").dropWhile(_ == "\n"),
         "\n"
       ).mkString("\n")
-      Files.write(Paths.get("conf/evolutions/default/6.sql"), sql.getBytes("utf-8"))
+      Files.write(Paths.get("conf/evolutions/default/7.sql"), sql.getBytes("utf-8"))
     }*/
 
   }

app/services/VulnerabilityNotificationService.scala

@@ -1,20 +1,34 @@
 package services
 
+import _root_.org.joda.time.DateTime
 import com.google.inject.Inject
 import com.mohiva.play.silhouette.api.LoginInfo
 import com.ysoft.odc.SetDiff
 import controllers.{ProjectsWithReports, ReportInfo}
 import models._
-import play.api.db.slick.{HasDatabaseConfigProvider, DatabaseConfigProvider}
+import play.api.db.slick.{DatabaseConfigProvider, HasDatabaseConfigProvider}
+import slick.jdbc.TransactionIsolation
 
-import scala.collection.immutable.Iterable
-import scala.concurrent.{Future, ExecutionContext}
+import scala.concurrent.{ExecutionContext, Future}
+
+final class SingleFutureExecutionThrottler() (implicit executionContext: ExecutionContext){
+  private var nextFuture: Future[_] = Future.successful(null)
+
+  def throttle[T](f: => Future[T]): Future[T] = synchronized{
+    val newFuture = nextFuture.recover{ case _ => null}.flatMap(_ => f)
+    nextFuture = newFuture
+    newFuture
+  }
+
+}
+
+final class NoThrottler() (implicit executionContext: ExecutionContext){
+  def throttle[T](f: => Future[T]): Future[T] = f
+}
 
 class VulnerabilityNotificationService @Inject() (protected val dbConfigProvider: DatabaseConfigProvider)(implicit executionContext: ExecutionContext) extends HasDatabaseConfigProvider[models.profile.type]{
   import dbConfig.driver.api._
-
-  import models.tables
-  import models.tables.vulnerabilitySubscriptions
+  import models.tables._
 
   def watchedProjectsByUser(identity: LoginInfo) = db.run(vulnerabilitySubscriptions.filter(_.user === identity).result)
   def subscribe(user: LoginInfo, project: String) = db.run(vulnerabilitySubscriptions += VulnerabilitySubscription(user = user, project = project))
@@ -61,8 +75,33 @@ class VulnerabilityNotificationService @Inject() (protected val dbConfigProvider
 
   }
 
+  /**
+    * The changelogThrottler is a temporary hack than prevents some congestion that seems to occur in HikariCP or maybe in Slick.
+    * It prevents exceptions like “java.sql.SQLException: Timeout of 1001ms encountered waiting for connection”.
+    * It is probably prevented at a wrong level, but it works :)
+    */
+  private val changelogThrottler = new SingleFutureExecutionThrottler()
+  // private val changelogThrottler = new NoThrottler()
+
+  def changeAffectedProjects(vulnerabilityName: String, affectedProjectsDiff: SetDiff[String]): Future[Unit] = {
+    val time = DateTime.now()
+    def createRecord(projectName: String, direction: Change.Direction) = Change(time, vulnerabilityName, projectName, direction)
+    val recordsToAdd = affectedProjectsDiff.added.map(projectName => createRecord(projectName, Change.Direction.Added)) ++
+      affectedProjectsDiff.removed.map(projectName => createRecord(projectName, Change.Direction.Removed))
+    /*
+    Transaction:
+    It is essential to ensure that records in changelog appear in order. Low level of isolation might be even worse than running outside of transaction.
+    In longer term, it should be wrapped with transaction *with a proper isolation level* together with the export status modification.
+    */
+    changelogThrottler.throttle(db.run(
+      (changelog.map(_.base) ++= recordsToAdd).withTransactionIsolation(TransactionIsolation.Serializable)
+    ).map(_ => ()))
+  }
+
   val issueTrackerExport = new ExportPlatform(tables.issueTrackerExportTables)
 
   val mailExport = new ExportPlatform(tables.mailExportTables)
 
+  val diffDbExport = new ExportPlatform(tables.diffDbExportTables)
+
 }

conf/evolutions/default/7.sql

@@ -0,0 +1,12 @@
+# --- !Ups
+create table "exported_diff_db_vulnerabilities" ("id" SERIAL NOT NULL PRIMARY KEY,"vulnerability_name" VARCHAR NOT NULL,"ticket_format_version" INTEGER NOT NULL);
+create unique index "idx_exported_diff_db_vulnerabilities_vulnerabilityName" on "exported_diff_db_vulnerabilities" ("vulnerability_name");
+create table "exported_diff_db_vulnerability_projects" ("exported_vulnerability_id" INTEGER NOT NULL,"full_project_id" VARCHAR NOT NULL);
+create unique index "idx_exported_diff_db_vulnerability_projects_all" on "exported_diff_db_vulnerability_projects" ("exported_vulnerability_id","full_project_id");
+create table "change" ("id" SERIAL NOT NULL PRIMARY KEY,"time" TIMESTAMP NOT NULL,"vulnerability_name" VARCHAR NOT NULL,"project_name" VARCHAR NOT NULL,"direction" VARCHAR NOT NULL);
+
+# --- !Downs
+drop table "change";
+drop table "exported_diff_db_vulnerability_projects";
+drop table "exported_diff_db_vulnerabilities";
+