Added support for mail digests

app/controllers/Notifications.scala

@@ -1,22 +1,21 @@
 package controllers
 
-import java.net.URLEncoder
 import java.util.concurrent.atomic.AtomicBoolean
 import javax.inject.Inject
 
+import com.ysoft.concurrent.FutureLock._
 import com.ysoft.odc.{Absolutizer, SetDiff}
 import controllers.Statistics.LibDepStatistics
 import models.{EmailMessageId, ExportedVulnerability}
 import play.api.i18n.MessagesApi
 import play.api.libs.Crypto
-import play.api.mvc.{RequestHeader, Action}
+import play.api.mvc.Action
 import play.api.{Configuration, Logger}
-import services.{EmailExportService, IssueTrackerService, LibrariesService, VulnerabilityNotificationService}
+import services._
 import views.html.DefaultRequest
 
 import scala.concurrent.Future.{successful => Fut}
 import scala.concurrent.{ExecutionContext, Future}
-import com.ysoft.concurrent.FutureLock._
 
 class Notifications @Inject()(
   config: Configuration,
@@ -105,15 +104,18 @@ class Notifications @Inject()(
           lds = LibDepStatistics(dependencies = parsedReports.groupedDependencies.toSet, libraries = libraries.toSet)
           issuesExportResultFuture = exportToIssueTracker(lds, parsedReports.projectsReportInfo)
           diffDbExportResultFuture = exportToDiffDb(lds, parsedReports.projectsReportInfo)
-          //mailExportResultFuture = diffDbExportResultFuture.flatMap(_ => exportToEmailDigest(lds, parsedReports.projectsReportInfo))
-          mailExportResultFuture = exportToEmail(lds, parsedReports.projectsReportInfo)
+          mailExportResultFuture = emailExportServiceOption.map(_.exportType) match{
+            case Some(EmailExportType.Vulnerabilities) => exportToEmail(lds, parsedReports.projectsReportInfo).map((_: (_, _, _)) => ())
+            case Some(EmailExportType.Digest) => diffDbExportResultFuture.flatMap(_ => exportToEmailDigest(lds, parsedReports.projectsReportInfo))
+            case None => Future(())
+          }
           (missingTickets, newTicketIds, updatedTickets) <- issuesExportResultFuture
-          (missingEmails, newMessageIds, updatedEmails) <- mailExportResultFuture
+          (_: Unit) <- mailExportResultFuture
           (missingVulns, newVulnIds, updatedVulns) <- diffDbExportResultFuture
         } yield Ok(
-          missingTickets.mkString("\n") + "\n\n" + newTicketIds.mkString("\n") + updatedTickets.toString +
-            "\n\n" +
-            missingEmails.mkString("\n") + "\n\n" + newMessageIds.mkString("\n") + updatedEmails.toString
+          missingTickets.mkString("\n") + "\n\n" + newTicketIds.mkString("\n") + updatedTickets.toString
+            //"\n\n" +
+            //missingEmails.mkString("\n") + "\n\n" + newMessageIds.mkString("\n") + updatedEmails.toString
         )
       } whenLocked {
         Fut(ServiceUnavailable("A cron job seems to be running at this time"))
@@ -151,10 +153,27 @@ class Notifications @Inject()(
         ExportedVulnerability[String](vulnerabilityName = vulnerability.name, ticket = vulnerability.name, ticketFormatVersion = 0)
       }
     }{ (vulnerability, diff, id) =>
-      notificationService.changeAffectedProjects(vulnerability.name, diff).map{_ =>
-        ()
+      notificationService.changeAffectedProjects(vulnerability.name, diff)
     }
   }
+
+  private val emailDigestThrottler = new SingleFutureExecutionThrottler()
+
+  private def exportToEmailDigest(lds: LibDepStatistics, p: ProjectsWithReports) = emailExportServiceOption.fold(Future.successful(())){ emailExportService =>
+    notificationService.subscribers.flatMap{ subscribers =>
+      Future.traverse(subscribers){ case (subscriber, subscriptions) =>
+        emailDigestThrottler.throttle {
+          notificationService.sendDigestToSubscriber(subscriber, subscriptions) {
+            case Seq() => Future.successful(())
+            case changes =>
+              for {
+                emailMessage <- emailExportService.emailDigest(subscriber, changes, p)
+                (_: String) <- emailExportService.sendEmail(emailMessage)
+              } yield ()
+          }
+        }
+      }.map((_ : Iterable[Unit]) => ())
+    }
   }
 
   // Redirection to a specific position does not look intuituve now, so it has been disabled for now.

app/controllers/ProjectsWithReports.scala

@@ -48,7 +48,11 @@ class ProjectsWithReports (val projects: Projects, val reports: Set[String]) {
     reportsMap ++ reportsMap.values.map(r => r.projectId -> ReportInfo(projectId = r.projectId, fullId = r.projectId, subprojectNameOption = None, projectName = r.projectName))
   }
 
-  def parseUnfriendlyName(unfriendlyName: String): ReportInfo = {
+  def parseUnfriendlyNameGracefully(unfriendlyName: String) = parseUnfriendlyName(unfriendlyName, identity)
+
+  def parseUnfriendlyName(unfriendlyName: String): ReportInfo = parseUnfriendlyName(unfriendlyName, _ => sys.error(s"Project $unfriendlyName not found!"))
+
+  private def parseUnfriendlyName(unfriendlyName: String, missingProject: String => String): ReportInfo = {
     val (baseName, theRest) = unfriendlyName.span(_ != '/')
     val removeLeadingMess = RestMessBeginRegexp.replaceAllIn(_: String, "")
     val removeTrailingMess = RestMessEndRegexp.replaceAllIn(_: String, "")
@@ -57,7 +61,7 @@ class ProjectsWithReports (val projects: Projects, val reports: Set[String]) {
     ReportInfo(
       projectId = baseName,
       fullId = unfriendlyName,
-      projectName = projects.projectMap(baseName),
+      projectName = projects.projectMap.getOrElse(baseName, missingProject(baseName)),
       subprojectNameOption = subProjectOption.orElse(Some("root project"))
     )
   }

app/models/Change.scala

@@ -23,7 +23,7 @@ object Change {
 
 }
 
-case class Change (time: DateTime, vulnerabilityName: String, projectName: String, direction: Change.Direction)
+case class Change (time: DateTime, vulnerabilityName: String, projectName: String, direction: Change.Direction, notifiedToSomebody: Boolean)
 
 class Changes(tag: Tag) extends Table[(Int, Change)](tag, "change"){
   def id = column[Int]("id", O.PrimaryKey, O.AutoInc)
@@ -32,7 +32,8 @@ class Changes(tag: Tag) extends Table[(Int, Change)](tag, "change"){
   def vulnerabilityName = column[String]("vulnerability_name")
   def projectName = column[String]("project_name")
   def direction = column[Change.Direction]("direction")
+  def notifiedToSomebody = column[Boolean]("notified_to_somebody")
 
-  def base = (time, vulnerabilityName, projectName, direction) <> ((Change.apply _).tupled, Change.unapply)
+  def base = (time, vulnerabilityName, projectName, direction, notifiedToSomebody) <> ((Change.apply _).tupled, Change.unapply)
   override def * = (id, base)
 }

app/models/NotificationDigestStatus.scala

@@ -0,0 +1,15 @@
+package models
+
+import com.mohiva.play.silhouette.api.LoginInfo
+import models.profile.api._
+import slick.lifted.Tag
+
+case class NotificationDigestStatus(user: LoginInfo, lastChangelogIdOption: Option[Int])
+
+
+class NotificationDigestStatuses(tag: Tag) extends Table[NotificationDigestStatus](tag, "notification_digest_status"){
+  val user = new LoginInfoColumns("user", this)
+  def lastChangelogId = column[Int]("last_changelog_id").?
+  def * = (user(), lastChangelogId) <> (NotificationDigestStatus.tupled, NotificationDigestStatus.unapply)
+  def idx = index("notification_digest_status_user_idx", user(), unique = true)
+}

app/models/package.scala

@@ -21,6 +21,7 @@ package object models {
     val authTokens = TableQuery[CookieAuthenticators]
     val vulnerabilitySubscriptions = TableQuery[VulnerabilitySubscriptions]
     val changelog = TableQuery[Changes]
+    val notificationDigestStatuses = TableQuery[NotificationDigestStatuses]
 
     val issueTrackerExportTables = new ExportPlatformTables[String, (String, String, Int)](){
       val tableNamePart = "issue_tracker"
@@ -64,7 +65,7 @@ package object models {
     /*{
       import profile.SchemaDescription
       val schema = Seq[Any{def schema: SchemaDescription}](
-        diffDbExportTables, changelog
+        notificationDigestStatuses
       ).map(_.schema).foldLeft(profile.DDL(Seq(), Seq()))(_ ++ _)
 
       val sql = Seq(
@@ -75,7 +76,7 @@ package object models {
         schema.dropStatements.toSeq.map(_+";").mkString("\n").dropWhile(_ == "\n"),
         "\n"
       ).mkString("\n")
-      Files.write(Paths.get("conf/evolutions/default/7.sql"), sql.getBytes("utf-8"))
+      Files.write(Paths.get("conf/evolutions/default/8.sql"), sql.getBytes("utf-8"))
     }*/
 
   }

app/modules/EmailExportModule.scala

@@ -8,8 +8,8 @@ import net.ceedubs.ficus.Ficus._
 import net.codingwell.scalaguice.ScalaModule
 import play.api.Configuration
 import play.api.libs.mailer.MailerClient
-import services.{EmailExportService, VulnerabilityNotificationService}
-
+import services.{OdcService, EmailExportService, EmailExportType, VulnerabilityNotificationService}
+import net.ceedubs.ficus.readers.EnumerationReader._
 import scala.concurrent.ExecutionContext
 
 class EmailExportModule extends AbstractModule with ScalaModule{
@@ -17,11 +17,20 @@ class EmailExportModule extends AbstractModule with ScalaModule{
   }
 
   @Provides
-  def provideIssueTrackerOption(conf: Configuration, mailerClient: MailerClient, notificationService: VulnerabilityNotificationService, absolutizer: Absolutizer, @Named("email-sending") emailSendingExecutionContext: ExecutionContext)(implicit executionContext: ExecutionContext): Option[EmailExportService] = {
+  def provideIssueTrackerOption(
+                                 conf: Configuration,
+                                 mailerClient: MailerClient,
+                                 notificationService: VulnerabilityNotificationService,
+                                 absolutizer: Absolutizer,
+                                 odcService: OdcService,
+                                 @Named("email-sending") emailSendingExecutionContext: ExecutionContext
+                               )(implicit executionContext: ExecutionContext): Option[EmailExportService] = {
     println(s"emailSendingExecutionContext = $emailSendingExecutionContext")
     conf.getConfig("yssdc.export.email").map{c =>
       new EmailExportService(
         from = c.underlying.as[String]("from"),
+        odcService = odcService,
+        exportType = c.underlying.getAs[EmailExportType.Value]("type").ensuring{ x => println(x) ; true}.getOrElse(EmailExportType.Vulnerabilities),
         mailerClient = mailerClient,
         emailSendingExecutionContext = emailSendingExecutionContext,
         absolutizer = absolutizer,

app/services/EmailExportService.scala

@@ -1,21 +1,32 @@
 package services
 
 import java.util.NoSuchElementException
-import javax.inject.Named
 
-import com.ysoft.odc.{SetDiff, Absolutizer}
+import com.mohiva.play.silhouette.api.LoginInfo
+import com.ysoft.odc.{Absolutizer, SetDiff}
 import controllers._
-import models.EmailMessageId
-import play.api.libs.mailer.{MailerClient, Email}
+import models.Change.Direction
+import models.{Change, EmailMessageId}
+import play.api.libs.mailer.{Email, MailerClient}
 
 import scala.concurrent.{ExecutionContext, Future}
 
-class EmailExportService(from: String, nobodyInterestedContact: String, mailerClient: MailerClient, notificationService: VulnerabilityNotificationService, emailSendingExecutionContext: ExecutionContext, absolutizer: Absolutizer)(implicit executionContext: ExecutionContext) {
+
+object EmailExportType extends Enumeration {
+  val Vulnerabilities = Value("vulnerabilities")
+  val Digest = Value("digest")
+}
+
+class EmailExportService(from: String, nobodyInterestedContact: String, val exportType: EmailExportType.Value, odcService: OdcService, mailerClient: MailerClient, notificationService: VulnerabilityNotificationService, emailSendingExecutionContext: ExecutionContext, absolutizer: Absolutizer)(implicit executionContext: ExecutionContext) {
+  // Maybe it is not the best place for exportType, but I am not sure if we want this to be configurable. If no, then we can get rid of it. If yes, we should refactor it.
+
+
+  private def getEmail(loginInfo: LoginInfo) = loginInfo.providerKey // TODO: get the email in a cleaner way
 
   def recipientsForProjects(projects: Set[ReportInfo]) = for{
     recipients <- notificationService.getRecipientsForProjects(projects)
   } yield {
-    recipients.map(_.providerKey) match { // TODO: get the email in a cleaner way
+    recipients.map(getEmail) match {
       case Seq() => Seq(nobodyInterestedContact) -> false
       case other => other -> true
     }
@@ -61,4 +72,47 @@ class EmailExportService(from: String, nobodyInterestedContact: String, mailerCl
     bodyText = Some(vulnerability.description + "\n\n" + s"More details: "+absolutizer.absolutize(routes.Statistics.vulnerability(vulnerability.name, None)))
   )
 
+
+
+  def emailDigest(subscriber: LoginInfo, changes: Seq[Change], projects: ProjectsWithReports): Future[Email] = {
+    val vulnNames = changes.map(_.vulnerabilityName).toSet
+    for {
+      vulns <- Future.traverse(vulnNames.toSeq)(name => odcService.getVulnerabilityDetails(name).map(v => name -> v.get)).map(_.toMap)
+      groups = changes.groupBy(_.direction).withDefaultValue(Seq())
+    } yield {
+      val changesMarks = Map(Direction.Added -> "❢", Direction.Removed -> "☑")
+      def vulnerabilityText(change: Change, vulnerability: Vulnerability) = (
+        s"#### ${changesMarks(change.direction)} ${vulnerability.name}${vulnerability.cvssScore.fold("")(sev => s" (CVSS severity: $sev)")}"
+        +"\n"+vulnerability.description
+        +"\nmore info: "+absolutizer.absolutize(routes.Statistics.vulnerability(vulnerability.name, None))
+      )
+      def vulnChanges(changes: Seq[Change]) =
+        changes.map(c => c -> vulns(c.vulnerabilityName))
+          .sortBy{case (change, vuln) => (vuln.cvssScore.map(-_), vuln.name)}
+          .map((vulnerabilityText _).tupled)
+          .mkString("\n\n")
+      def vulnerableProjects(projectIdToChanges: Map[String, Seq[Change]]) =
+        projectIdToChanges.toIndexedSeq.map{case (project, ch) => (projects.parseUnfriendlyNameGracefully(project), ch)}
+          .sortBy{case (ri, _) => friendlyProjectNameString(ri).toLowerCase}
+          .map{case (project, changes) => "### "+friendlyProjectNameString(project)+"\n"+vulnChanges(changes)}
+          .mkString("\n\n")
+      def section(title: String, direction: Direction) = {
+        groups(direction) match {
+          case Seq() => None
+          case list => Some("## "+title + "\n\n" + vulnerableProjects(list.groupBy(_.projectName)))
+        }
+      }
+      Email(
+        subject = s"New changes in vulnerabilities (${changes.size}: +${groups(Direction.Added).size} -${groups(Direction.Removed).size})",
+        to = Seq(getEmail(subscriber)),
+        from = from,
+        bodyText = Some(Seq(
+          section("Projects newly affected by a vulnerability", Direction.Added),
+          section("Projects no longer affected by a vulnerability", Direction.Removed)
+        ).flatten.mkString("\n\n"))
+        //bodyHtml =  TODO
+      )
+    }
+  }
+
 }

app/services/VulnerabilityNotificationService.scala

@@ -6,7 +6,10 @@ import com.mohiva.play.silhouette.api.LoginInfo
 import com.ysoft.odc.SetDiff
 import controllers.{ProjectsWithReports, ReportInfo}
 import models._
+import models.tables._
+import play.api.Logger
 import play.api.db.slick.{DatabaseConfigProvider, HasDatabaseConfigProvider}
+import slick.dbio.FutureAction
 import slick.jdbc.TransactionIsolation
 
 import scala.concurrent.{ExecutionContext, Future}
@@ -30,8 +33,25 @@ class VulnerabilityNotificationService @Inject() (protected val dbConfigProvider
   import dbConfig.driver.api._
   import models.tables._
 
+  def subscribers = db.run(vulnerabilitySubscriptions.result).map(_.groupBy(_.user))
   def watchedProjectsByUser(identity: LoginInfo) = db.run(vulnerabilitySubscriptions.filter(_.user === identity).result)
-  def subscribe(user: LoginInfo, project: String) = db.run(vulnerabilitySubscriptions += VulnerabilitySubscription(user = user, project = project))
+
+  def subscribe(user: LoginInfo, project: String) = db.run(
+    DBIO.seq(
+      ensureUserHasNotificationDigestStatus(user),
+      vulnerabilitySubscriptions += VulnerabilitySubscription(user = user, project = project)
+    )
+  )
+
+  private def ensureUserHasNotificationDigestStatus(user: LoginInfo): DBIOAction[Unit, slick.dbio.NoStream, Effect.Read with Effect.Write] = (
+    notificationDigestStatuses.filter(_.user === user).result.map(_.nonEmpty) flatMap {
+      case true => DBIO.seq()
+      case false => for{
+        changelogTopIdOption <- changelogTopIdOptionQuery
+        (res: Int) <- notificationDigestStatuses += NotificationDigestStatus(user = user, lastChangelogIdOption = changelogTopIdOption)
+      } yield ()
+    }
+  ).withTransactionIsolation(TransactionIsolation.Serializable)
 
   def unsubscribe(user: LoginInfo, project: String) = db.run(vulnerabilitySubscriptions.filter(vs => vs.user === user && vs.project === project).delete)
 
@@ -85,7 +105,13 @@ class VulnerabilityNotificationService @Inject() (protected val dbConfigProvider
 
   def changeAffectedProjects(vulnerabilityName: String, affectedProjectsDiff: SetDiff[String]): Future[Unit] = {
     val time = DateTime.now()
-    def createRecord(projectName: String, direction: Change.Direction) = Change(time, vulnerabilityName, projectName, direction)
+    def createRecord(projectName: String, direction: Change.Direction) = Change(
+      time = time,
+      vulnerabilityName = vulnerabilityName,
+      projectName = projectName,
+      direction = direction,
+      notifiedToSomebody = false
+    )
     val recordsToAdd = affectedProjectsDiff.added.map(projectName => createRecord(projectName, Change.Direction.Added)) ++
       affectedProjectsDiff.removed.map(projectName => createRecord(projectName, Change.Direction.Removed))
     /*
@@ -98,6 +124,34 @@ class VulnerabilityNotificationService @Inject() (protected val dbConfigProvider
     ).map(_ => ()))
   }
 
+  private def changelogTopIdOptionQuery = changelog.map(_.id).max.result
+
+  def sendDigestToSubscriber(subscriber: LoginInfo, subscriptions: Seq[VulnerabilitySubscription])(sendDigest: Seq[Change] => Future[Unit]): Future[Unit] = {
+    def subscriptionCondition(change: Changes, subscription: VulnerabilitySubscription):  Rep[Boolean] = {
+      //noinspection ScalaUnnecessaryParentheses – because it looks less confusing
+      (change.projectName === subscription.project) ||
+        (if (subscription.project contains '/') (false: Rep[Boolean]) else change.projectName.startsWith(subscription.project + "/"))
+    }
+    val projectCondition: Changes => Rep[Boolean] = (change) =>
+      subscriptions.foldLeft(false: Rep[Boolean])((cond, subscription) => cond || subscriptionCondition(change, subscription))
+    val notificationDigestStatusSelection = notificationDigestStatuses.filter(_.user === subscriber)
+    db.run(
+      (
+        for{
+          oldStatus <- notificationDigestStatusSelection.result.map(_.head)
+          _ = println(oldStatus.lastChangelogIdOption.fold(changelog.filter(_ => true: Rep[Boolean]))(lastChangelogId => changelog.filter(_.id > lastChangelogId)).filter(projectCondition).result.statements.mkString("\n"))
+          changelogSinceLastNotified <- oldStatus.lastChangelogIdOption.fold(changelog.filter(_ => true: Rep[Boolean]))(lastChangelogId => changelog.filter(_.id > lastChangelogId)).filter(projectCondition).result
+          changelogIds = changelogSinceLastNotified.map(_._1)
+          changelogTopIdOption <- changelogTopIdOptionQuery
+          newLastChangelogIdOption = changelogTopIdOption.orElse(oldStatus.lastChangelogIdOption)
+          (_: Unit) <- FutureAction(sendDigest(changelogSinceLastNotified.map(_._2))) // Yes, we are waiting for user to be notified over some slow I/O when having an open transaction
+          (_: Int) <- notificationDigestStatusSelection.map(_.lastChangelogId).update(newLastChangelogIdOption)
+          (_: Int) <- changelog.filter(_.id inSet changelogIds).map(_.notifiedToSomebody).update(true)
+        } yield ()
+      ).withTransactionIsolation(TransactionIsolation.Serializable)
+    )
+  }
+
   val issueTrackerExport = new ExportPlatform(tables.issueTrackerExportTables)
 
   val mailExport = new ExportPlatform(tables.mailExportTables)

build.sbt

@@ -8,6 +8,8 @@ scalaVersion := "2.11.7"
 
 resolvers += "Atlassian Releases" at "https://maven.atlassian.com/public/"
 
+resolvers += Resolver.jcenterRepo
+
 libraryDependencies ++= Seq(
   //jdbc,
   cache,
@@ -71,7 +73,7 @@ libraryDependencies += "org.webjars.bower" % "jquery.scrollTo" % "2.1.2"
 
 libraryDependencies += "net.codingwell" %% "scala-guice" % "4.0.0"
 
-libraryDependencies += "net.ceedubs" %% "ficus" % "1.1.2"
+libraryDependencies += "com.iheart" %% "ficus" % "1.2.3"
 
 libraryDependencies += "org.owasp" % "dependency-check-core" % "1.3.0"
 

conf/application.conf.-example

@@ -51,6 +51,7 @@ yssdc{
     email{
       from = "info@example.com"
       noSubscriberContact = "foobar@example.com"
+      //optional: type = "digest" or type="vulnerabilities" (default); Digest is WIP.
     }
   }
   projects = {jobId:humanReadableName, …}

conf/evolutions/default/8.sql

@@ -0,0 +1,24 @@
+# --- !Ups
+CREATE TABLE "notification_digest_status" (
+  "user_provider_id" VARCHAR NOT NULL,
+  "user_provider_key" VARCHAR NOT NULL,
+  "last_changelog_id" INTEGER NULL
+);
+CREATE UNIQUE INDEX "notification_digest_status_user_idx" ON "notification_digest_status" ("user_provider_id","user_provider_key");
+
+INSERT INTO notification_digest_status
+(user_provider_id, user_provider_key, last_changelog_id)
+  SELECT
+    subscriber_provider_id  AS user_provider_id,
+    subscriber_provider_key AS user_provider_key,
+    (SELECT MAX(id) from change) AS last_changelog_id
+  FROM vulnerability_subscription
+  GROUP BY subscriber_provider_id, subscriber_provider_key;
+
+ALTER TABLE change ADD COLUMN "notified_to_somebody" BOOLEAN NOT NULL DEFAULT FALSE;
+UPDATE change SET notified_to_somebody = TRUE;
+
+# --- !Downs
+drop table "notification_digest_status";
+
+ALTER TABLE change DROP COLUMN "notified_to_somebody";

production.conf-example

@@ -49,6 +49,7 @@ yssdc{
     email{
       from = "info@example.com"
       noSubscriberContact = "foobar@example.com"
+      //optional: type = "digest" or type="vulnerabilities" (default); Digest is WIP.
     }
   }
   projects = {jobId:humanReadableName, …}