diff --git a/app/controllers/Notifications.scala b/app/controllers/Notifications.scala index 7d57e5a..dab3bde 100644 --- a/app/controllers/Notifications.scala +++ b/app/controllers/Notifications.scala @@ -1,22 +1,21 @@ package controllers -import java.net.URLEncoder import java.util.concurrent.atomic.AtomicBoolean import javax.inject.Inject +import com.ysoft.concurrent.FutureLock._ import com.ysoft.odc.{Absolutizer, SetDiff} import controllers.Statistics.LibDepStatistics import models.{EmailMessageId, ExportedVulnerability} import play.api.i18n.MessagesApi import play.api.libs.Crypto -import play.api.mvc.{RequestHeader, Action} +import play.api.mvc.Action import play.api.{Configuration, Logger} -import services.{EmailExportService, IssueTrackerService, LibrariesService, VulnerabilityNotificationService} +import services._ import views.html.DefaultRequest import scala.concurrent.Future.{successful => Fut} import scala.concurrent.{ExecutionContext, Future} -import com.ysoft.concurrent.FutureLock._ class Notifications @Inject()( config: Configuration, @@ -105,15 +104,18 @@ class Notifications @Inject()( lds = LibDepStatistics(dependencies = parsedReports.groupedDependencies.toSet, libraries = libraries.toSet) issuesExportResultFuture = exportToIssueTracker(lds, parsedReports.projectsReportInfo) diffDbExportResultFuture = exportToDiffDb(lds, parsedReports.projectsReportInfo) - //mailExportResultFuture = diffDbExportResultFuture.flatMap(_ => exportToEmailDigest(lds, parsedReports.projectsReportInfo)) - mailExportResultFuture = exportToEmail(lds, parsedReports.projectsReportInfo) + mailExportResultFuture = emailExportServiceOption.map(_.exportType) match{ + case Some(EmailExportType.Vulnerabilities) => exportToEmail(lds, parsedReports.projectsReportInfo).map((_: (_, _, _)) => ()) + case Some(EmailExportType.Digest) => diffDbExportResultFuture.flatMap(_ => exportToEmailDigest(lds, parsedReports.projectsReportInfo)) + case None => Future(()) + } (missingTickets, newTicketIds, updatedTickets) <- issuesExportResultFuture - (missingEmails, newMessageIds, updatedEmails) <- mailExportResultFuture + (_: Unit) <- mailExportResultFuture (missingVulns, newVulnIds, updatedVulns) <- diffDbExportResultFuture } yield Ok( - missingTickets.mkString("\n") + "\n\n" + newTicketIds.mkString("\n") + updatedTickets.toString + - "\n\n" + - missingEmails.mkString("\n") + "\n\n" + newMessageIds.mkString("\n") + updatedEmails.toString + missingTickets.mkString("\n") + "\n\n" + newTicketIds.mkString("\n") + updatedTickets.toString + //"\n\n" + + //missingEmails.mkString("\n") + "\n\n" + newMessageIds.mkString("\n") + updatedEmails.toString ) } whenLocked { Fut(ServiceUnavailable("A cron job seems to be running at this time")) @@ -151,9 +153,26 @@ class Notifications @Inject()( ExportedVulnerability[String](vulnerabilityName = vulnerability.name, ticket = vulnerability.name, ticketFormatVersion = 0) } }{ (vulnerability, diff, id) => - notificationService.changeAffectedProjects(vulnerability.name, diff).map{_ => - () - } + notificationService.changeAffectedProjects(vulnerability.name, diff) + } + } + + private val emailDigestThrottler = new SingleFutureExecutionThrottler() + + private def exportToEmailDigest(lds: LibDepStatistics, p: ProjectsWithReports) = emailExportServiceOption.fold(Future.successful(())){ emailExportService => + notificationService.subscribers.flatMap{ subscribers => + Future.traverse(subscribers){ case (subscriber, subscriptions) => + emailDigestThrottler.throttle { + notificationService.sendDigestToSubscriber(subscriber, subscriptions) { + case Seq() => Future.successful(()) + case changes => + for { + emailMessage <- emailExportService.emailDigest(subscriber, changes, p) + (_: String) <- emailExportService.sendEmail(emailMessage) + } yield () + } + } + }.map((_ : Iterable[Unit]) => ()) } } diff --git a/app/controllers/ProjectsWithReports.scala b/app/controllers/ProjectsWithReports.scala index 0bc46c7..24492b8 100644 --- a/app/controllers/ProjectsWithReports.scala +++ b/app/controllers/ProjectsWithReports.scala @@ -48,7 +48,11 @@ class ProjectsWithReports (val projects: Projects, val reports: Set[String]) { reportsMap ++ reportsMap.values.map(r => r.projectId -> ReportInfo(projectId = r.projectId, fullId = r.projectId, subprojectNameOption = None, projectName = r.projectName)) } - def parseUnfriendlyName(unfriendlyName: String): ReportInfo = { + def parseUnfriendlyNameGracefully(unfriendlyName: String) = parseUnfriendlyName(unfriendlyName, identity) + + def parseUnfriendlyName(unfriendlyName: String): ReportInfo = parseUnfriendlyName(unfriendlyName, _ => sys.error(s"Project $unfriendlyName not found!")) + + private def parseUnfriendlyName(unfriendlyName: String, missingProject: String => String): ReportInfo = { val (baseName, theRest) = unfriendlyName.span(_ != '/') val removeLeadingMess = RestMessBeginRegexp.replaceAllIn(_: String, "") val removeTrailingMess = RestMessEndRegexp.replaceAllIn(_: String, "") @@ -57,7 +61,7 @@ class ProjectsWithReports (val projects: Projects, val reports: Set[String]) { ReportInfo( projectId = baseName, fullId = unfriendlyName, - projectName = projects.projectMap(baseName), + projectName = projects.projectMap.getOrElse(baseName, missingProject(baseName)), subprojectNameOption = subProjectOption.orElse(Some("root project")) ) } diff --git a/app/models/Change.scala b/app/models/Change.scala index d557112..c1a149f 100644 --- a/app/models/Change.scala +++ b/app/models/Change.scala @@ -23,7 +23,7 @@ object Change { } -case class Change (time: DateTime, vulnerabilityName: String, projectName: String, direction: Change.Direction) +case class Change (time: DateTime, vulnerabilityName: String, projectName: String, direction: Change.Direction, notifiedToSomebody: Boolean) class Changes(tag: Tag) extends Table[(Int, Change)](tag, "change"){ def id = column[Int]("id", O.PrimaryKey, O.AutoInc) @@ -32,7 +32,8 @@ class Changes(tag: Tag) extends Table[(Int, Change)](tag, "change"){ def vulnerabilityName = column[String]("vulnerability_name") def projectName = column[String]("project_name") def direction = column[Change.Direction]("direction") + def notifiedToSomebody = column[Boolean]("notified_to_somebody") - def base = (time, vulnerabilityName, projectName, direction) <> ((Change.apply _).tupled, Change.unapply) + def base = (time, vulnerabilityName, projectName, direction, notifiedToSomebody) <> ((Change.apply _).tupled, Change.unapply) override def * = (id, base) } \ No newline at end of file diff --git a/app/models/NotificationDigestStatus.scala b/app/models/NotificationDigestStatus.scala new file mode 100644 index 0000000..f040c7b --- /dev/null +++ b/app/models/NotificationDigestStatus.scala @@ -0,0 +1,15 @@ +package models + +import com.mohiva.play.silhouette.api.LoginInfo +import models.profile.api._ +import slick.lifted.Tag + +case class NotificationDigestStatus(user: LoginInfo, lastChangelogIdOption: Option[Int]) + + +class NotificationDigestStatuses(tag: Tag) extends Table[NotificationDigestStatus](tag, "notification_digest_status"){ + val user = new LoginInfoColumns("user", this) + def lastChangelogId = column[Int]("last_changelog_id").? + def * = (user(), lastChangelogId) <> (NotificationDigestStatus.tupled, NotificationDigestStatus.unapply) + def idx = index("notification_digest_status_user_idx", user(), unique = true) +} diff --git a/app/models/package.scala b/app/models/package.scala index 72ef896..dc659d2 100644 --- a/app/models/package.scala +++ b/app/models/package.scala @@ -21,6 +21,7 @@ package object models { val authTokens = TableQuery[CookieAuthenticators] val vulnerabilitySubscriptions = TableQuery[VulnerabilitySubscriptions] val changelog = TableQuery[Changes] + val notificationDigestStatuses = TableQuery[NotificationDigestStatuses] val issueTrackerExportTables = new ExportPlatformTables[String, (String, String, Int)](){ val tableNamePart = "issue_tracker" @@ -64,7 +65,7 @@ package object models { /*{ import profile.SchemaDescription val schema = Seq[Any{def schema: SchemaDescription}]( - diffDbExportTables, changelog + notificationDigestStatuses ).map(_.schema).foldLeft(profile.DDL(Seq(), Seq()))(_ ++ _) val sql = Seq( @@ -75,7 +76,7 @@ package object models { schema.dropStatements.toSeq.map(_+";").mkString("\n").dropWhile(_ == "\n"), "\n" ).mkString("\n") - Files.write(Paths.get("conf/evolutions/default/7.sql"), sql.getBytes("utf-8")) + Files.write(Paths.get("conf/evolutions/default/8.sql"), sql.getBytes("utf-8")) }*/ } diff --git a/app/modules/EmailExportModule.scala b/app/modules/EmailExportModule.scala index 759686c..4478520 100644 --- a/app/modules/EmailExportModule.scala +++ b/app/modules/EmailExportModule.scala @@ -8,8 +8,8 @@ import net.ceedubs.ficus.Ficus._ import net.codingwell.scalaguice.ScalaModule import play.api.Configuration import play.api.libs.mailer.MailerClient -import services.{EmailExportService, VulnerabilityNotificationService} - +import services.{OdcService, EmailExportService, EmailExportType, VulnerabilityNotificationService} +import net.ceedubs.ficus.readers.EnumerationReader._ import scala.concurrent.ExecutionContext class EmailExportModule extends AbstractModule with ScalaModule{ @@ -17,11 +17,20 @@ class EmailExportModule extends AbstractModule with ScalaModule{ } @Provides - def provideIssueTrackerOption(conf: Configuration, mailerClient: MailerClient, notificationService: VulnerabilityNotificationService, absolutizer: Absolutizer, @Named("email-sending") emailSendingExecutionContext: ExecutionContext)(implicit executionContext: ExecutionContext): Option[EmailExportService] = { + def provideIssueTrackerOption( + conf: Configuration, + mailerClient: MailerClient, + notificationService: VulnerabilityNotificationService, + absolutizer: Absolutizer, + odcService: OdcService, + @Named("email-sending") emailSendingExecutionContext: ExecutionContext + )(implicit executionContext: ExecutionContext): Option[EmailExportService] = { println(s"emailSendingExecutionContext = $emailSendingExecutionContext") conf.getConfig("yssdc.export.email").map{c => new EmailExportService( from = c.underlying.as[String]("from"), + odcService = odcService, + exportType = c.underlying.getAs[EmailExportType.Value]("type").ensuring{ x => println(x) ; true}.getOrElse(EmailExportType.Vulnerabilities), mailerClient = mailerClient, emailSendingExecutionContext = emailSendingExecutionContext, absolutizer = absolutizer, diff --git a/app/services/EmailExportService.scala b/app/services/EmailExportService.scala index 57e44fe..cc4941d 100644 --- a/app/services/EmailExportService.scala +++ b/app/services/EmailExportService.scala @@ -1,21 +1,32 @@ package services import java.util.NoSuchElementException -import javax.inject.Named -import com.ysoft.odc.{SetDiff, Absolutizer} +import com.mohiva.play.silhouette.api.LoginInfo +import com.ysoft.odc.{Absolutizer, SetDiff} import controllers._ -import models.EmailMessageId -import play.api.libs.mailer.{MailerClient, Email} +import models.Change.Direction +import models.{Change, EmailMessageId} +import play.api.libs.mailer.{Email, MailerClient} import scala.concurrent.{ExecutionContext, Future} -class EmailExportService(from: String, nobodyInterestedContact: String, mailerClient: MailerClient, notificationService: VulnerabilityNotificationService, emailSendingExecutionContext: ExecutionContext, absolutizer: Absolutizer)(implicit executionContext: ExecutionContext) { + +object EmailExportType extends Enumeration { + val Vulnerabilities = Value("vulnerabilities") + val Digest = Value("digest") +} + +class EmailExportService(from: String, nobodyInterestedContact: String, val exportType: EmailExportType.Value, odcService: OdcService, mailerClient: MailerClient, notificationService: VulnerabilityNotificationService, emailSendingExecutionContext: ExecutionContext, absolutizer: Absolutizer)(implicit executionContext: ExecutionContext) { + // Maybe it is not the best place for exportType, but I am not sure if we want this to be configurable. If no, then we can get rid of it. If yes, we should refactor it. + + + private def getEmail(loginInfo: LoginInfo) = loginInfo.providerKey // TODO: get the email in a cleaner way def recipientsForProjects(projects: Set[ReportInfo]) = for{ recipients <- notificationService.getRecipientsForProjects(projects) } yield { - recipients.map(_.providerKey) match { // TODO: get the email in a cleaner way + recipients.map(getEmail) match { case Seq() => Seq(nobodyInterestedContact) -> false case other => other -> true } @@ -61,4 +72,47 @@ class EmailExportService(from: String, nobodyInterestedContact: String, mailerCl bodyText = Some(vulnerability.description + "\n\n" + s"More details: "+absolutizer.absolutize(routes.Statistics.vulnerability(vulnerability.name, None))) ) + + + def emailDigest(subscriber: LoginInfo, changes: Seq[Change], projects: ProjectsWithReports): Future[Email] = { + val vulnNames = changes.map(_.vulnerabilityName).toSet + for { + vulns <- Future.traverse(vulnNames.toSeq)(name => odcService.getVulnerabilityDetails(name).map(v => name -> v.get)).map(_.toMap) + groups = changes.groupBy(_.direction).withDefaultValue(Seq()) + } yield { + val changesMarks = Map(Direction.Added -> "❢", Direction.Removed -> "☑") + def vulnerabilityText(change: Change, vulnerability: Vulnerability) = ( + s"#### ${changesMarks(change.direction)} ${vulnerability.name}${vulnerability.cvssScore.fold("")(sev => s" (CVSS severity: $sev)")}" + +"\n"+vulnerability.description + +"\nmore info: "+absolutizer.absolutize(routes.Statistics.vulnerability(vulnerability.name, None)) + ) + def vulnChanges(changes: Seq[Change]) = + changes.map(c => c -> vulns(c.vulnerabilityName)) + .sortBy{case (change, vuln) => (vuln.cvssScore.map(-_), vuln.name)} + .map((vulnerabilityText _).tupled) + .mkString("\n\n") + def vulnerableProjects(projectIdToChanges: Map[String, Seq[Change]]) = + projectIdToChanges.toIndexedSeq.map{case (project, ch) => (projects.parseUnfriendlyNameGracefully(project), ch)} + .sortBy{case (ri, _) => friendlyProjectNameString(ri).toLowerCase} + .map{case (project, changes) => "### "+friendlyProjectNameString(project)+"\n"+vulnChanges(changes)} + .mkString("\n\n") + def section(title: String, direction: Direction) = { + groups(direction) match { + case Seq() => None + case list => Some("## "+title + "\n\n" + vulnerableProjects(list.groupBy(_.projectName))) + } + } + Email( + subject = s"New changes in vulnerabilities (${changes.size}: +${groups(Direction.Added).size} -${groups(Direction.Removed).size})", + to = Seq(getEmail(subscriber)), + from = from, + bodyText = Some(Seq( + section("Projects newly affected by a vulnerability", Direction.Added), + section("Projects no longer affected by a vulnerability", Direction.Removed) + ).flatten.mkString("\n\n")) + //bodyHtml = TODO + ) + } + } + } diff --git a/app/services/VulnerabilityNotificationService.scala b/app/services/VulnerabilityNotificationService.scala index 90070fb..e23eb55 100644 --- a/app/services/VulnerabilityNotificationService.scala +++ b/app/services/VulnerabilityNotificationService.scala @@ -6,7 +6,10 @@ import com.mohiva.play.silhouette.api.LoginInfo import com.ysoft.odc.SetDiff import controllers.{ProjectsWithReports, ReportInfo} import models._ +import models.tables._ +import play.api.Logger import play.api.db.slick.{DatabaseConfigProvider, HasDatabaseConfigProvider} +import slick.dbio.FutureAction import slick.jdbc.TransactionIsolation import scala.concurrent.{ExecutionContext, Future} @@ -30,8 +33,25 @@ class VulnerabilityNotificationService @Inject() (protected val dbConfigProvider import dbConfig.driver.api._ import models.tables._ + def subscribers = db.run(vulnerabilitySubscriptions.result).map(_.groupBy(_.user)) def watchedProjectsByUser(identity: LoginInfo) = db.run(vulnerabilitySubscriptions.filter(_.user === identity).result) - def subscribe(user: LoginInfo, project: String) = db.run(vulnerabilitySubscriptions += VulnerabilitySubscription(user = user, project = project)) + + def subscribe(user: LoginInfo, project: String) = db.run( + DBIO.seq( + ensureUserHasNotificationDigestStatus(user), + vulnerabilitySubscriptions += VulnerabilitySubscription(user = user, project = project) + ) + ) + + private def ensureUserHasNotificationDigestStatus(user: LoginInfo): DBIOAction[Unit, slick.dbio.NoStream, Effect.Read with Effect.Write] = ( + notificationDigestStatuses.filter(_.user === user).result.map(_.nonEmpty) flatMap { + case true => DBIO.seq() + case false => for{ + changelogTopIdOption <- changelogTopIdOptionQuery + (res: Int) <- notificationDigestStatuses += NotificationDigestStatus(user = user, lastChangelogIdOption = changelogTopIdOption) + } yield () + } + ).withTransactionIsolation(TransactionIsolation.Serializable) def unsubscribe(user: LoginInfo, project: String) = db.run(vulnerabilitySubscriptions.filter(vs => vs.user === user && vs.project === project).delete) @@ -85,7 +105,13 @@ class VulnerabilityNotificationService @Inject() (protected val dbConfigProvider def changeAffectedProjects(vulnerabilityName: String, affectedProjectsDiff: SetDiff[String]): Future[Unit] = { val time = DateTime.now() - def createRecord(projectName: String, direction: Change.Direction) = Change(time, vulnerabilityName, projectName, direction) + def createRecord(projectName: String, direction: Change.Direction) = Change( + time = time, + vulnerabilityName = vulnerabilityName, + projectName = projectName, + direction = direction, + notifiedToSomebody = false + ) val recordsToAdd = affectedProjectsDiff.added.map(projectName => createRecord(projectName, Change.Direction.Added)) ++ affectedProjectsDiff.removed.map(projectName => createRecord(projectName, Change.Direction.Removed)) /* @@ -98,6 +124,34 @@ class VulnerabilityNotificationService @Inject() (protected val dbConfigProvider ).map(_ => ())) } + private def changelogTopIdOptionQuery = changelog.map(_.id).max.result + + def sendDigestToSubscriber(subscriber: LoginInfo, subscriptions: Seq[VulnerabilitySubscription])(sendDigest: Seq[Change] => Future[Unit]): Future[Unit] = { + def subscriptionCondition(change: Changes, subscription: VulnerabilitySubscription): Rep[Boolean] = { + //noinspection ScalaUnnecessaryParentheses – because it looks less confusing + (change.projectName === subscription.project) || + (if (subscription.project contains '/') (false: Rep[Boolean]) else change.projectName.startsWith(subscription.project + "/")) + } + val projectCondition: Changes => Rep[Boolean] = (change) => + subscriptions.foldLeft(false: Rep[Boolean])((cond, subscription) => cond || subscriptionCondition(change, subscription)) + val notificationDigestStatusSelection = notificationDigestStatuses.filter(_.user === subscriber) + db.run( + ( + for{ + oldStatus <- notificationDigestStatusSelection.result.map(_.head) + _ = println(oldStatus.lastChangelogIdOption.fold(changelog.filter(_ => true: Rep[Boolean]))(lastChangelogId => changelog.filter(_.id > lastChangelogId)).filter(projectCondition).result.statements.mkString("\n")) + changelogSinceLastNotified <- oldStatus.lastChangelogIdOption.fold(changelog.filter(_ => true: Rep[Boolean]))(lastChangelogId => changelog.filter(_.id > lastChangelogId)).filter(projectCondition).result + changelogIds = changelogSinceLastNotified.map(_._1) + changelogTopIdOption <- changelogTopIdOptionQuery + newLastChangelogIdOption = changelogTopIdOption.orElse(oldStatus.lastChangelogIdOption) + (_: Unit) <- FutureAction(sendDigest(changelogSinceLastNotified.map(_._2))) // Yes, we are waiting for user to be notified over some slow I/O when having an open transaction + (_: Int) <- notificationDigestStatusSelection.map(_.lastChangelogId).update(newLastChangelogIdOption) + (_: Int) <- changelog.filter(_.id inSet changelogIds).map(_.notifiedToSomebody).update(true) + } yield () + ).withTransactionIsolation(TransactionIsolation.Serializable) + ) + } + val issueTrackerExport = new ExportPlatform(tables.issueTrackerExportTables) val mailExport = new ExportPlatform(tables.mailExportTables) diff --git a/build.sbt b/build.sbt index 5572732..be87075 100644 --- a/build.sbt +++ b/build.sbt @@ -8,6 +8,8 @@ scalaVersion := "2.11.7" resolvers += "Atlassian Releases" at "https://maven.atlassian.com/public/" +resolvers += Resolver.jcenterRepo + libraryDependencies ++= Seq( //jdbc, cache, @@ -71,7 +73,7 @@ libraryDependencies += "org.webjars.bower" % "jquery.scrollTo" % "2.1.2" libraryDependencies += "net.codingwell" %% "scala-guice" % "4.0.0" -libraryDependencies += "net.ceedubs" %% "ficus" % "1.1.2" +libraryDependencies += "com.iheart" %% "ficus" % "1.2.3" libraryDependencies += "org.owasp" % "dependency-check-core" % "1.3.0" diff --git a/conf/application.conf.-example b/conf/application.conf.-example index f0e7cbc..cb7c15a 100644 --- a/conf/application.conf.-example +++ b/conf/application.conf.-example @@ -51,6 +51,7 @@ yssdc{ email{ from = "info@example.com" noSubscriberContact = "foobar@example.com" + //optional: type = "digest" or type="vulnerabilities" (default); Digest is WIP. } } projects = {jobId:humanReadableName, …} diff --git a/conf/evolutions/default/8.sql b/conf/evolutions/default/8.sql new file mode 100644 index 0000000..ba296a5 --- /dev/null +++ b/conf/evolutions/default/8.sql @@ -0,0 +1,24 @@ +# --- !Ups +CREATE TABLE "notification_digest_status" ( + "user_provider_id" VARCHAR NOT NULL, + "user_provider_key" VARCHAR NOT NULL, + "last_changelog_id" INTEGER NULL +); +CREATE UNIQUE INDEX "notification_digest_status_user_idx" ON "notification_digest_status" ("user_provider_id","user_provider_key"); + +INSERT INTO notification_digest_status +(user_provider_id, user_provider_key, last_changelog_id) + SELECT + subscriber_provider_id AS user_provider_id, + subscriber_provider_key AS user_provider_key, + (SELECT MAX(id) from change) AS last_changelog_id + FROM vulnerability_subscription + GROUP BY subscriber_provider_id, subscriber_provider_key; + +ALTER TABLE change ADD COLUMN "notified_to_somebody" BOOLEAN NOT NULL DEFAULT FALSE; +UPDATE change SET notified_to_somebody = TRUE; + +# --- !Downs +drop table "notification_digest_status"; + +ALTER TABLE change DROP COLUMN "notified_to_somebody"; diff --git a/production.conf-example b/production.conf-example index 263f45b..9ce402e 100644 --- a/production.conf-example +++ b/production.conf-example @@ -49,6 +49,7 @@ yssdc{ email{ from = "info@example.com" noSubscriberContact = "foobar@example.com" + //optional: type = "digest" or type="vulnerabilities" (default); Digest is WIP. } } projects = {jobId:humanReadableName, …}