From b1f04c3987981a7467f3287cbf7cd3c75ed93aae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0est=C3=A1k=20V=C3=ADt?= Date: Wed, 9 Mar 2016 09:55:00 +0100 Subject: [PATCH] Added support for changelog --- app/com/ysoft/odc/SetDiff.scala | 6 +++ app/controllers/Notifications.scala | 19 +++++++ app/models/Change.scala | 38 ++++++++++++++ app/models/package.scala | 21 +++++++- .../VulnerabilityNotificationService.scala | 51 ++++++++++++++++--- conf/evolutions/default/7.sql | 12 +++++ 6 files changed, 139 insertions(+), 8 deletions(-) create mode 100644 app/models/Change.scala create mode 100644 conf/evolutions/default/7.sql diff --git a/app/com/ysoft/odc/SetDiff.scala b/app/com/ysoft/odc/SetDiff.scala index 8b7c2d2..8e73386 100644 --- a/app/com/ysoft/odc/SetDiff.scala +++ b/app/com/ysoft/odc/SetDiff.scala @@ -5,4 +5,10 @@ class SetDiff[T](val oldSet: Set[T], val newSet: Set[T]) { lazy val removed = oldSet -- newSet lazy val isEmpty = newSet == oldSet def nonEmpty = !isEmpty + + def map[U](f: T => U): SetDiff[U] = new SetDiff[U]( + oldSet = oldSet.map(f), + newSet = newSet.map(f) + ) + } diff --git a/app/controllers/Notifications.scala b/app/controllers/Notifications.scala index 3e97346..7d57e5a 100644 --- a/app/controllers/Notifications.scala +++ b/app/controllers/Notifications.scala @@ -104,9 +104,12 @@ class Notifications @Inject()( parsedReports = dependencyCheckReportsParser.parseReports(successfulReports) lds = LibDepStatistics(dependencies = parsedReports.groupedDependencies.toSet, libraries = libraries.toSet) issuesExportResultFuture = exportToIssueTracker(lds, parsedReports.projectsReportInfo) + diffDbExportResultFuture = exportToDiffDb(lds, parsedReports.projectsReportInfo) + //mailExportResultFuture = diffDbExportResultFuture.flatMap(_ => exportToEmailDigest(lds, parsedReports.projectsReportInfo)) mailExportResultFuture = exportToEmail(lds, parsedReports.projectsReportInfo) (missingTickets, newTicketIds, updatedTickets) <- issuesExportResultFuture (missingEmails, newMessageIds, updatedEmails) <- mailExportResultFuture + (missingVulns, newVulnIds, updatedVulns) <- diffDbExportResultFuture } yield Ok( missingTickets.mkString("\n") + "\n\n" + newTicketIds.mkString("\n") + updatedTickets.toString + "\n\n" + @@ -130,6 +133,7 @@ class Notifications @Inject()( } } + // FIXME: In case of crash during export, one change might be exported multiple times. This can't be solved in e-mail exports, but it might be solved in issueTracker and diffDb exports. private def exportToIssueTracker(lds: LibDepStatistics, p: ProjectsWithReports) = forService(issueTrackerServiceOption){ issueTrackerService => notifyVulnerabilities[String](lds, notificationService.issueTrackerExport, p) { (vulnerability, dependencies) => issueTrackerService.reportVulnerability(vulnerability) @@ -138,6 +142,21 @@ class Notifications @Inject()( } } + private def exportToDiffDb(lds: LibDepStatistics, p: ProjectsWithReports) = { + notifyVulnerabilities[String](lds, notificationService.diffDbExport, p){ (vulnerability, dependencies) => + //?save_new_vulnerability + val affectedProjects = dependencies.flatMap(_.projects) + val diff = new SetDiff(Set(), affectedProjects) + notificationService.changeAffectedProjects(vulnerability.name, diff.map(_.fullId)).map{_ => + ExportedVulnerability[String](vulnerabilityName = vulnerability.name, ticket = vulnerability.name, ticketFormatVersion = 0) + } + }{ (vulnerability, diff, id) => + notificationService.changeAffectedProjects(vulnerability.name, diff).map{_ => + () + } + } + } + // Redirection to a specific position does not look intuituve now, so it has been disabled for now. private def redirectToProject(project: String)(implicit th: DefaultRequest) = Redirect(routes.Notifications.listProjects()/*.withFragment("project-" + URLEncoder.encode(project, "utf-8")).absoluteURL()*/) diff --git a/app/models/Change.scala b/app/models/Change.scala new file mode 100644 index 0000000..d557112 --- /dev/null +++ b/app/models/Change.scala @@ -0,0 +1,38 @@ +package models + +import java.time.LocalTime + +import models.profile.MappedJdbcType +import models.profile.api._ +import models.jodaSupport._ +import models.profile.api._ +import org.joda.time.{DateTime, LocalDate} +import play.api.data.Form +import slick.lifted.{ProvenShape, Tag} + + +object Change { + abstract sealed class Direction private[Change] (private[Change] val description: String) + object Direction{ + object Added extends Direction("added") + object Removed extends Direction("removed") + val All = Set(Added, Removed) + val ByName = All.map(x => x.description -> x).toMap + implicit val TypeMapper = MappedJdbcType.base[Direction, String](_.description, ByName) + } + +} + +case class Change (time: DateTime, vulnerabilityName: String, projectName: String, direction: Change.Direction) + +class Changes(tag: Tag) extends Table[(Int, Change)](tag, "change"){ + def id = column[Int]("id", O.PrimaryKey, O.AutoInc) + import Change.Direction.TypeMapper + def time = column[DateTime]("time") + def vulnerabilityName = column[String]("vulnerability_name") + def projectName = column[String]("project_name") + def direction = column[Change.Direction]("direction") + + def base = (time, vulnerabilityName, projectName, direction) <> ((Change.apply _).tupled, Change.unapply) + override def * = (id, base) +} \ No newline at end of file diff --git a/app/models/package.scala b/app/models/package.scala index 4a0f2d9..72ef896 100644 --- a/app/models/package.scala +++ b/app/models/package.scala @@ -1,6 +1,8 @@ import java.nio.file.{Paths, Files} +import slick.lifted.MappedProjection + import scala.language.reflectiveCalls package object models { @@ -18,6 +20,7 @@ package object models { val snoozesTable = TableQuery[Snoozes] val authTokens = TableQuery[CookieAuthenticators] val vulnerabilitySubscriptions = TableQuery[VulnerabilitySubscriptions] + val changelog = TableQuery[Changes] val issueTrackerExportTables = new ExportPlatformTables[String, (String, String, Int)](){ val tableNamePart = "issue_tracker" @@ -44,10 +47,24 @@ package object models { override val tickets = TableQuery[EmailExportedVulnerabilities] } + val diffDbExportTables = new ExportPlatformTables[String, (String, Int)] { + val tableNamePart = "diff_db" + class DiffDbVulnerabilities(tag: Tag) extends ExportedVulnerabilities[String, (String, Int)](tag, tableNamePart){ + override def base: MappedProjection[ExportedVulnerability[String], (String, Int)] = (vulnerabilityName, ticketFormatVersion) <> ( + ((n: String, v: Int) => ExportedVulnerability[String](n, n, v)).tupled, + obj => ExportedVulnerability.unapply[String](obj).map{case (n, _, v) => (n, v)} + ) + } + class DiffDbVulnerabilityProject(tag: Tag) extends ExportedVulnerabilityProjects(tag, tableNamePart) + + override val projects = TableQuery[DiffDbVulnerabilityProject] + override val tickets = TableQuery[DiffDbVulnerabilities] + } + /*{ import profile.SchemaDescription val schema = Seq[Any{def schema: SchemaDescription}]( - vulnerabilitySubscriptions, issueTrackerExportTables, mailExportTables + diffDbExportTables, changelog ).map(_.schema).foldLeft(profile.DDL(Seq(), Seq()))(_ ++ _) val sql = Seq( @@ -58,7 +75,7 @@ package object models { schema.dropStatements.toSeq.map(_+";").mkString("\n").dropWhile(_ == "\n"), "\n" ).mkString("\n") - Files.write(Paths.get("conf/evolutions/default/6.sql"), sql.getBytes("utf-8")) + Files.write(Paths.get("conf/evolutions/default/7.sql"), sql.getBytes("utf-8")) }*/ } diff --git a/app/services/VulnerabilityNotificationService.scala b/app/services/VulnerabilityNotificationService.scala index abbe4ee..90070fb 100644 --- a/app/services/VulnerabilityNotificationService.scala +++ b/app/services/VulnerabilityNotificationService.scala @@ -1,20 +1,34 @@ package services +import _root_.org.joda.time.DateTime import com.google.inject.Inject import com.mohiva.play.silhouette.api.LoginInfo import com.ysoft.odc.SetDiff import controllers.{ProjectsWithReports, ReportInfo} import models._ -import play.api.db.slick.{HasDatabaseConfigProvider, DatabaseConfigProvider} +import play.api.db.slick.{DatabaseConfigProvider, HasDatabaseConfigProvider} +import slick.jdbc.TransactionIsolation -import scala.collection.immutable.Iterable -import scala.concurrent.{Future, ExecutionContext} +import scala.concurrent.{ExecutionContext, Future} + +final class SingleFutureExecutionThrottler() (implicit executionContext: ExecutionContext){ + private var nextFuture: Future[_] = Future.successful(null) + + def throttle[T](f: => Future[T]): Future[T] = synchronized{ + val newFuture = nextFuture.recover{ case _ => null}.flatMap(_ => f) + nextFuture = newFuture + newFuture + } + +} + +final class NoThrottler() (implicit executionContext: ExecutionContext){ + def throttle[T](f: => Future[T]): Future[T] = f +} class VulnerabilityNotificationService @Inject() (protected val dbConfigProvider: DatabaseConfigProvider)(implicit executionContext: ExecutionContext) extends HasDatabaseConfigProvider[models.profile.type]{ import dbConfig.driver.api._ - - import models.tables - import models.tables.vulnerabilitySubscriptions + import models.tables._ def watchedProjectsByUser(identity: LoginInfo) = db.run(vulnerabilitySubscriptions.filter(_.user === identity).result) def subscribe(user: LoginInfo, project: String) = db.run(vulnerabilitySubscriptions += VulnerabilitySubscription(user = user, project = project)) @@ -61,8 +75,33 @@ class VulnerabilityNotificationService @Inject() (protected val dbConfigProvider } + /** + * The changelogThrottler is a temporary hack than prevents some congestion that seems to occur in HikariCP or maybe in Slick. + * It prevents exceptions like “java.sql.SQLException: Timeout of 1001ms encountered waiting for connection”. + * It is probably prevented at a wrong level, but it works :) + */ + private val changelogThrottler = new SingleFutureExecutionThrottler() + // private val changelogThrottler = new NoThrottler() + + def changeAffectedProjects(vulnerabilityName: String, affectedProjectsDiff: SetDiff[String]): Future[Unit] = { + val time = DateTime.now() + def createRecord(projectName: String, direction: Change.Direction) = Change(time, vulnerabilityName, projectName, direction) + val recordsToAdd = affectedProjectsDiff.added.map(projectName => createRecord(projectName, Change.Direction.Added)) ++ + affectedProjectsDiff.removed.map(projectName => createRecord(projectName, Change.Direction.Removed)) + /* + Transaction: + It is essential to ensure that records in changelog appear in order. Low level of isolation might be even worse than running outside of transaction. + In longer term, it should be wrapped with transaction *with a proper isolation level* together with the export status modification. + */ + changelogThrottler.throttle(db.run( + (changelog.map(_.base) ++= recordsToAdd).withTransactionIsolation(TransactionIsolation.Serializable) + ).map(_ => ())) + } + val issueTrackerExport = new ExportPlatform(tables.issueTrackerExportTables) val mailExport = new ExportPlatform(tables.mailExportTables) + val diffDbExport = new ExportPlatform(tables.diffDbExportTables) + } diff --git a/conf/evolutions/default/7.sql b/conf/evolutions/default/7.sql new file mode 100644 index 0000000..c81a84b --- /dev/null +++ b/conf/evolutions/default/7.sql @@ -0,0 +1,12 @@ +# --- !Ups +create table "exported_diff_db_vulnerabilities" ("id" SERIAL NOT NULL PRIMARY KEY,"vulnerability_name" VARCHAR NOT NULL,"ticket_format_version" INTEGER NOT NULL); +create unique index "idx_exported_diff_db_vulnerabilities_vulnerabilityName" on "exported_diff_db_vulnerabilities" ("vulnerability_name"); +create table "exported_diff_db_vulnerability_projects" ("exported_vulnerability_id" INTEGER NOT NULL,"full_project_id" VARCHAR NOT NULL); +create unique index "idx_exported_diff_db_vulnerability_projects_all" on "exported_diff_db_vulnerability_projects" ("exported_vulnerability_id","full_project_id"); +create table "change" ("id" SERIAL NOT NULL PRIMARY KEY,"time" TIMESTAMP NOT NULL,"vulnerability_name" VARCHAR NOT NULL,"project_name" VARCHAR NOT NULL,"direction" VARCHAR NOT NULL); + +# --- !Downs +drop table "change"; +drop table "exported_diff_db_vulnerability_projects"; +drop table "exported_diff_db_vulnerabilities"; +