Added support for mail notifications and WIP JIRA export.

app/controllers/DependencyCheckReportsParser.scala

@@ -75,8 +75,8 @@ private final case class BadFilter(pattern: String) extends Filter{
 
 object DependencyCheckReportsParser{
   final case class ResultWithSelection(result: Result, projectsWithSelection: ProjectsWithSelection)
-  final case class Result(bareFlatReports: Map[String, Analysis], bareFailedAnalysises: Map[String, Throwable], projects: Projects){
-    lazy val projectsReportInfo = new ProjectsWithReports(projects, bareFlatReports.keySet ++ bareFailedAnalysises.keySet)
+  final case class Result(bareFlatReports: Map[String, Analysis], bareFailedAnalysises: Map[String, Throwable], projects: Projects /*TODO: maybe rename to rootProjects*/){
+    lazy val projectsReportInfo = new ProjectsWithReports(projects, bareFlatReports.keySet ++ bareFailedAnalysises.keySet) // TODO: consider renaming to projectsWithReports
     lazy val flatReports: Map[ReportInfo, Analysis] = bareFlatReports.map{case (k, v) => projectsReportInfo.reportIdToReportInfo(k) -> v}
     lazy val failedAnalysises: Map[ReportInfo, Throwable] = bareFailedAnalysises.map{case (k, v) => projectsReportInfo.reportIdToReportInfo(k) -> v}
     lazy val allDependencies = flatReports.toSeq.flatMap(r => r._2.dependencies.map(_ -> r._1))

app/controllers/Notifications.scala

@@ -0,0 +1,139 @@
+package controllers
+
+import javax.inject.Inject
+
+import com.ysoft.odc.{Absolutizer, SetDiff}
+import controllers.Statistics.LibDepStatistics
+import models.{EmailMessageId, ExportedVulnerability}
+import play.api.i18n.MessagesApi
+import play.api.libs.Crypto
+import play.api.mvc.Action
+import play.api.{Configuration, Logger}
+import services.{EmailExportService, IssueTrackerService, LibrariesService, VulnerabilityNotificationService}
+
+import scala.concurrent.Future.{successful => Fut}
+import scala.concurrent.{ExecutionContext, Future}
+
+class Notifications @Inject()(
+  config: Configuration,
+  librariesService: LibrariesService,
+  notificationService: VulnerabilityNotificationService,
+  projectReportsProvider: ProjectReportsProvider,
+  dependencyCheckReportsParser: DependencyCheckReportsParser,
+  issueTrackerServiceOption: Option[IssueTrackerService],
+  emailExportServiceOption: Option[EmailExportService],
+  absolutizer: Absolutizer,
+  val env: AuthEnv
+)(implicit val messagesApi: MessagesApi, executionContext: ExecutionContext) extends AuthenticatedController {
+
+  private val versions = Map[String, Int]()
+
+  import secureRequestConversion._
+
+  def listProjects() = SecuredAction.async { implicit req =>
+    val (lastRefreshTime, resultsFuture) = projectReportsProvider.resultsForVersions(versions)
+    val myWatchesFuture = notificationService.watchedProjectsByUser(req.identity.loginInfo).map(_.map(_.project).toSet)
+    for{
+      (successfulReports, failedReports) <- resultsFuture
+      myWatches <- myWatchesFuture
+    } yield {
+      val projects = dependencyCheckReportsParser.parseReports(successfulReports).projectsReportInfo.sortedReportsInfo
+      Ok(views.html.notifications.index(projects, myWatches))
+    }
+  }
+
+  //@inline private def filterMissingTickets(missingTickets: Set[String]) = missingTickets take 1 // for debug purposes
+  @inline private def filterMissingTickets(missingTickets: Set[String]) = missingTickets // for production purposes
+
+  def notifyVulnerabilities[T](
+    lds: LibDepStatistics, ep: notificationService.ExportPlatform[T, _], projects: ProjectsWithReports
+  )(
+    reportVulnerability: (Vulnerability, Set[GroupedDependency]) => Future[ExportedVulnerability[T]]
+  )(
+    reportChangedProjectsForVulnerability: (Vulnerability, SetDiff[String], T) => Future[Unit]
+  ) = {
+    val vulnerabilitiesByName = lds.vulnerabilitiesToDependencies.map{case (v, deps) => (v.name, (v, deps))}
+    for{
+      tickets <- ep.ticketsForVulnerabilities(lds.vulnerabilityNames)
+      // Check existing tickets
+      existingTicketsIds = tickets.values.map(_._1).toSet
+      ticketsById = tickets.values.map{case (id, ev) => id -> ev}.toMap
+      existingTicketsProjects <- ep.projectsForTickets(existingTicketsIds)
+      _ = Logger.warn("existingTicketsProjects for "+ep+": "+existingTicketsProjects.filter(_._2.exists(_.toLowerCase.contains("wps"))).toString)
+      projectUpdates <- Future.traverse(existingTicketsIds){ ticketId =>  // If we traversed over existingTicketsProjects, we would skip vulns with no projects
+        val oldProjectIdsSet = existingTicketsProjects(ticketId)
+        val exportedVulnerability = ticketsById(ticketId)
+        val vulnerabilityName = exportedVulnerability.vulnerabilityName
+        val newProjectIdsSet = vulnerabilitiesByName(vulnerabilityName)._2.flatMap(_.projects).map(_.fullId)
+        val diff = new SetDiff(oldSet = oldProjectIdsSet, newSet = newProjectIdsSet)
+        if(diff.nonEmpty) {
+          reportChangedProjectsForVulnerability(lds.vulnerabilitiesByName(vulnerabilityName), diff, exportedVulnerability.ticket).flatMap { _ =>
+            ep.changeProjects(ticketId, diff, projects)
+          }.map( _ => Some(diff))
+        } else {
+          Fut(None)
+        }
+      }
+      // Check new tickets
+      missingTickets = vulnerabilitiesByName.keySet -- tickets.keySet
+      newTicketIds <- Future.traverse(filterMissingTickets(missingTickets)){vulnerabilityName =>
+        val (vulnerability, dependencies) = vulnerabilitiesByName(vulnerabilityName)
+        reportVulnerability(vulnerability, dependencies).flatMap{ ticket =>
+          ep.addTicket(ticket, dependencies.flatMap(_.projects)).map(_ => ticket.ticket)
+        }
+      }
+    } yield (missingTickets, newTicketIds, projectUpdates.toSet: Set[Any])
+  }
+
+  def cron(key: String, purgeCache: Boolean) = Action.async{
+    if(Crypto.constantTimeEquals(key, config.getString("yssdc.cronKey").get)){
+      if(purgeCache){
+        projectReportsProvider.purgeCache(Map())
+      }
+      val (lastRefreshTime, resultsFuture) = projectReportsProvider.resultsForVersions(versions)
+      for{
+        (successfulReports, failedReports) <- resultsFuture
+        libraries <- librariesService.all
+        parsedReports = dependencyCheckReportsParser.parseReports(successfulReports)
+        lds = LibDepStatistics(dependencies = parsedReports.groupedDependencies.toSet, libraries = libraries.toSet)
+        issuesExportResultFuture = exportToIssueTracker(lds, parsedReports.projectsReportInfo)
+        mailExportResultFuture = exportToEmail(lds, parsedReports.projectsReportInfo)
+        (missingTickets, newTicketIds, updatedTickets) <- issuesExportResultFuture
+        (missingEmails, newMessageIds, updatedEmails) <- mailExportResultFuture
+      } yield Ok(
+        missingTickets.mkString("\n")+"\n\n"+newTicketIds.mkString("\n")+ updatedTickets.toString+
+          "\n\n" +
+          missingEmails.mkString("\n")+"\n\n"+newMessageIds.mkString("\n") + updatedEmails.toString
+      )
+    }else{
+      Fut(Unauthorized("unauthorized"))
+    }
+  }
+
+  private def forService[S, T](serviceOption: Option[S])(f: S => Future[(Set[String], Set[T], Set[Any])]) = serviceOption.fold(Fut((Set[String](), Set[T](), Set[Any]())))(f)
+
+  private def exportToEmail(lds: LibDepStatistics, p: ProjectsWithReports) = forService(emailExportServiceOption){ emailExportService =>
+    notifyVulnerabilities[EmailMessageId](lds, notificationService.mailExport, p) { (vulnerability, dependencies) =>
+      emailExportService.mailForVulnerability(vulnerability, dependencies).flatMap(emailExportService.sendEmail).map(id => ExportedVulnerability(vulnerability.name, EmailMessageId(id), 0))
+    }{ (vuln, diff, msgid) =>
+      emailExportService.mailForVulnerabilityProjectsChange(vuln, msgid, diff, p).flatMap(emailExportService.sendEmail).map(_ => ())
+    }
+  }
+
+  private def exportToIssueTracker(lds: LibDepStatistics, p: ProjectsWithReports) = forService(issueTrackerServiceOption){ issueTrackerService =>
+    notifyVulnerabilities[String](lds, notificationService.issueTrackerExport, p) { (vulnerability, dependencies) =>
+      issueTrackerService.reportVulnerability(vulnerability)
+    }{ (vuln, diff, ticket) =>
+      Fut(())
+    }
+  }
+
+  def watch(project: String) = SecuredAction.async{ implicit req =>
+    for( _ <-notificationService.subscribe(req.identity.loginInfo, project) ) yield Redirect(routes.Notifications.listProjects())
+  }
+
+  def unwatch(project: String) = SecuredAction.async{implicit req =>
+    for( _ <-notificationService.unsubscribe(req.identity.loginInfo, project) ) yield Redirect(routes.Notifications.listProjects())
+  }
+
+}

app/controllers/ProjectsWithReports.scala

@@ -22,6 +22,8 @@ final case class ReportInfo(
 
   override def hashCode(): Int = 517+fullId.hashCode
 
+  def bare = copy(subprojectNameOption = None, fullId = fullId.takeWhile(_ != '/'))
+
 }
 
 object ProjectsWithReports{
@@ -38,21 +40,27 @@ class ProjectsWithReports (val projects: Projects, val reports: Set[String]) {
 
   val reportIdToReportInfo = {
     val reportsMap = reports.map{ unfriendlyName =>
-      val (baseName, theRest) = unfriendlyName.span(_ != '/')
-      val removeLeadingMess = RestMessBeginRegexp.replaceAllIn(_: String, "")
-      val removeTrailingMess = RestMessEndRegexp.replaceAllIn(_: String, "")
-      val removeMess = removeLeadingMess andThen removeTrailingMess
-      val subProjectOption = Some(removeMess(theRest)).filter(_ != "")
-      unfriendlyName -> ReportInfo(
-        projectId = baseName,
-        fullId = unfriendlyName,
-        projectName = projects.projectMap(baseName),
-        subprojectNameOption = subProjectOption.orElse(Some("root project"))
-      )
+      unfriendlyName -> parseUnfriendlyName(unfriendlyName)
     }.toMap
     reportsMap ++ reportsMap.values.map(r => r.projectId -> ReportInfo(projectId = r.projectId, fullId = r.projectId, subprojectNameOption = None, projectName = r.projectName))
   }
 
+  def parseUnfriendlyName(unfriendlyName: String): ReportInfo = {
+    val (baseName, theRest) = unfriendlyName.span(_ != '/')
+    val removeLeadingMess = RestMessBeginRegexp.replaceAllIn(_: String, "")
+    val removeTrailingMess = RestMessEndRegexp.replaceAllIn(_: String, "")
+    val removeMess = removeLeadingMess andThen removeTrailingMess
+    val subProjectOption = Some(removeMess(theRest)).filter(_ != "")
+    ReportInfo(
+      projectId = baseName,
+      fullId = unfriendlyName,
+      projectName = projects.projectMap(baseName),
+      subprojectNameOption = subProjectOption.orElse(Some("root project"))
+    )
+  }
+
   val ungroupedReportsInfo = reportIdToReportInfo.values.toSet
 
+  def sortedReportsInfo = ungroupedReportsInfo.toSeq.sortBy(p => p.projectName -> p.projectId -> p.subprojectNameOption)
+
 }

app/controllers/Statistics.scala

@@ -8,7 +8,7 @@ import models.{Library, LibraryTag}
 import org.joda.time.DateTime
 import play.api.i18n.MessagesApi
 import play.twirl.api.Txt
-import services.{LibrariesService, LibraryTagAssignmentsService, OdcService, TagsService}
+import services._
 import views.html.DefaultRequest
 
 import scala.concurrent.{ExecutionContext, Future}
@@ -17,6 +17,8 @@ object Statistics{
   case class LibDepStatistics(libraries: Set[(Int, Library)], dependencies: Set[GroupedDependency]){
     def vulnerableRatio = vulnerableDependencies.size.toDouble / dependencies.size.toDouble
     lazy val vulnerabilities: Set[Vulnerability] = dependencies.flatMap(_.vulnerabilities)
+    lazy val vulnerabilitiesByName = vulnerabilities.map(v => v.name -> v).toMap
+    lazy val vulnerabilityNames = vulnerabilities.map(_.name)
     lazy val vulnerabilitiesToDependencies: Map[Vulnerability, Set[GroupedDependency]] = vulnerableDependencies.flatMap(dep =>
       dep.vulnerabilities.map(vuln => (vuln, dep))
     ).groupBy(_._1).mapValues(_.map(_._2)).map(identity)
@@ -50,6 +52,8 @@ class Statistics @Inject() (
   libraryTagAssignmentsService: LibraryTagAssignmentsService,
   @Named("missing-GAV-exclusions") missingGAVExclusions: MissingGavExclusions,
   projects: Projects,
+  vulnerabilityNotificationService: VulnerabilityNotificationService,
+  issueTrackerServiceOption: Option[IssueTrackerService],
   val env: AuthEnv
 )(implicit val messagesApi: MessagesApi, executionContext: ExecutionContext) extends AuthenticatedController {
 
@@ -167,14 +171,19 @@ class Statistics @Inject() (
           name = name,
           projectsWithSelection = selection.projectsWithSelection
         )))){ case (vuln, vulnerableDependencies) =>
-          for(
+          for {
             plainLibs <- librariesService.byPlainLibraryIdentifiers(vulnerableDependencies.flatMap(_.plainLibraryIdentifiers)).map(_.keySet)
-          ) yield Ok(views.html.statistics.vulnerability(
+            ticketOption <- vulnerabilityNotificationService.issueTrackerExport.ticketForVulnerability(name)
+          } yield Ok(views.html.statistics.vulnerability(
             vulnerability = vuln,
             affectedProjects = vulnerableDependencies.flatMap(dep => dep.projects.map(proj => (proj, dep))).groupBy(_._1).mapValues(_.map(_._2)),
             vulnerableDependencies = vulnerableDependencies,
             affectedLibraries = plainLibs,
-            projectsWithSelection = selection.projectsWithSelection
+            projectsWithSelection = selection.projectsWithSelection,
+            issueOption = for{
+              ticket <- ticketOption
+              issueTrackerService <- issueTrackerServiceOption
+            } yield ticket -> issueTrackerService.ticketLink(ticket)
           ))
         }