Better JIRA export

app/controllers/Notifications.scala

@@ -64,7 +64,6 @@ class Notifications @Inject()(
       existingTicketsIds = tickets.values.map(_._1).toSet
       ticketsById = tickets.values.map{case (id, ev) => id -> ev}.toMap
       existingTicketsProjects <- ep.projectsForTickets(existingTicketsIds)
-      _ = Logger.warn("existingTicketsProjects for "+ep+": "+existingTicketsProjects.filter(_._2.exists(_.toLowerCase.contains("wps"))).toString)
       projectUpdates <- Future.traverse(existingTicketsIds){ ticketId =>  // If we traversed over existingTicketsProjects, we would skip vulns with no projects
         val oldProjectIdsSet = existingTicketsProjects(ticketId)
         val exportedVulnerability = ticketsById(ticketId)
@@ -175,10 +174,13 @@ class Notifications @Inject()(
   // FIXME: In case of crash during export, one change might be exported multiple times. This can't be solved in e-mail exports, but it might be solved in issueTracker and diffDb exports.
   private def exportToIssueTracker(lds: LibDepStatistics, failedProjects: FailedProjects, p: ProjectsWithReports) = forService(issueTrackerServiceOption){ issueTrackerService =>
     notifyVulnerabilities[String](lds, failedProjects, notificationService.issueTrackerExport, p) { (vulnerability, dependencies) =>
-      issueTrackerService.reportVulnerability(vulnerability)
+      issueTrackerService.reportVulnerability(vulnerability, dependencies.flatMap{_.projects})
     }{ (vuln, diff, ticket) =>
-      Fut(())
+      issueTrackerService.updateVulnerability(vuln, diff.map(p.parseUnfriendlyNameGracefully), ticket)
-    }
+    }/*.flatMap{ v => <- Maybe this approach of migrating is completely wrong, because the issue tracker does not have access to the export DB.
+      // Perform the migration after main operations, propagate exceptions, but don't change the resulting value
+      issueTrackerService.migrateOldIssues().map((_: Unit) => v)
+    }*/
   }
 
   private def exportToDiffDb(lds: LibDepStatistics, failedProjects: FailedProjects, p: ProjectsWithReports) = {

app/modules/IssueTrackerExportModule.scala

@@ -1,17 +1,42 @@
 package modules
 
 import com.google.inject.{AbstractModule, Provides}
+import com.typesafe.config.{Config, ConfigObject, ConfigValue}
 import com.ysoft.odc.{Absolutizer, CredentialsAtlassianAuthentication}
 import net.ceedubs.ficus.Ficus._
 import net.ceedubs.ficus.readers.ArbitraryTypeReader._
+import net.ceedubs.ficus.readers.ValueReader
 import net.codingwell.scalaguice.ScalaModule
 import play.api.Configuration
+import play.api.libs.json._
 import play.api.libs.ws.WSClient
 import services.{IssueTrackerService, JiraIssueTrackerService}
 
 import scala.concurrent.ExecutionContext
 
 class IssueTrackerExportModule extends AbstractModule with ScalaModule{
+
+  private implicit object JsonValueReader extends ValueReader[JsObject] {
+
+    implicit def me = this
+
+    import scala.collection.JavaConversions._
+
+    private def extractJson(value: ConfigValue): JsValue = value match {
+      case co: ConfigObject => extractJsonFromObject(co)
+      case cv: ConfigValue => cv.unwrapped() match {
+        case s: String => JsString(s)
+        case i: java.lang.Integer => JsNumber(BigDecimal(i))
+        case b: java.lang.Boolean => JsBoolean(b)
+        //case b: List (ConfigList) => JsArray(b)
+      }
+    }
+
+    private def extractJsonFromObject(co: ConfigObject): JsObject = JsObject(co.keySet().map{ key => key -> extractJson(co.get(key))}.toMap)
+
+    override def read(config: Config, path: String): JsObject = extractJsonFromObject(config.getObject(path))
+  }
+
   override def configure(): Unit = {
   }
 
@@ -26,12 +51,17 @@ class IssueTrackerExportModule extends AbstractModule with ScalaModule{
         case Some("credentials") =>
         case other => sys.error("unknown authentication type: "+other)
       }
+      val fields = conf.underlying.as[Option[JiraIssueTrackerService.Fields]]("fields").getOrElse(JiraIssueTrackerService.NoFields)
       new JiraIssueTrackerService(
         absolutizer = absolutizer,
         server = conf.underlying.as[String]("server"),
         projectId = conf.underlying.as[Int]("projectId"),
         vulnerabilityIssueType = conf.underlying.as[Int]("vulnerabilityIssueType"),
-        atlassianAuthentication = conf.underlying.as[CredentialsAtlassianAuthentication]("authentication")
+        atlassianAuthentication = conf.underlying.as[CredentialsAtlassianAuthentication]("authentication"),
+        newProjectAddedTransitionNameOption = conf.underlying.as[Option[String]]("newProjectAddedTransitionName"),
+        noRelevantProjectAffectedTransitionNameOption = conf.underlying.as[Option[String]]("noRelevantProjectAffectedTransitionName"),
+        ticketFormatVersion = conf.underlying.as[Option[Int]]("ticketFormatVersion").getOrElse(1),
+        fields = fields
       )
     case other => sys.error("unknown provider for issue tracker: "+other)
   }

app/services/IssueTrackerService.scala

@@ -1,12 +1,15 @@
 package services
 
-import controllers.Vulnerability
+import com.ysoft.odc.SetDiff
+import controllers.{ReportInfo, Vulnerability}
 import models.ExportedVulnerability
 
 import scala.concurrent.Future
 
 trait IssueTrackerService {
-  def reportVulnerability(vulnerability: Vulnerability): Future[ExportedVulnerability[String]]
+  def reportVulnerability(vulnerability: Vulnerability, projects: Set[ReportInfo]): Future[ExportedVulnerability[String]]
   def ticketLink(ticket: String): String
   def ticketLink(ticket: ExportedVulnerability[String]): String = ticketLink(ticket.ticket)
+  def updateVulnerability(vuln: Vulnerability, diff: SetDiff[ReportInfo], ticket: String): Future[Unit]
+  //def migrateOldIssues(): Future[Unit]
 }

app/services/JiraIssueTrackerService.scala

@@ -3,37 +3,91 @@ package services
 import javax.inject.Inject
 
 import com.google.inject.name.Named
-import com.ysoft.odc.{Absolutizer, AtlassianAuthentication}
+import com.ysoft.odc.{Absolutizer, AtlassianAuthentication, SetDiff}
-import controllers.{Vulnerability, routes}
+import controllers.{ReportInfo, Vulnerability, friendlyProjectNameString, routes}
 import models.ExportedVulnerability
 import play.api.libs.json.Json.JsValueWrapper
-import play.api.libs.json.{JsObject, Json}
+import play.api.libs.json._
 import play.api.libs.ws.{WS, WSClient}
+import services.JiraIssueTrackerService.Fields
 
 import scala.concurrent.{ExecutionContext, Future}
 
 private case class JiraNewIssueResponse(id: String, key: String, self: String)
 
+private case class Transition(id: String/* heh, id is a numeric String */, name: String/*to: …*/)
+
+private case class Transitions(expand: String, transitions: Seq[Transition])
+
+object JiraIssueTrackerService {
+
+  final case class Fields(
+    cweId: Option[String],
+    linkId: Option[String],
+    severityId: Option[String],
+    projectsId: Option[String],
+    /*
+    teamsId: Option[String],
+    librariesId: Option[String],
+    */
+    constantFields: JsObject
+  )
+
+  val NoFields = Fields(cweId = None, linkId = None, severityId = None, projectsId = None, constantFields = JsObject(Seq()))
+
+}
+
 /**
   * status: WIP
   * It basically works, but there is much to be discussed and implemented.
   */
-class JiraIssueTrackerService @Inject() (absolutizer: Absolutizer, @Named("jira-server") server: String, @Named("jira-project-id") projectId: Int, @Named("jira-vulnerability-issue-type") vulnerabilityIssueType: Int, @Named("jira-authentication") atlassianAuthentication: AtlassianAuthentication)(implicit executionContext: ExecutionContext, wSClient: WSClient) extends IssueTrackerService{
+class JiraIssueTrackerService @Inject()(absolutizer: Absolutizer, @Named("jira-server") server: String, noRelevantProjectAffectedTransitionNameOption: Option[String], newProjectAddedTransitionNameOption: Option[String], fields: Fields, @Named("jira-project-id") projectId: Int, @Named("jira-vulnerability-issue-type") vulnerabilityIssueType: Int, ticketFormatVersion: Int, @Named("jira-authentication") atlassianAuthentication: AtlassianAuthentication)(implicit executionContext: ExecutionContext, wSClient: WSClient) extends IssueTrackerService{
   private def jiraUrl(url: String) = atlassianAuthentication.addAuth(WS.clientUrl(url))
+  private def api(endpoint: String) = jiraUrl(server+"/rest/api/2/"+endpoint)
 
-  private val formatVersion = 1
+  private implicit val TransitionFormats = Json.format[Transition]
+  private implicit val TransitionsFormats = Json.format[Transitions]
 
-  override def reportVulnerability(vulnerability: Vulnerability): Future[ExportedVulnerability[String]] = jiraUrl(server+"/rest/api/2/issue").post(Json.obj(
+  override def reportVulnerability(vulnerability: Vulnerability, projects: Set[ReportInfo]): Future[ExportedVulnerability[String]] = api("issue").post(Json.obj(
-    "fields" -> (extractInitialFields(vulnerability) ++ extractManagedFields(vulnerability))
+    "fields" -> (extractInitialFields(vulnerability) ++ extractManagedFields(vulnerability, projects))
   )).map(response => // returns responses like {"id":"1234","key":"PROJ-6","self":"https://…/rest/api/2/issue/1234"}
     try{
       val issueInfo = Json.reads[JiraNewIssueResponse].reads(response.json).get
-      ExportedVulnerability(vulnerabilityName = vulnerability.name, ticket = issueInfo.key, ticketFormatVersion = formatVersion)
+      ExportedVulnerability(vulnerabilityName = vulnerability.name, ticket = issueInfo.key, ticketFormatVersion = ticketFormatVersion)
     }catch{
       case e:Throwable=>sys.error("bad data: "+response.body)
     }
   )
 
+  override def updateVulnerability(vuln: Vulnerability, diff: SetDiff[ReportInfo], ticket: String): Future[Unit] = {
+    val requiredTransitionOption = diff.whichNonEmpty match {
+      case SetDiff.Selection.Old => noRelevantProjectAffectedTransitionNameOption
+      case SetDiff.Selection.New | SetDiff.Selection.Both => newProjectAddedTransitionNameOption
+      case SetDiff.Selection.None => sys.error("this should not happpen")
+    }
+    val transitionOptionFuture = requiredTransitionOption.map{ requiredTransition =>
+      api(s"issue/$ticket/transitions").get().map{resp =>
+        assert(resp.status == 200)
+        resp.json.validate[Transitions].recover{case e => sys.error(s"Bad JSON: "+e+"\n\n"+resp.json)}.get.transitions.filter(_.name == requiredTransition) match {
+          case Seq() => None
+          case Seq(i) => Some(i)
+        }
+      }
+    }.getOrElse(Future.successful(None))
+    transitionOptionFuture flatMap {transitionOption =>
+      val transitionJson = transitionOption.fold(Json.obj())(transition => Json.obj("transition" -> Json.obj("id" -> transition.id)))
+      val fieldsJson = Json.obj(
+        "fields" -> extractManagedFields(vuln, diff.newSet)
+      )
+      api(s"issue/$ticket").put(transitionJson ++ fieldsJson).map{ resp =>
+        if(resp.status != 204){
+          sys.error("Update failed: "+resp.body)
+        }
+        ()
+      }
+    }
+  }
+
   private def extractInitialFields(vulnerability: Vulnerability): JsObject = Json.obj(
     "project" -> Json.obj(
       "id" -> projectId.toString
@@ -41,18 +95,34 @@ class JiraIssueTrackerService @Inject() (absolutizer: Absolutizer, @Named("jira-
     "summary" -> s"${vulnerability.name} – ${vulnerability.cweOption.map(_ + ": ").getOrElse("")}${vulnerability.description.take(50)}…"
   )
 
-  private def extractManagedFields(vulnerability: Vulnerability): JsObject = Json.obj(
+  private def extractManagedFields(vulnerability: Vulnerability, projects: Set[ReportInfo]): JsObject = {
-    "issuetype" -> Json.obj(
+    val base = Json.obj(
-      "id" -> vulnerabilityIssueType.toString
+      "issuetype" -> Json.obj(
-    ),
+        "id" -> vulnerabilityIssueType.toString
-    "description" -> extractDescription(vulnerability)
+      ),
-    // TODO: add affected releases
+      "description" -> extractDescription(vulnerability)
-    // TODO: add affected projects
+    )
-    //"customfield_10100" -> Json.arr("xxxx")
+    val additionalFields = Seq[Option[(String, JsValueWrapper)]](
-  )
+      fields.cweId.map(id => id -> vulnerability.cweOption.fold("")(_.brief)),
+      fields.linkId.map(id => id -> link(vulnerability)),
+      fields.severityId.map(id => id -> vulnerability.cvssScore),
+      fields.projectsId.map(id => id -> projects.map(friendlyProjectNameString).toSeq.sortBy( x => (x.toLowerCase(), x)).mkString("\n"))
+      // TODO: add affected releases
+    )
+    val additionalObj = Json.obj(additionalFields.flatten : _*)
+    val constantObj = fields.constantFields //Json.obj(fields.constantFields.map{case (k, v) => k -> (v: JsValueWrapper) }.toSeq: _*)
+    base ++ additionalObj ++ constantObj
+  }
 
-  private def extractDescription(vulnerability: Vulnerability): JsValueWrapper = {
+
-    vulnerability.description + "\n\n" + s"Details: ${absolutizer.absolutize(routes.Statistics.vulnerability(vulnerability.name, None))}"
+  /*override def migrateOldIssues(): Future[Unit] = {
+
+  }*/
+
+  private def extractDescription(vulnerability: Vulnerability): String = vulnerability.description + "\n\n" + s"Details: ${link(vulnerability)}"
+
+  private def link(vulnerability: Vulnerability): String = {
+    absolutizer.absolutize(routes.Statistics.vulnerability(vulnerability.name, None))
   }
 
   override def ticketLink(ticket: String): String = s"$server/browse/$ticket"

conf/application.conf.-example

@@ -46,6 +46,20 @@ yssdc{
         user = "…"
         password = "…"
       }
+      newProjectAddedTransitionName: "Add new project"
+      noRelevantProjectAffectedTransitionName: "No longer applicable"
+      ticketFormatVersion: 1 // Increment this when you reconfigure the export format. In a future version, it should cause update of the issues.
+      fields: {
+        cweId: "customfield_10100"
+        linkId: "customfield_10103"
+        severityId: "customfield_10101"
+        projectsId: "customfield_10200"
+        teamsId: "customfield_10105"
+        librariesId: "customfield_10110"
+        constantFields: {
+          "customfield_10102": {"id": "10100"}
+        }
+      }
     }
     # Optional section: email notifications
     email{