From 39ba123efc570fb287b733d6081af45d03542e9d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0est=C3=A1k=20V=C3=ADt?= Date: Fri, 8 Dec 2017 10:18:25 +0100 Subject: [PATCH] Added support for comparison of scans --- app/com/ysoft/odc/OdcParser.scala | 6 ++-- .../DependencyCheckReportsParser.scala | 6 ++++ app/controllers/Statistics.scala | 33 ++++++++++++++++++- app/controllers/api/ApiResources.scala | 3 +- conf/routes | 1 + 5 files changed, 45 insertions(+), 4 deletions(-) diff --git a/app/com/ysoft/odc/OdcParser.scala b/app/com/ysoft/odc/OdcParser.scala index e0efb5e..abae7e1 100644 --- a/app/com/ysoft/odc/OdcParser.scala +++ b/app/com/ysoft/odc/OdcParser.scala @@ -387,8 +387,10 @@ object OdcParser { def parseDependencies(nodes: NodeSeq): Seq[Dependency] = nodes.map(parseDependency(_)) - def parseXmlReport(data: Array[Byte]) = { - val xml = SecureXml.loadString(new String(data, "utf-8")) + def parseXmlReport(data: Array[Byte]): Analysis = parseXmlReport(new String(data, "utf-8")) + + def parseXmlReport(xmlData: String): Analysis = { + val xml = SecureXml.loadString(xmlData) Analysis( scanInfo = SerializableXml((xml \ "scanInfo").head), name = (xml \ "projectInfo" \ "name").text, diff --git a/app/controllers/DependencyCheckReportsParser.scala b/app/controllers/DependencyCheckReportsParser.scala index 579ee22..7d6d03d 100644 --- a/app/controllers/DependencyCheckReportsParser.scala +++ b/app/controllers/DependencyCheckReportsParser.scala @@ -114,6 +114,12 @@ private final case class BadFilter(pattern: String) extends Filter{ object DependencyCheckReportsParser{ def forAdHocScan(analysis: Analysis): Result = Result(Map(ReportInfo("adHocScan", "Ad hoc scan", "AHS", None) -> analysis), Map(), new ProjectsWithReports(new Projects(Map(), Map(), Map()), Set()), Map()) + def forAdHocScans(analysises: Map[String, Analysis]): Result = Result( + bareFlatReports = analysises.map{case (key, analysis) => ReportInfo("adHocScan", "Ad hoc scan", "AHS:"+key, Some(key)) -> analysis}, + bareFailedAnalysises = Map(), + projectsReportInfo = new ProjectsWithReports(new Projects(Map(), Map(), Map()), Set()), + failedReportDownloads = Map() + ) final case class ResultWithSelection(result: Result, projectsWithSelection: ProjectsWithSelection) final case class Result(bareFlatReports: Map[ReportInfo, Analysis], bareFailedAnalysises: Map[ReportInfo, Throwable], projectsReportInfo: ProjectsWithReports/*TODO: maybe rename to rootProjects*/, failedReportDownloads: Map[ReportInfo, Throwable]){ //lazy val projectsReportInfo = new ProjectsWithReports(projects, (bareFlatReports.keySet ++ bareFailedAnalysises.keySet ++ failedReportDownloads.keySet).map(_.fullId)) // TODO: consider renaming to projectsWithReports diff --git a/app/controllers/Statistics.scala b/app/controllers/Statistics.scala index f9e17fc..4eca436 100644 --- a/app/controllers/Statistics.scala +++ b/app/controllers/Statistics.scala @@ -6,7 +6,7 @@ import com.google.inject.name.Named import com.ysoft.odc.Confidence.Confidence import com.ysoft.odc.statistics.{LibDepStatistics, TagStatistics} import com.ysoft.odc._ -import controllers.DependencyCheckReportsParser.ResultWithSelection +import controllers.DependencyCheckReportsParser.{Result, ResultWithSelection} import controllers.api.{ApiConfig, ApiController} import models.LibraryTag import modules.TemplateCustomization @@ -25,6 +25,8 @@ final case class ScannedProject(name: String, repos: Seq[ScannedRepository], tea final case class GroupedDependencyIdentifier(hashes: Hashes, identifiers: Seq[Identifier]) +final case class CompareScanRequest(plan: String, reports: Map[String, String]) + object GroupedDependencyIdentifier{ def fromGroupedDependency(groupedDependency: GroupedDependency): GroupedDependencyIdentifier = GroupedDependencyIdentifier( hashes = groupedDependency.hashes, @@ -386,5 +388,34 @@ class Statistics @Inject()( } } + private implicit val compareScanRequestFormats = Json.format[CompareScanRequest] + + def showSet[T: Writes](set: Set[T]) = JsArray(set.toSeq.map(implicitly[Writes[T]].writes)) + def showDiff[T: Writes](diff: SetDiff[T]) = Json.obj("added" -> showSet(diff.added), "removed" -> showSet(diff.removed), "old"->showSet(diff.oldSet), "new"->showSet(diff.newSet)) + + def compareScan() = ApiAction(ScanResults).async(parse.json[CompareScanRequest]){ implicit req => + val unparsedReports = req.body.reports + val reportMapFuture = Future { + unparsedReports.mapValues(OdcParser.parseXmlReport).view.force + } + val (lastRefreshTime, resultsFuture) = projectReportsProvider.resultsForVersions(versions) + resultsFuture flatMap { allResults => + select(allResults, Some("project:"+req.body.plan)).fold(Future.successful(NotFound(Json.obj("error"->"not found")))) { selection => + reportMapFuture.map {reportMap => + def extractVulnerabilities(r: Result) = { + r.vulnerableDependencies.flatMap(_.vulnerabilities.map(_.name)).toSet + } + val adHocReports = DependencyCheckReportsParser.forAdHocScans(reportMap) + def compare[T](f: Result => Set[T]) = new SetDiff(f(selection.result), f(adHocReports)) + //adHocReports.dep + Ok(Json.obj( + "vulnerabilities"->showDiff(compare(extractVulnerabilities)), + "dependencies"->showDiff(compare(_.groupedDependencies.map(GroupedDependencyIdentifier.fromGroupedDependency).toSet)) + )) + } + } + } + } + } diff --git a/app/controllers/api/ApiResources.scala b/app/controllers/api/ApiResources.scala index 860234c..2dd8968 100644 --- a/app/controllers/api/ApiResources.scala +++ b/app/controllers/api/ApiResources.scala @@ -3,10 +3,11 @@ package controllers.api trait ApiResources { val ProjectTable = ApiResource("project-table") val Dependencies = ApiResource("dependencies") + val ScanResults = ApiResource("scan-results") } object ApiResources extends ApiResources{ - val All = Set(ProjectTable, Dependencies) + val All = Set(ProjectTable, Dependencies, ScanResults) private val AllByName = All.map(res => res.name -> res).toMap def byName(name: String): Option[ApiResource] = AllByName.get(name) } \ No newline at end of file diff --git a/conf/routes b/conf/routes index 7a2e912..3beb8dd 100644 --- a/conf/routes +++ b/conf/routes @@ -7,6 +7,7 @@ GET / controllers.Application.homepage() GET /api/table controllers.Statistics.table() GET /api/all-dependencies.json controllers.Statistics.allDependencies(selector: Option[String]) GET /api/all-dependencies-verbose.json controllers.Statistics.allDependenciesVerbose(selector: Option[String]) +POST /api/compare-scan controllers.Statistics.compareScan() GET /status controllers.Application.index(versions: Map[String, Int] = Map()) GET /versions controllers.Application.index(versions: Map[String, Int]) GET /dependencies controllers.Application.dependencies(classified: Option[Boolean] = None, requiredTags: Seq[Int] ?= Seq(), noTag: Boolean ?= false)