diff --git a/app/com/ysoft/odc/OdcParser.scala b/app/com/ysoft/odc/OdcParser.scala index ebd1b99..50896b8 100644 --- a/app/com/ysoft/odc/OdcParser.scala +++ b/app/com/ysoft/odc/OdcParser.scala @@ -35,7 +35,9 @@ object SerializableXml{ def apply(xml: NodeSeq): SerializableXml = SerializableXml(xml.toString()) } -final case class Analysis(scanInfo: SerializableXml, name: String, reportDate: DateTime, dependencies: Seq[Dependency]) +final case class Analysis(scanInfo: SerializableXml, name: String, groupId: String, artifactId: String, version: String, reportDate: DateTime, dependencies: Seq[Dependency]){ + def groupIdAndArtifactId = (groupId, artifactId) +} final case class Hashes(sha1: String, md5: String){ // TODO: consider adding SHA256 without breaking backward compatibility @@ -423,6 +425,9 @@ object OdcParser { Analysis( scanInfo = SerializableXml((xml \ "scanInfo").head), name = (xml \ "projectInfo" \ "name").text, + groupId = (xml \ "projectInfo" \ "groupID").text, + artifactId = (xml \ "projectInfo" \ "artifactID").text, + version = (xml \ "projectInfo" \ "version").text, reportDate = DateTime.parse((xml \ "projectInfo" \ "reportDate").text), dependencies = parseDependencies(xml \ "dependencies" \ "dependency").toIndexedSeq ) diff --git a/app/controllers/Statistics.scala b/app/controllers/Statistics.scala index 12d42da..0eb8be4 100644 --- a/app/controllers/Statistics.scala +++ b/app/controllers/Statistics.scala @@ -293,21 +293,27 @@ class Statistics @Inject()( implicit val scannedRepositoryFormat = Json.format[ScannedRepository] implicit val scannedProjectFormats = Json.format[ScannedProject] + private val RepoFetchLogLine = """.*Fetching 'refs/heads/(.*)' from '(.*)'\..*""".r // Bamboo does not seem to have a suitable API, so we are parsing it from logs… + def table() = ApiAction(ProjectTable).async{ - val RepoFetch = """.*Fetching 'refs/heads/(.*)' from '(.*)'\..*""".r // Bamboo does not seem to have a suitable API, so we are parsing it from logs… val (lastRefreshTime, resultsFuture) = projectReportsProvider.resultsForVersions(versions) resultsFuture map { allResults => val t = projects.projectMap val rows = t.toIndexedSeq.sortBy(r => (r._2.toLowerCase, r._2)).map{case (key, name) => - val repos = allResults._1.get(key).map(_._3.dataString.lines.collect{ - case RepoFetch(branch, repo) => ScannedRepository(repo, branch) - }.toSet).getOrElse(Set.empty).toIndexedSeq.sortBy(ScannedRepository.unapply) + val repos: _root_.scala.collection.immutable.IndexedSeq[_root_.controllers.ScannedRepository] = getRepositoryForScan(allResults._1, key) ScannedProject(name, repos, projects.teamsByProjectId(key).toIndexedSeq.map(_.name).sorted, key) } Ok(Json.toJson(rows)) } } + private def getRepositoryForScan(successfulResults: Map[String, (Build, ArtifactItem, ArtifactFile)], key: String) = { + val repos = successfulResults.get(key).map(_._3.dataString.lines.collect { + case RepoFetchLogLine(branch, repo) => ScannedRepository(repo, branch) + }.toSet).getOrElse(Set.empty).toIndexedSeq.sortBy(ScannedRepository.unapply) + repos + } + def allDependencies(selectorOption: Option[String]) = ApiAction(Dependencies).async { implicit req => val (lastRefreshTime, resultsFuture) = projectReportsProvider.resultsForVersions(versions) resultsFuture flatMap { allResults => @@ -504,5 +510,28 @@ class Statistics @Inject()( } } + def internalDependencies(selector: Option[String]) = ApiAction(Dependencies).async { + val (lastRefreshTime, resultsFuture) = projectReportsProvider.resultsForVersions(versions) + resultsFuture flatMap { case (successfulResults, failedResults) => + val reports = dependencyCheckReportsParser.parseReports(successfulResults, failedResults) + reports.selection(selector).fold(Future.successful(NotFound(Json.obj("error" -> "not found")))) { selection => + val dependenciesByVersionlessIdentifiers = reports.flatReports.groupBy(_._2.groupIdAndArtifactId) + val allVersionlessIdentifiers = dependenciesByVersionlessIdentifiers.keySet + val scopedVersionlessIdentifiers = selection.result.groupedDependencies.flatMap(_.mavenIdentifiers).map( x => (x.name+":ignored").split(':') match { + case Array(groupId, artifactId, version, _) => (groupId, artifactId) + case other => sys.error("Unexpected array: "+other.toSeq) + }).toSet + Future.successful(Ok(Json.toJson(Map( + "internalMavenDependencies" -> Json.toJson(allVersionlessIdentifiers.intersect(scopedVersionlessIdentifiers).map(id => + Map( + "mavenIdentifier" -> Json.toJson(id match {case (groupId, artifactId) => s"$groupId:$artifactId"}), + "repositories" -> Json.toJson(dependenciesByVersionlessIdentifiers(id).map(_._1.projectId).flatMap(getRepositoryForScan(successfulResults, _)).toSet) + ) + )) + )))) + } + } + } + } diff --git a/conf/routes b/conf/routes index a6f1bb9..538d599 100644 --- a/conf/routes +++ b/conf/routes @@ -7,6 +7,7 @@ GET / controllers.Application.homepage() GET /api/stats/libraries/count.json controllers.Statistics.librariesCountApi(selector: Option[String], operator: Option[String], threshold: Option[Double], strict: Boolean) GET /api/table controllers.Statistics.table() GET /api/all-dependencies.json controllers.Statistics.allDependencies(selector: Option[String]) +GET /api/internal-dependencies.json controllers.Statistics.internalDependencies(selector: Option[String]) GET /api/all-dependencies-verbose.json controllers.Statistics.allDependenciesVerbose(selector: Option[String]) POST /api/compare-scan controllers.Statistics.compareScan() GET /api/library/:depId/vulnerabilities controllers.Statistics.libraryVulnerabilities(depId: com.ysoft.odc.Hashes) diff --git a/test/factories/ReportsFactory.scala b/test/factories/ReportsFactory.scala index a527af7..9ed3318 100644 --- a/test/factories/ReportsFactory.scala +++ b/test/factories/ReportsFactory.scala @@ -18,7 +18,10 @@ object ReportsFactory{ reportDate = DateTime.lastDay, dependencies = Seq( buildDependency(projectId) - ) + ), + groupId = "com.ysoft.something", + artifactId = "someArtifact", + version = "3.1.4.1.5.9.2.6.5.3.6" ) } projectId -> thuck