diff --git a/app/com/ysoft/odc/statistics/FailedProjects.scala b/app/com/ysoft/odc/statistics/FailedProjects.scala index 829d205..82394aa 100644 --- a/app/com/ysoft/odc/statistics/FailedProjects.scala +++ b/app/com/ysoft/odc/statistics/FailedProjects.scala @@ -11,7 +11,7 @@ final class FailedProjects(val failedProjectsSet: Set[String]){ } object FailedProjects { - private[statistics] def combineFails(failedReportDownloads: Map[String, Throwable], parsingFailures: Map[ReportInfo, Throwable]): FailedProjects = { + def combineFails(failedReportDownloads: Map[String, Throwable], parsingFailures: Map[ReportInfo, Throwable]): FailedProjects = { /* Fail can happen at multiple places: 1. Build cannot be downloaded (auth error, connection error, …) or is failed (failedReportDownloads) diff --git a/app/controllers/DependencyCheckReportsParser.scala b/app/controllers/DependencyCheckReportsParser.scala index 76b24f1..ef1d062 100644 --- a/app/controllers/DependencyCheckReportsParser.scala +++ b/app/controllers/DependencyCheckReportsParser.scala @@ -4,6 +4,7 @@ import java.net.URLEncoder import com.google.inject.Inject import com.ysoft.odc._ +import com.ysoft.odc.statistics.FailedProjects import controllers.DependencyCheckReportsParser.Result import models.PlainLibraryIdentifier import play.api.Logger @@ -31,7 +32,12 @@ private final case class ProjectFilter(project: ReportInfo) extends Filter{ val newFlatReports = f(r.flatReports) val newFailedAnalysises = f(r.failedAnalysises) if(newFlatReports.isEmpty && newFailedAnalysises.isEmpty) None - else Some(Result(bareFlatReports = newFlatReports, bareFailedAnalysises = newFailedAnalysises, projects = r.projects)) + else Some(Result( + bareFlatReports = newFlatReports, + bareFailedAnalysises = newFailedAnalysises, + projects = r.projects, + failedReportDownloads = r.failedReportDownloads // TODO: consider filtering of failedReportDownloads + )) } override def selector = Some(s"project:${project.fullId}") } @@ -44,14 +50,22 @@ private final case class TeamFilter(team: Team) extends Filter{ case other => sys.error("some duplicate value: "+other) }.map(identity) val ProjectName = """^(.*): (.*)$""".r - val rootProjectReports = reportInfoByFriendlyProjectNameMap.collect{case (ProjectName(rootProject, subproject), v) => (rootProject, v)}.groupBy(_._1).mapValues(_.map(_._2)) + val failedProjectsFriendlyNames = r.failedProjects.failedProjectsSet.map(r.projectsReportInfo.parseUnfriendlyName).map(_.projectName) + println(failedProjectsFriendlyNames) + val rootProjectReports = reportInfoByFriendlyProjectNameMap.collect{ case (ProjectName(rootProject, subproject), v) => + (rootProject, v) + }.groupBy(_._1).mapValues(_.values).withDefault(name => + if(failedProjectsFriendlyNames contains name) Seq() + else sys.error("Unknown project: "+name) + ) def reportInfoByFriendlyProjectName(fpn: String) = reportInfoByFriendlyProjectNameMap.get(fpn).map(Set(_)).getOrElse(rootProjectReports(fpn.takeWhile(_ != ':'))) val reportInfos = team.projectNames.flatMap(reportInfoByFriendlyProjectName) def submap[T](m: Map[String, T]) = reportInfos.toSeq.flatMap(ri => m.get(ri.fullId).map(ri.fullId -> _) ).toMap Some(Result( bareFlatReports = submap(r.bareFlatReports), bareFailedAnalysises = submap(r.bareFailedAnalysises), - projects = r.projects + projects = r.projects, + failedReportDownloads = r.failedReportDownloads // TODO: consider filtering of failedReportDownloads )) } override def descriptionHtml: Html = views.html.filters.team(team.id) @@ -75,10 +89,11 @@ private final case class BadFilter(pattern: String) extends Filter{ object DependencyCheckReportsParser{ final case class ResultWithSelection(result: Result, projectsWithSelection: ProjectsWithSelection) - final case class Result(bareFlatReports: Map[String, Analysis], bareFailedAnalysises: Map[String, Throwable], projects: Projects /*TODO: maybe rename to rootProjects*/){ + final case class Result(bareFlatReports: Map[String, Analysis], bareFailedAnalysises: Map[String, Throwable], projects: Projects /*TODO: maybe rename to rootProjects*/, failedReportDownloads: Map[String, Throwable]){ lazy val projectsReportInfo = new ProjectsWithReports(projects, bareFlatReports.keySet ++ bareFailedAnalysises.keySet) // TODO: consider renaming to projectsWithReports lazy val flatReports: Map[ReportInfo, Analysis] = bareFlatReports.map{case (k, v) => projectsReportInfo.reportIdToReportInfo(k) -> v} lazy val failedAnalysises: Map[ReportInfo, Throwable] = bareFailedAnalysises.map{case (k, v) => projectsReportInfo.reportIdToReportInfo(k) -> v} + lazy val failedProjects = FailedProjects.combineFails(parsingFailures = failedAnalysises, failedReportDownloads = failedReportDownloads) lazy val allDependencies = flatReports.toSeq.flatMap(r => r._2.dependencies.map(_ -> r._1)) lazy val groupedDependencies = allDependencies.groupBy(_._1.hashes).values.map(GroupedDependency(_)).toSeq lazy val groupedDependenciesByPlainLibraryIdentifier: Map[PlainLibraryIdentifier, Set[GroupedDependency]] = @@ -109,7 +124,7 @@ object DependencyCheckReportsParser{ final class DependencyCheckReportsParser @Inject() (cache: CacheApi, projects: Projects) { - def parseReports(successfulResults: Map[String, (Build, ArtifactItem, ArtifactFile)]) = { + def parseReports(successfulResults: Map[String, (Build, ArtifactItem, ArtifactFile)], failedReportDownloads: Map[String, Throwable]) = { val rid = math.random.toString // for logging @volatile var parseFailedForSomeAnalysis = false val deepReportsTriesIterable: Iterable[Map[String, Try[Analysis]]] = for((k, (build, data, log)) <- successfulResults) yield { @@ -140,7 +155,7 @@ final class DependencyCheckReportsParser @Inject() (cache: CacheApi, projects: P val failedAnalysises = deepReportsAndFailuresIterable.map(_._2).toSeq.flatten.toMap val flatReports = deepSuccessfulReports.flatten.toMap Logger.debug(s"[$rid] parse finished") - Result(flatReports, failedAnalysises, projects) + Result(flatReports, failedAnalysises, projects, failedReportDownloads = failedReportDownloads) } } diff --git a/app/controllers/DependencyCheckReportsProcessor.scala b/app/controllers/DependencyCheckReportsProcessor.scala index 6005377..64c8bcf 100644 --- a/app/controllers/DependencyCheckReportsProcessor.scala +++ b/app/controllers/DependencyCheckReportsProcessor.scala @@ -46,7 +46,7 @@ final class DependencyCheckReportsProcessor @Inject() ( requiredVersions: Map[String, Int] )(implicit requestHeader: DefaultRequest, snoozesInfo: SnoozesInfo, executionContext: ExecutionContext) = try{ for((successfulResults, failedResults) <- resultsFuture) yield{ - val reportResult = dependencyCheckReportsParser.parseReports(successfulResults) + val reportResult = dependencyCheckReportsParser.parseReports(successfulResults, failedResults) import reportResult.{allDependencies, failedAnalysises, flatReports, groupedDependencies, vulnerableDependencies} val now = DateTime.now val oldReportThreshold = now - 1.day diff --git a/app/controllers/Notifications.scala b/app/controllers/Notifications.scala index eaa4d65..bb94364 100644 --- a/app/controllers/Notifications.scala +++ b/app/controllers/Notifications.scala @@ -42,7 +42,7 @@ class Notifications @Inject()( (successfulReports, failedReports) <- resultsFuture myWatches <- myWatchesFuture } yield { - val projects = dependencyCheckReportsParser.parseReports(successfulReports).projectsReportInfo.sortedReportsInfo + val projects = dependencyCheckReportsParser.parseReports(successfulReports, failedReports).projectsReportInfo.sortedReportsInfo Ok(views.html.notifications.index(projects, myWatches)) } } @@ -110,7 +110,7 @@ class Notifications @Inject()( // TODO: process failedReports, parsedReports.failedAnalysises and successfulResults.filter(x => x._2._1.state != "Successful" || x._2._1.buildState != "Successful") (successfulReports, failedReports) <- resultsFuture libraries <- librariesService.all - parsedReports = dependencyCheckReportsParser.parseReports(successfulReports) + parsedReports = dependencyCheckReportsParser.parseReports(successfulReports, failedReports) lds = LibDepStatistics(dependencies = parsedReports.groupedDependencies.toSet, libraries = libraries.toSet, failedReportDownloads = failedReports, parsedReports = parsedReports) failed = lds.failedProjects failedReportsExportFuture = Fut(()) // TODO: exportFailedReports(lds, failed) diff --git a/app/controllers/Statistics.scala b/app/controllers/Statistics.scala index 0d5e306..3fee34f 100644 --- a/app/controllers/Statistics.scala +++ b/app/controllers/Statistics.scala @@ -5,6 +5,7 @@ import com.google.inject.Inject import com.google.inject.name.Named import com.ysoft.odc.statistics.{LibDepStatistics, TagStatistics} import com.ysoft.odc.{ArtifactFile, ArtifactItem} +import controllers.DependencyCheckReportsParser.ResultWithSelection import models.LibraryTag import org.joda.time.DateTime import play.api.i18n.MessagesApi @@ -39,7 +40,8 @@ class Statistics @Inject() ( import secureRequestConversion._ - private def select(successfulResults: Map[String, (Build, ArtifactItem, ArtifactFile)], selectorOption: Option[String]) = dependencyCheckReportsParser.parseReports(successfulResults).selection(selectorOption) + private def select(allResults: (Map[String, (Build, ArtifactItem, ArtifactFile)], Map[String, Throwable]), selectorOption: Option[String]): Option[ResultWithSelection] = select(allResults._1, allResults._2, selectorOption) + private def select(successfulResults: Map[String, (Build, ArtifactItem, ArtifactFile)], failedResults: Map[String, Throwable], selectorOption: Option[String]): Option[ResultWithSelection] = dependencyCheckReportsParser.parseReports(successfulResults, failedResults).selection(selectorOption) def searchVulnerableSoftware(versionlessCpes: Seq[String], versionOption: Option[String]) = ReadAction.async{ implicit req => if(versionlessCpes.isEmpty){ @@ -71,26 +73,26 @@ class Statistics @Inject() ( } } - def basic(projectOption: Option[String]) = ReadAction.async{ implicit req => + def basic(selectorOption: Option[String]) = ReadAction.async{ implicit req => val (lastRefreshTime, resultsFuture) = projectReportsProvider.resultsForVersions(versions) - resultsFuture flatMap { case (successfulResults, failedResults) => - select(successfulResults, projectOption).fold(Future.successful(notFound())){ selection => + resultsFuture flatMap { allResults => + select(allResults, selectorOption).fold(Future.successful(notFound())){ selection => val tagsFuture = tagsService.all val parsedReports = selection.result for{ - tagStatistics <- statisticsForTags(parsedReports, failedResults, tagsFuture) + tagStatistics <- statisticsForTags(parsedReports, tagsFuture) libraries <- librariesService.all } yield Ok(views.html.statistics.basic( tagStatistics = tagStatistics, projectsWithSelection = selection.projectsWithSelection, parsedReports = parsedReports, - lds = LibDepStatistics(libraries.toSet, parsedReports.groupedDependencies.toSet, failedResults, parsedReports) + lds = LibDepStatistics(libraries.toSet, parsedReports.groupedDependencies.toSet, selection.result.failedReportDownloads, parsedReports) )) } } } - def statisticsForTags(parsedReports: DependencyCheckReportsParser.Result, failedReports: Map[String, Throwable], tagsFuture: Future[Seq[(Int, LibraryTag)]]): Future[Seq[TagStatistics]] = { + def statisticsForTags(parsedReports: DependencyCheckReportsParser.Result, tagsFuture: Future[Seq[(Int, LibraryTag)]]): Future[Seq[TagStatistics]] = { val librariesFuture = librariesService.byPlainLibraryIdentifiers(parsedReports.allDependencies.flatMap(_._1.plainLibraryIdentifiers).toSet) val libraryTagAssignmentsFuture = librariesFuture.flatMap{libraries => libraryTagAssignmentsService.forLibraries(libraries.values.map(_._1).toSet)} val tagsToLibrariesFuture = libraryTagAssignmentsService.tagsToLibraries(libraryTagAssignmentsFuture) @@ -110,7 +112,7 @@ class Statistics @Inject() ( stats = LibDepStatistics( libraries = tagLibraries, dependencies = tagDependencies, - failedReportDownloads = failedReports, + failedReportDownloads = parsedReports.failedReportDownloads, parsedReports = parsedReports ) )) @@ -119,8 +121,8 @@ class Statistics @Inject() ( def vulnerabilities(projectOption: Option[String], tagIdOption: Option[Int]) = ReadAction.async {implicit req => val (lastRefreshTime, resultsFuture) = projectReportsProvider.resultsForVersions(versions) - resultsFuture flatMap { case (successfulResults, failedResults) => - select(successfulResults, projectOption).fold(Future.successful(notFound())){ selection => + resultsFuture flatMap { allResults => + select(allResults, projectOption).fold(Future.successful(notFound())){ selection => val parsedReports = selection.result for{ libraries <- librariesService.byPlainLibraryIdentifiers(parsedReports.allDependencies.flatMap(_._1.plainLibraryIdentifiers).toSet) @@ -128,12 +130,12 @@ class Statistics @Inject() ( statistics <- tagOption.fold(Future.successful(LibDepStatistics( dependencies = parsedReports.groupedDependencies.toSet, libraries = libraries.values.toSet, - failedReportDownloads = failedResults, + failedReportDownloads = selection.result.failedReportDownloads, parsedReports = parsedReports ))){ tag => - statisticsForTags(parsedReports, failedResults, Future.successful(Seq(tag))).map{ + statisticsForTags(parsedReports, Future.successful(Seq(tag))).map{ case Seq(TagStatistics(_, stats)) => stats // statisticsForTags is designed for multiple tags, but we have just one… - case Seq() => LibDepStatistics(libraries = Set(), dependencies = Set(), failedReportDownloads = failedResults, parsedReports) // We don't want to crash when no dependencies are there… + case Seq() => LibDepStatistics(libraries = Set(), dependencies = Set(), failedReportDownloads = selection.result.failedReportDownloads, parsedReports) // We don't want to crash when no dependencies are there… } } } yield Ok(views.html.statistics.vulnerabilities( @@ -145,10 +147,10 @@ class Statistics @Inject() ( } } - def vulnerability(name: String, projectOption: Option[String]) = ReadAction.async { implicit req => + def vulnerability(name: String, selectorOption: Option[String]) = ReadAction.async { implicit req => val (lastRefreshTime, resultsFuture) = projectReportsProvider.resultsForVersions(versions) - resultsFuture flatMap { case (successfulResults, failedResults) => - select(successfulResults, projectOption).fold(Future.successful(notFound())){ selection => + resultsFuture flatMap { allResults => + select(allResults, selectorOption).fold(Future.successful(notFound())){ selection => val relevantReports = selection.result val vulns = relevantReports.vulnerableDependencies.flatMap(dep => dep.vulnerabilities.map(vuln => (vuln, dep))).groupBy(_._1.name).mapValues{case vulnsWithDeps => val (vulnSeq, depSeq) = vulnsWithDeps.unzip @@ -186,10 +188,10 @@ class Statistics @Inject() ( } } - def vulnerableLibraries(project: Option[String]) = ReadAction.async { implicit req => + def vulnerableLibraries(selectorOption: Option[String]) = ReadAction.async { implicit req => val (lastRefreshTime, resultsFuture) = projectReportsProvider.resultsForVersions(versions) - resultsFuture flatMap { case (successfulResults, failedResults) => - select(successfulResults, project).fold(Future.successful(notFound())){selection => + resultsFuture flatMap { allResults => + select(allResults, selectorOption).fold(Future.successful(notFound())){ selection => val reports = selection.result Future.successful(Ok(views.html.statistics.vulnerableLibraries( projectsWithSelection = selection.projectsWithSelection, @@ -200,10 +202,10 @@ class Statistics @Inject() ( } } - def allLibraries(project: Option[String]) = ReadAction.async { implicit req => + def allLibraries(selectorOption: Option[String]) = ReadAction.async { implicit req => val (lastRefreshTime, resultsFuture) = projectReportsProvider.resultsForVersions(versions) - resultsFuture flatMap { case (successfulResults, failedResults) => - select(successfulResults, project).fold(Future.successful(notFound())){selection => + resultsFuture flatMap { allResults => + select(allResults, selectorOption).fold(Future.successful(notFound())){ selection => Future.successful(Ok(views.html.statistics.allLibraries( projectsWithSelection = selection.projectsWithSelection, allDependencies = selection.result.groupedDependencies @@ -212,10 +214,10 @@ class Statistics @Inject() ( } } - def allGavs(project: Option[String]) = ReadAction.async { implicit req => + def allGavs(selectorOption: Option[String]) = ReadAction.async { implicit req => val (lastRefreshTime, resultsFuture) = projectReportsProvider.resultsForVersions(versions) - resultsFuture flatMap { case (successfulResults, failedResults) => - select(successfulResults, project).fold(Future.successful(notFound())){selection => + resultsFuture flatMap { allResults => + select(allResults, selectorOption).fold(Future.successful(notFound())){ selection => Future.successful(Ok(Txt( selection.result.groupedDependencies.flatMap(_.mavenIdentifiers).toSet.toIndexedSeq.sortBy((id: Identifier) => (id.identifierType, id.name)).map(id => id.name.split(':') match { case Array(g, a, v) =>