diff --git a/app/com/ysoft/odc/statistics/FailedProjects.scala b/app/com/ysoft/odc/statistics/FailedProjects.scala index 82394aa..6b4f481 100644 --- a/app/com/ysoft/odc/statistics/FailedProjects.scala +++ b/app/com/ysoft/odc/statistics/FailedProjects.scala @@ -2,22 +2,27 @@ package com.ysoft.odc.statistics import controllers.ReportInfo -final class FailedProjects(val failedProjectsSet: Set[String]){ +final class FailedProjects(val failedProjectsSet: Set[ReportInfo]){ + + val failedProjectIdsSet = failedProjectsSet.map(_.projectId) + + def nonEmpty: Boolean = failedProjectsSet.nonEmpty + def isFailed(projectFullId: String): Boolean = { val projectBareId = projectFullId.takeWhile(_ != '/') - failedProjectsSet contains projectBareId + failedProjectIdsSet contains projectBareId } } object FailedProjects { - def combineFails(failedReportDownloads: Map[String, Throwable], parsingFailures: Map[ReportInfo, Throwable]): FailedProjects = { + def combineFails(failedReportDownloads: Map[ReportInfo, Throwable], parsingFailures: Map[ReportInfo, Throwable]): FailedProjects = { /* Fail can happen at multiple places: 1. Build cannot be downloaded (auth error, connection error, …) or is failed (failedReportDownloads) 2. Build is successful and can be downloaded, but it cannot be parsed (parsingFailures) */ - val failedProjectsSet = failedReportDownloads.keySet ++ parsingFailures.keySet.map(_.projectId) + val failedProjectsSet = failedReportDownloads.keySet ++ parsingFailures.keySet new FailedProjects(failedProjectsSet) } } diff --git a/app/com/ysoft/odc/statistics/LibDepStatistics.scala b/app/com/ysoft/odc/statistics/LibDepStatistics.scala index ed42ac3..242e316 100644 --- a/app/com/ysoft/odc/statistics/LibDepStatistics.scala +++ b/app/com/ysoft/odc/statistics/LibDepStatistics.scala @@ -23,12 +23,9 @@ case class LibDepStatistics(libraries: Set[(Int, Library)], dependencies: Set[Gr object LibDepStatistics{ private def computeWeaknessesFrequency(vulnerabilities: Set[Vulnerability]) = vulnerabilities.toSeq.map(_.cweOption).groupBy(identity).mapValues(_.size).map(identity).withDefaultValue(0) - def apply(libraries: Set[(Int, Library)], dependencies: Set[GroupedDependency], failedReportDownloads: Map[String, Throwable], parsedReports: Result): LibDepStatistics = LibDepStatistics( + def apply(libraries: Set[(Int, Library)], dependencies: Set[GroupedDependency], parsedReports: Result): LibDepStatistics = LibDepStatistics( libraries = libraries, dependencies = dependencies, - failedProjects = FailedProjects.combineFails( - failedReportDownloads = failedReportDownloads, - parsingFailures = parsedReports.failedAnalysises - ) + failedProjects = parsedReports.failedProjects ) } diff --git a/app/controllers/DependencyCheckReportsParser.scala b/app/controllers/DependencyCheckReportsParser.scala index 4768a53..bd16612 100644 --- a/app/controllers/DependencyCheckReportsParser.scala +++ b/app/controllers/DependencyCheckReportsParser.scala @@ -26,17 +26,19 @@ private final case class ProjectFilter(project: ReportInfo) extends Filter{ override def descriptionText: String = s"project ${friendlyProjectNameString(project)}" override def subReports(r: Result): Option[Result] = { @inline def reportInfo = project - def f[T](m: Map[ReportInfo, T]): Map[String, T] = ( - if(reportInfo.subprojectNameOption.isEmpty) m.filter(_._1.projectId == project.projectId) else m.get(reportInfo).fold(Map.empty[ReportInfo, T])(x => Map(reportInfo -> x)) - ).map{case (k, v) => k.fullId -> v} - val newFlatReports = f(r.flatReports) - val newFailedAnalysises = f(r.failedAnalysises) - if(newFlatReports.isEmpty && newFailedAnalysises.isEmpty) None + def filter[T](m: Map[ReportInfo, T]): Map[ReportInfo, T] = ( + if(reportInfo.subprojectNameOption.isEmpty) m.filter(_._1.projectId == project.projectId) + else m.get(reportInfo).fold(Map.empty[ReportInfo, T])(x => Map(reportInfo -> x)) + ) + val newFlatReports = filter(r.flatReports) + val newFailedAnalysises = filter(r.failedAnalysises) + val newFailedReportDownloads = filter(r.failedReportDownloads) + if(newFlatReports.isEmpty && newFailedAnalysises.isEmpty && newFailedReportDownloads.isEmpty) None else Some(Result( bareFlatReports = newFlatReports, bareFailedAnalysises = newFailedAnalysises, - projects = r.projects, - failedReportDownloads = r.failedReportDownloads // TODO: consider filtering of failedReportDownloads + projectsReportInfo = r.projectsReportInfo, + failedReportDownloads = newFailedReportDownloads )) } override def selector = Some(s"project:${project.fullId}") @@ -45,27 +47,33 @@ private final case class TeamFilter(team: Team) extends Filter{ override def filters: Boolean = true override def subReports(r: Result): Option[Result] = { val Wildcard = """^(.*): \*$""".r - val reportInfoByFriendlyProjectNameMap = r.projectsReportInfo.ungroupedReportsInfo.map(ri => friendlyProjectNameString(ri) -> ri).toSeq.groupBy(_._1).mapValues{ - case Seq((_, ri)) => ri + @inline def toMapStrict[K, V](l: Traversable[(K, V)]) = l.toSeq.groupBy(_._1).mapValues{ // without toSeq, the pattern matching might fail + case Seq((_, v)) => v case other => sys.error("some duplicate value: "+other) }.map(identity) + val reportInfoByFriendlyProjectNameMap = toMapStrict(r.projectsReportInfo.ungroupedReportsInfo.map(ri => friendlyProjectNameString(ri) -> ri)) val ProjectName = """^(.*): (.*)$""".r - val failedProjectsFriendlyNames = r.failedProjects.failedProjectsSet.map(r.projectsReportInfo.parseUnfriendlyName).map(_.projectName) - println(failedProjectsFriendlyNames) - val rootProjectReports = reportInfoByFriendlyProjectNameMap.collect{ case (ProjectName(rootProject, subproject), v) => - (rootProject, v) + val failedProjectsFriendlyNames = r.failedProjects.failedProjectsSet.map(_.projectName) + Logger.error("failedProjectsFriendlyNames: "+failedProjectsFriendlyNames) + val rootProjectReports = reportInfoByFriendlyProjectNameMap.map{ + case (ProjectName(rootProject, _subproject), v) => (rootProject, v) + case value @ (rootProject, v) => value }.groupBy(_._1).mapValues(_.values).withDefault(name => if(failedProjectsFriendlyNames contains name) Seq() else sys.error("Unknown project: "+name) ) - def reportInfoByFriendlyProjectName(fpn: String) = reportInfoByFriendlyProjectNameMap.get(fpn).map(Set(_)).getOrElse(rootProjectReports(fpn.takeWhile(_ != ':'))) + def reportInfoByFriendlyProjectName(fpn: String) = fpn match{ + case Wildcard(rfpn) => rootProjectReports(rfpn) + case _ => Set(reportInfoByFriendlyProjectNameMap(fpn)) + } val reportInfos = team.projectNames.flatMap(reportInfoByFriendlyProjectName) - def submap[T](m: Map[String, T]) = reportInfos.toSeq.flatMap(ri => m.get(ri.fullId).map(ri.fullId -> _) ).toMap + def submap[T](m: Map[ReportInfo, T]) = reportInfos.toSeq.flatMap(ri => m.get(ri).map(ri -> _) ).toMap + def submapBare[T](m: Map[ReportInfo, T]): Map[ReportInfo, T] = reportInfos.toSeq.flatMap(ri => m.get(ri.bare.ensuring{x => println(x.fullId); true}).map(ri -> _) ).toMap Some(Result( bareFlatReports = submap(r.bareFlatReports), - bareFailedAnalysises = submap(r.bareFailedAnalysises), - projects = r.projects, - failedReportDownloads = r.failedReportDownloads // TODO: consider filtering of failedReportDownloads + bareFailedAnalysises = submapBare(r.bareFailedAnalysises), + projectsReportInfo = r.projectsReportInfo, + failedReportDownloads = submapBare(r.failedReportDownloads) )) } override def descriptionHtml: Html = views.html.filters.team(team.id) @@ -89,10 +97,11 @@ private final case class BadFilter(pattern: String) extends Filter{ object DependencyCheckReportsParser{ final case class ResultWithSelection(result: Result, projectsWithSelection: ProjectsWithSelection) - final case class Result(bareFlatReports: Map[String, Analysis], bareFailedAnalysises: Map[String, Throwable], projects: Projects /*TODO: maybe rename to rootProjects*/, failedReportDownloads: Map[String, Throwable]){ - lazy val projectsReportInfo = new ProjectsWithReports(projects, bareFlatReports.keySet ++ bareFailedAnalysises.keySet ++ failedReportDownloads.keySet) // TODO: consider renaming to projectsWithReports - lazy val flatReports: Map[ReportInfo, Analysis] = bareFlatReports.map{case (k, v) => projectsReportInfo.reportIdToReportInfo(k) -> v} - lazy val failedAnalysises: Map[ReportInfo, Throwable] = bareFailedAnalysises.map{case (k, v) => projectsReportInfo.reportIdToReportInfo(k) -> v} + final case class Result(bareFlatReports: Map[ReportInfo, Analysis], bareFailedAnalysises: Map[ReportInfo, Throwable], projectsReportInfo: ProjectsWithReports/*TODO: maybe rename to rootProjects*/, failedReportDownloads: Map[ReportInfo, Throwable]){ + //lazy val projectsReportInfo = new ProjectsWithReports(projects, (bareFlatReports.keySet ++ bareFailedAnalysises.keySet ++ failedReportDownloads.keySet).map(_.fullId)) // TODO: consider renaming to projectsWithReports + @inline def flatReports: Map[ReportInfo, Analysis] = bareFlatReports // TODO: unify + @inline def projects = projectsReportInfo.projects + @inline def failedAnalysises: Map[ReportInfo, Throwable] = bareFailedAnalysises // TODO: unify lazy val failedProjects = FailedProjects.combineFails(parsingFailures = failedAnalysises, failedReportDownloads = failedReportDownloads) lazy val allDependencies = flatReports.toSeq.flatMap(r => r._2.dependencies.map(_ -> r._1)) lazy val groupedDependencies = allDependencies.groupBy(_._1.hashes).values.map(GroupedDependency(_)).toSeq @@ -124,7 +133,7 @@ object DependencyCheckReportsParser{ final class DependencyCheckReportsParser @Inject() (cache: CacheApi, projects: Projects) { - def parseReports(successfulResults: Map[String, (Build, ArtifactItem, ArtifactFile)], failedReportDownloads: Map[String, Throwable]) = { + def parseReports(successfulResults: Map[String, (Build, ArtifactItem, ArtifactFile)], failedReportDownloads: Map[String, Throwable]): Result = { val rid = math.random.toString // for logging @volatile var parseFailedForSomeAnalysis = false val deepReportsTriesIterable: Iterable[Map[String, Try[Analysis]]] = for((k, (build, data, log)) <- successfulResults) yield { @@ -155,7 +164,14 @@ final class DependencyCheckReportsParser @Inject() (cache: CacheApi, projects: P val failedAnalysises = deepReportsAndFailuresIterable.map(_._2).toSeq.flatten.toMap val flatReports = deepSuccessfulReports.flatten.toMap Logger.debug(s"[$rid] parse finished") - Result(flatReports, failedAnalysises, projects, failedReportDownloads = failedReportDownloads) + val projectReportInfo = new ProjectsWithReports(projects, flatReports.keySet++failedAnalysises.keySet++failedReportDownloads.keySet) + def convertKeys[T](m: Map[String, T]) = m.map{case (k, v) => projectReportInfo.reportIdToReportInfo(k) -> v} + Result( + convertKeys(flatReports), + convertKeys(failedAnalysises), + projectReportInfo, + failedReportDownloads = convertKeys(failedReportDownloads) + ) } } diff --git a/app/controllers/Notifications.scala b/app/controllers/Notifications.scala index bb94364..27c6d81 100644 --- a/app/controllers/Notifications.scala +++ b/app/controllers/Notifications.scala @@ -43,7 +43,7 @@ class Notifications @Inject()( myWatches <- myWatchesFuture } yield { val projects = dependencyCheckReportsParser.parseReports(successfulReports, failedReports).projectsReportInfo.sortedReportsInfo - Ok(views.html.notifications.index(projects, myWatches)) + Ok(views.html.notifications.index(projects, myWatches, failedReports.keySet)) } } @@ -111,7 +111,7 @@ class Notifications @Inject()( (successfulReports, failedReports) <- resultsFuture libraries <- librariesService.all parsedReports = dependencyCheckReportsParser.parseReports(successfulReports, failedReports) - lds = LibDepStatistics(dependencies = parsedReports.groupedDependencies.toSet, libraries = libraries.toSet, failedReportDownloads = failedReports, parsedReports = parsedReports) + lds = LibDepStatistics(dependencies = parsedReports.groupedDependencies.toSet, libraries = libraries.toSet, parsedReports = parsedReports) failed = lds.failedProjects failedReportsExportFuture = Fut(()) // TODO: exportFailedReports(lds, failed) issuesExportResultFuture = exportToIssueTracker(lds, failed, parsedReports.projectsReportInfo) diff --git a/app/controllers/ProjectsWithReports.scala b/app/controllers/ProjectsWithReports.scala index 24492b8..f59f2d4 100644 --- a/app/controllers/ProjectsWithReports.scala +++ b/app/controllers/ProjectsWithReports.scala @@ -1,6 +1,6 @@ package controllers -final case class ReportInfo( +final case class ReportInfo private[controllers] ( projectId: String, projectName: String, fullId: String, @@ -22,7 +22,12 @@ final case class ReportInfo( override def hashCode(): Int = 517+fullId.hashCode - def bare = copy(subprojectNameOption = None, fullId = fullId.takeWhile(_ != '/')) + def bare = ReportInfo( + fullId = fullId.takeWhile(_ != '/'), + projectId = projectId, + projectName = projectName, + subprojectNameOption = None + ) def isBare = subprojectNameOption.isEmpty def isNotBare = !isBare diff --git a/app/controllers/Statistics.scala b/app/controllers/Statistics.scala index 3fee34f..174118d 100644 --- a/app/controllers/Statistics.scala +++ b/app/controllers/Statistics.scala @@ -86,7 +86,7 @@ class Statistics @Inject() ( tagStatistics = tagStatistics, projectsWithSelection = selection.projectsWithSelection, parsedReports = parsedReports, - lds = LibDepStatistics(libraries.toSet, parsedReports.groupedDependencies.toSet, selection.result.failedReportDownloads, parsedReports) + lds = LibDepStatistics(libraries.toSet, parsedReports.groupedDependencies.toSet, parsedReports) )) } } @@ -112,7 +112,6 @@ class Statistics @Inject() ( stats = LibDepStatistics( libraries = tagLibraries, dependencies = tagDependencies, - failedReportDownloads = parsedReports.failedReportDownloads, parsedReports = parsedReports ) )) @@ -130,12 +129,11 @@ class Statistics @Inject() ( statistics <- tagOption.fold(Future.successful(LibDepStatistics( dependencies = parsedReports.groupedDependencies.toSet, libraries = libraries.values.toSet, - failedReportDownloads = selection.result.failedReportDownloads, parsedReports = parsedReports ))){ tag => statisticsForTags(parsedReports, Future.successful(Seq(tag))).map{ case Seq(TagStatistics(_, stats)) => stats // statisticsForTags is designed for multiple tags, but we have just one… - case Seq() => LibDepStatistics(libraries = Set(), dependencies = Set(), failedReportDownloads = selection.result.failedReportDownloads, parsedReports) // We don't want to crash when no dependencies are there… + case Seq() => LibDepStatistics(libraries = Set(), dependencies = Set(), parsedReports = parsedReports) // We don't want to crash when no dependencies are there… } } } yield Ok(views.html.statistics.vulnerabilities( @@ -162,7 +160,8 @@ class Statistics @Inject() ( vulnOption <- odcService.getVulnerabilityDetails(name) } yield Ok(views.html.statistics.vulnerabilityNotFound( // TODO: the not found page might be replaced by some page explaining that there is no project affected by that vulnerability name = name, - projectsWithSelection = selection.projectsWithSelection + projectsWithSelection = selection.projectsWithSelection, + failedProjects = selection.result.failedProjects )) }{ vulnerableDependencies => for { @@ -173,6 +172,7 @@ class Statistics @Inject() ( sys.error("The vulnerability is not in the database, you seem to have outdated the local vulnerability database") // TODO: consider fallback or more friendly error message }{vuln => Ok(views.html.statistics.vulnerability( vulnerability = vuln, + failedProjects = selection.result.failedProjects, affectedProjects = vulnerableDependencies.flatMap(dep => dep.projects.map(proj => (proj, dep))).groupBy(_._1).mapValues(_.map(_._2)), vulnerableDependencies = vulnerableDependencies, affectedLibraries = plainLibs, @@ -196,7 +196,8 @@ class Statistics @Inject() ( Future.successful(Ok(views.html.statistics.vulnerableLibraries( projectsWithSelection = selection.projectsWithSelection, vulnerableDependencies = reports.vulnerableDependencies, - allDependenciesCount = reports.groupedDependencies.size + allDependenciesCount = reports.groupedDependencies.size, + reports = reports ))) } } @@ -208,7 +209,8 @@ class Statistics @Inject() ( select(allResults, selectorOption).fold(Future.successful(notFound())){ selection => Future.successful(Ok(views.html.statistics.allLibraries( projectsWithSelection = selection.projectsWithSelection, - allDependencies = selection.result.groupedDependencies + allDependencies = selection.result.groupedDependencies, + failedProjects = selection.result.failedProjects ))) } } diff --git a/app/views/healthReport.scala.html b/app/views/healthReport.scala.html new file mode 100644 index 0000000..f31843b --- /dev/null +++ b/app/views/healthReport.scala.html @@ -0,0 +1,12 @@ +@import com.ysoft.odc.statistics.FailedProjects +@(failedProjects: FailedProjects) +@if(failedProjects.nonEmpty){ +
+ Some projects are excluded from the report, because there is some failure for them: + +
+} diff --git a/app/views/notifications/index.scala.html b/app/views/notifications/index.scala.html index 02106d6..3f0e94a 100644 --- a/app/views/notifications/index.scala.html +++ b/app/views/notifications/index.scala.html @@ -1,5 +1,5 @@ -@(projects: Seq[ReportInfo], watchedProjects: Set[String])(implicit req: DefaultRequest) @import helper._ +@(projects: Seq[ReportInfo], watchedProjects: Set[String], failedReports: Set[String])(implicit req: DefaultRequest) @button(action: Call)(label: String) = { @form(action, 'style -> "display: inline-block"){ @CSRF.formField @@ -13,6 +13,7 @@ @for( isWatchedDirectly <- Some(watchedProjects contains project.fullId); // hack allowing one to define a variable isWatchedByParent = project.isNotBare && (watchedProjects contains project.bare.fullId); + isFailed = failedReports contains project.fullId; // actually fullId should be equal bare id in such cases watchedChildCount = subprojects.count(p => watchedProjects contains p.fullId); hasWatchedChild = watchedChildCount > 0; hasButtons = !subprojects.isEmpty; @@ -31,6 +32,9 @@ } @friendlyProjectName(project) + @if(isFailed){ + (!) + } @if(project.isBare){ @if(isWatchedDirectly){ You watch this project with all subprojects. diff --git a/app/views/statistics/allLibraries.scala.html b/app/views/statistics/allLibraries.scala.html index a9e89ce..7b8ccae 100644 --- a/app/views/statistics/allLibraries.scala.html +++ b/app/views/statistics/allLibraries.scala.html @@ -1,12 +1,15 @@ +@import com.ysoft.odc.statistics.FailedProjects @( projectsWithSelection: ProjectsWithSelection, - allDependencies: Seq[GroupedDependency] + allDependencies: Seq[GroupedDependency], + failedProjects: FailedProjects )(implicit header: DefaultRequest) @main( title = s"All libraries for ${projectsWithSelection.projectNameText}", projectsOption = Some((projectsWithSelection, routes.Statistics.allLibraries(_))) ){ + @healthReport(failedProjects) @dependencyList( "all", allDependencies.sortBy(_.identifiers.toIndexedSeq.sortBy(i => (i.confidence.id, i.identifierType, i.name)).mkString(", ")), diff --git a/app/views/statistics/basic.scala.html b/app/views/statistics/basic.scala.html index 9d870ac..0d59ed3 100644 --- a/app/views/statistics/basic.scala.html +++ b/app/views/statistics/basic.scala.html @@ -42,7 +42,7 @@ headExtension = he, projectsOption = Some((projectsWithSelection, routes.Statistics.basic(_))) ){ - + @healthReport(lds.failedProjects) All dependencies: @parsedReports.groupedDependencies.size
Vulnerable dependencies: @parsedReports.vulnerableDependencies.size
Vulnerabilities: @parsedReports.vulnerableDependencies.flatMap(_.vulnerabilities.map(_.name)).toSet.size
diff --git a/app/views/statistics/vulnerabilities.scala.html b/app/views/statistics/vulnerabilities.scala.html index 6e188d9..b36effa 100644 --- a/app/views/statistics/vulnerabilities.scala.html +++ b/app/views/statistics/vulnerabilities.scala.html @@ -8,6 +8,7 @@ title = s"details for ${projectsWithSelection.projectNameText}${tagOption.map(_._2.name).fold("")(" and tag "+_)}", projectsOption = Some((projectsWithSelection, x => routes.Statistics.vulnerabilities(x, tagOption.map(_._1)))) ){ + @healthReport(statistics.failedProjects) We have @statistics.vulnerabilitiesToDependencies.size vulnerabilities of @statistics.vulnerabilitiesToDependencies.flatMap(_._2).toSet.size dependencies (@statistics.vulnerabilitiesToDependencies.flatMap(_._2.flatMap(_.plainLibraryIdentifiers)).toSet.size libraries). @if(!projectsWithSelection.isProjectSpecified){ diff --git a/app/views/statistics/vulnerability.scala.html b/app/views/statistics/vulnerability.scala.html index 3f727d4..5c1e402 100644 --- a/app/views/statistics/vulnerability.scala.html +++ b/app/views/statistics/vulnerability.scala.html @@ -1,16 +1,19 @@ +@import com.ysoft.odc.statistics.FailedProjects @( projectsWithSelection: ProjectsWithSelection, vulnerability: Vulnerability, affectedProjects: Map[ReportInfo, Set[GroupedDependency]], vulnerableDependencies: Set[GroupedDependency], affectedLibraries: Set[PlainLibraryIdentifier], - issueOption: Option[(ExportedVulnerability[String], String)] + issueOption: Option[(ExportedVulnerability[String], String)], + failedProjects: FailedProjects )(implicit header: DefaultRequest) @section = @{views.html.genericSection("vuln")("h2") _} @main( title = s"vulnerability ${vulnerability.name} for ${projectsWithSelection.projectNameText}", projectsOption = Some((projectsWithSelection, p => routes.Statistics.vulnerability(vulnerability.name, p))) ) { + @healthReport(failedProjects) @if(projectsWithSelection.isProjectSpecified){
The vulnerability details are limited to some subset of projects.
Show it for all projects!
} diff --git a/app/views/statistics/vulnerabilityNotFound.scala.html b/app/views/statistics/vulnerabilityNotFound.scala.html index 616d45c..ffbcaf6 100644 --- a/app/views/statistics/vulnerabilityNotFound.scala.html +++ b/app/views/statistics/vulnerabilityNotFound.scala.html @@ -1,12 +1,15 @@ +@import com.ysoft.odc.statistics.FailedProjects @( projectsWithSelection: ProjectsWithSelection, - name: String + name: String, + failedProjects: FailedProjects )(implicit header: DefaultRequest) @main( title = s"Unknown vulnerability $name for ${projectsWithSelection.projectNameText}", projectsOption = Some((projectsWithSelection, p => routes.Statistics.vulnerability(name, p))) ){ + @healthReport(failedProjects)
Vulnerability @name is not found@if(projectsWithSelection.isProjectSpecified){ for selected project(s)}.

Possible solutions