Added notice when there are some failed projects. This required some refactorings and fixes, that are also included in this commit.

This commit is contained in:
Šesták Vít
2016-05-13 19:31:47 +02:00
parent 604fc56d76
commit 6c8b2cf859
14 changed files with 104 additions and 51 deletions

View File

@@ -2,22 +2,27 @@ package com.ysoft.odc.statistics
import controllers.ReportInfo
final class FailedProjects(val failedProjectsSet: Set[String]){
final class FailedProjects(val failedProjectsSet: Set[ReportInfo]){
val failedProjectIdsSet = failedProjectsSet.map(_.projectId)
def nonEmpty: Boolean = failedProjectsSet.nonEmpty
def isFailed(projectFullId: String): Boolean = {
val projectBareId = projectFullId.takeWhile(_ != '/')
failedProjectsSet contains projectBareId
failedProjectIdsSet contains projectBareId
}
}
object FailedProjects {
def combineFails(failedReportDownloads: Map[String, Throwable], parsingFailures: Map[ReportInfo, Throwable]): FailedProjects = {
def combineFails(failedReportDownloads: Map[ReportInfo, Throwable], parsingFailures: Map[ReportInfo, Throwable]): FailedProjects = {
/*
Fail can happen at multiple places:
1. Build cannot be downloaded (auth error, connection error, …) or is failed (failedReportDownloads)
2. Build is successful and can be downloaded, but it cannot be parsed (parsingFailures)
*/
val failedProjectsSet = failedReportDownloads.keySet ++ parsingFailures.keySet.map(_.projectId)
val failedProjectsSet = failedReportDownloads.keySet ++ parsingFailures.keySet
new FailedProjects(failedProjectsSet)
}
}

View File

@@ -23,12 +23,9 @@ case class LibDepStatistics(libraries: Set[(Int, Library)], dependencies: Set[Gr
object LibDepStatistics{
private def computeWeaknessesFrequency(vulnerabilities: Set[Vulnerability]) = vulnerabilities.toSeq.map(_.cweOption).groupBy(identity).mapValues(_.size).map(identity).withDefaultValue(0)
def apply(libraries: Set[(Int, Library)], dependencies: Set[GroupedDependency], failedReportDownloads: Map[String, Throwable], parsedReports: Result): LibDepStatistics = LibDepStatistics(
def apply(libraries: Set[(Int, Library)], dependencies: Set[GroupedDependency], parsedReports: Result): LibDepStatistics = LibDepStatistics(
libraries = libraries,
dependencies = dependencies,
failedProjects = FailedProjects.combineFails(
failedReportDownloads = failedReportDownloads,
parsingFailures = parsedReports.failedAnalysises
)
failedProjects = parsedReports.failedProjects
)
}

View File

@@ -26,17 +26,19 @@ private final case class ProjectFilter(project: ReportInfo) extends Filter{
override def descriptionText: String = s"project ${friendlyProjectNameString(project)}"
override def subReports(r: Result): Option[Result] = {
@inline def reportInfo = project
def f[T](m: Map[ReportInfo, T]): Map[String, T] = (
if(reportInfo.subprojectNameOption.isEmpty) m.filter(_._1.projectId == project.projectId) else m.get(reportInfo).fold(Map.empty[ReportInfo, T])(x => Map(reportInfo -> x))
).map{case (k, v) => k.fullId -> v}
val newFlatReports = f(r.flatReports)
val newFailedAnalysises = f(r.failedAnalysises)
if(newFlatReports.isEmpty && newFailedAnalysises.isEmpty) None
def filter[T](m: Map[ReportInfo, T]): Map[ReportInfo, T] = (
if(reportInfo.subprojectNameOption.isEmpty) m.filter(_._1.projectId == project.projectId)
else m.get(reportInfo).fold(Map.empty[ReportInfo, T])(x => Map(reportInfo -> x))
)
val newFlatReports = filter(r.flatReports)
val newFailedAnalysises = filter(r.failedAnalysises)
val newFailedReportDownloads = filter(r.failedReportDownloads)
if(newFlatReports.isEmpty && newFailedAnalysises.isEmpty && newFailedReportDownloads.isEmpty) None
else Some(Result(
bareFlatReports = newFlatReports,
bareFailedAnalysises = newFailedAnalysises,
projects = r.projects,
failedReportDownloads = r.failedReportDownloads // TODO: consider filtering of failedReportDownloads
projectsReportInfo = r.projectsReportInfo,
failedReportDownloads = newFailedReportDownloads
))
}
override def selector = Some(s"project:${project.fullId}")
@@ -45,27 +47,33 @@ private final case class TeamFilter(team: Team) extends Filter{
override def filters: Boolean = true
override def subReports(r: Result): Option[Result] = {
val Wildcard = """^(.*): \*$""".r
val reportInfoByFriendlyProjectNameMap = r.projectsReportInfo.ungroupedReportsInfo.map(ri => friendlyProjectNameString(ri) -> ri).toSeq.groupBy(_._1).mapValues{
case Seq((_, ri)) => ri
@inline def toMapStrict[K, V](l: Traversable[(K, V)]) = l.toSeq.groupBy(_._1).mapValues{ // without toSeq, the pattern matching might fail
case Seq((_, v)) => v
case other => sys.error("some duplicate value: "+other)
}.map(identity)
val reportInfoByFriendlyProjectNameMap = toMapStrict(r.projectsReportInfo.ungroupedReportsInfo.map(ri => friendlyProjectNameString(ri) -> ri))
val ProjectName = """^(.*): (.*)$""".r
val failedProjectsFriendlyNames = r.failedProjects.failedProjectsSet.map(r.projectsReportInfo.parseUnfriendlyName).map(_.projectName)
println(failedProjectsFriendlyNames)
val rootProjectReports = reportInfoByFriendlyProjectNameMap.collect{ case (ProjectName(rootProject, subproject), v) =>
(rootProject, v)
val failedProjectsFriendlyNames = r.failedProjects.failedProjectsSet.map(_.projectName)
Logger.error("failedProjectsFriendlyNames: "+failedProjectsFriendlyNames)
val rootProjectReports = reportInfoByFriendlyProjectNameMap.map{
case (ProjectName(rootProject, _subproject), v) => (rootProject, v)
case value @ (rootProject, v) => value
}.groupBy(_._1).mapValues(_.values).withDefault(name =>
if(failedProjectsFriendlyNames contains name) Seq()
else sys.error("Unknown project: "+name)
)
def reportInfoByFriendlyProjectName(fpn: String) = reportInfoByFriendlyProjectNameMap.get(fpn).map(Set(_)).getOrElse(rootProjectReports(fpn.takeWhile(_ != ':')))
def reportInfoByFriendlyProjectName(fpn: String) = fpn match{
case Wildcard(rfpn) => rootProjectReports(rfpn)
case _ => Set(reportInfoByFriendlyProjectNameMap(fpn))
}
val reportInfos = team.projectNames.flatMap(reportInfoByFriendlyProjectName)
def submap[T](m: Map[String, T]) = reportInfos.toSeq.flatMap(ri => m.get(ri.fullId).map(ri.fullId -> _) ).toMap
def submap[T](m: Map[ReportInfo, T]) = reportInfos.toSeq.flatMap(ri => m.get(ri).map(ri -> _) ).toMap
def submapBare[T](m: Map[ReportInfo, T]): Map[ReportInfo, T] = reportInfos.toSeq.flatMap(ri => m.get(ri.bare.ensuring{x => println(x.fullId); true}).map(ri -> _) ).toMap
Some(Result(
bareFlatReports = submap(r.bareFlatReports),
bareFailedAnalysises = submap(r.bareFailedAnalysises),
projects = r.projects,
failedReportDownloads = r.failedReportDownloads // TODO: consider filtering of failedReportDownloads
bareFailedAnalysises = submapBare(r.bareFailedAnalysises),
projectsReportInfo = r.projectsReportInfo,
failedReportDownloads = submapBare(r.failedReportDownloads)
))
}
override def descriptionHtml: Html = views.html.filters.team(team.id)
@@ -89,10 +97,11 @@ private final case class BadFilter(pattern: String) extends Filter{
object DependencyCheckReportsParser{
final case class ResultWithSelection(result: Result, projectsWithSelection: ProjectsWithSelection)
final case class Result(bareFlatReports: Map[String, Analysis], bareFailedAnalysises: Map[String, Throwable], projects: Projects /*TODO: maybe rename to rootProjects*/, failedReportDownloads: Map[String, Throwable]){
lazy val projectsReportInfo = new ProjectsWithReports(projects, bareFlatReports.keySet ++ bareFailedAnalysises.keySet ++ failedReportDownloads.keySet) // TODO: consider renaming to projectsWithReports
lazy val flatReports: Map[ReportInfo, Analysis] = bareFlatReports.map{case (k, v) => projectsReportInfo.reportIdToReportInfo(k) -> v}
lazy val failedAnalysises: Map[ReportInfo, Throwable] = bareFailedAnalysises.map{case (k, v) => projectsReportInfo.reportIdToReportInfo(k) -> v}
final case class Result(bareFlatReports: Map[ReportInfo, Analysis], bareFailedAnalysises: Map[ReportInfo, Throwable], projectsReportInfo: ProjectsWithReports/*TODO: maybe rename to rootProjects*/, failedReportDownloads: Map[ReportInfo, Throwable]){
//lazy val projectsReportInfo = new ProjectsWithReports(projects, (bareFlatReports.keySet ++ bareFailedAnalysises.keySet ++ failedReportDownloads.keySet).map(_.fullId)) // TODO: consider renaming to projectsWithReports
@inline def flatReports: Map[ReportInfo, Analysis] = bareFlatReports // TODO: unify
@inline def projects = projectsReportInfo.projects
@inline def failedAnalysises: Map[ReportInfo, Throwable] = bareFailedAnalysises // TODO: unify
lazy val failedProjects = FailedProjects.combineFails(parsingFailures = failedAnalysises, failedReportDownloads = failedReportDownloads)
lazy val allDependencies = flatReports.toSeq.flatMap(r => r._2.dependencies.map(_ -> r._1))
lazy val groupedDependencies = allDependencies.groupBy(_._1.hashes).values.map(GroupedDependency(_)).toSeq
@@ -124,7 +133,7 @@ object DependencyCheckReportsParser{
final class DependencyCheckReportsParser @Inject() (cache: CacheApi, projects: Projects) {
def parseReports(successfulResults: Map[String, (Build, ArtifactItem, ArtifactFile)], failedReportDownloads: Map[String, Throwable]) = {
def parseReports(successfulResults: Map[String, (Build, ArtifactItem, ArtifactFile)], failedReportDownloads: Map[String, Throwable]): Result = {
val rid = math.random.toString // for logging
@volatile var parseFailedForSomeAnalysis = false
val deepReportsTriesIterable: Iterable[Map[String, Try[Analysis]]] = for((k, (build, data, log)) <- successfulResults) yield {
@@ -155,7 +164,14 @@ final class DependencyCheckReportsParser @Inject() (cache: CacheApi, projects: P
val failedAnalysises = deepReportsAndFailuresIterable.map(_._2).toSeq.flatten.toMap
val flatReports = deepSuccessfulReports.flatten.toMap
Logger.debug(s"[$rid] parse finished")
Result(flatReports, failedAnalysises, projects, failedReportDownloads = failedReportDownloads)
val projectReportInfo = new ProjectsWithReports(projects, flatReports.keySet++failedAnalysises.keySet++failedReportDownloads.keySet)
def convertKeys[T](m: Map[String, T]) = m.map{case (k, v) => projectReportInfo.reportIdToReportInfo(k) -> v}
Result(
convertKeys(flatReports),
convertKeys(failedAnalysises),
projectReportInfo,
failedReportDownloads = convertKeys(failedReportDownloads)
)
}
}

View File

@@ -43,7 +43,7 @@ class Notifications @Inject()(
myWatches <- myWatchesFuture
} yield {
val projects = dependencyCheckReportsParser.parseReports(successfulReports, failedReports).projectsReportInfo.sortedReportsInfo
Ok(views.html.notifications.index(projects, myWatches))
Ok(views.html.notifications.index(projects, myWatches, failedReports.keySet))
}
}
@@ -111,7 +111,7 @@ class Notifications @Inject()(
(successfulReports, failedReports) <- resultsFuture
libraries <- librariesService.all
parsedReports = dependencyCheckReportsParser.parseReports(successfulReports, failedReports)
lds = LibDepStatistics(dependencies = parsedReports.groupedDependencies.toSet, libraries = libraries.toSet, failedReportDownloads = failedReports, parsedReports = parsedReports)
lds = LibDepStatistics(dependencies = parsedReports.groupedDependencies.toSet, libraries = libraries.toSet, parsedReports = parsedReports)
failed = lds.failedProjects
failedReportsExportFuture = Fut(()) // TODO: exportFailedReports(lds, failed)
issuesExportResultFuture = exportToIssueTracker(lds, failed, parsedReports.projectsReportInfo)

View File

@@ -1,6 +1,6 @@
package controllers
final case class ReportInfo(
final case class ReportInfo private[controllers] (
projectId: String,
projectName: String,
fullId: String,
@@ -22,7 +22,12 @@ final case class ReportInfo(
override def hashCode(): Int = 517+fullId.hashCode
def bare = copy(subprojectNameOption = None, fullId = fullId.takeWhile(_ != '/'))
def bare = ReportInfo(
fullId = fullId.takeWhile(_ != '/'),
projectId = projectId,
projectName = projectName,
subprojectNameOption = None
)
def isBare = subprojectNameOption.isEmpty
def isNotBare = !isBare

View File

@@ -86,7 +86,7 @@ class Statistics @Inject() (
tagStatistics = tagStatistics,
projectsWithSelection = selection.projectsWithSelection,
parsedReports = parsedReports,
lds = LibDepStatistics(libraries.toSet, parsedReports.groupedDependencies.toSet, selection.result.failedReportDownloads, parsedReports)
lds = LibDepStatistics(libraries.toSet, parsedReports.groupedDependencies.toSet, parsedReports)
))
}
}
@@ -112,7 +112,6 @@ class Statistics @Inject() (
stats = LibDepStatistics(
libraries = tagLibraries,
dependencies = tagDependencies,
failedReportDownloads = parsedReports.failedReportDownloads,
parsedReports = parsedReports
)
))
@@ -130,12 +129,11 @@ class Statistics @Inject() (
statistics <- tagOption.fold(Future.successful(LibDepStatistics(
dependencies = parsedReports.groupedDependencies.toSet,
libraries = libraries.values.toSet,
failedReportDownloads = selection.result.failedReportDownloads,
parsedReports = parsedReports
))){ tag =>
statisticsForTags(parsedReports, Future.successful(Seq(tag))).map{
case Seq(TagStatistics(_, stats)) => stats // statisticsForTags is designed for multiple tags, but we have just one…
case Seq() => LibDepStatistics(libraries = Set(), dependencies = Set(), failedReportDownloads = selection.result.failedReportDownloads, parsedReports) // We don't want to crash when no dependencies are there…
case Seq() => LibDepStatistics(libraries = Set(), dependencies = Set(), parsedReports = parsedReports) // We don't want to crash when no dependencies are there…
}
}
} yield Ok(views.html.statistics.vulnerabilities(
@@ -162,7 +160,8 @@ class Statistics @Inject() (
vulnOption <- odcService.getVulnerabilityDetails(name)
} yield Ok(views.html.statistics.vulnerabilityNotFound( // TODO: the not found page might be replaced by some page explaining that there is no project affected by that vulnerability
name = name,
projectsWithSelection = selection.projectsWithSelection
projectsWithSelection = selection.projectsWithSelection,
failedProjects = selection.result.failedProjects
))
}{ vulnerableDependencies =>
for {
@@ -173,6 +172,7 @@ class Statistics @Inject() (
sys.error("The vulnerability is not in the database, you seem to have outdated the local vulnerability database") // TODO: consider fallback or more friendly error message
}{vuln => Ok(views.html.statistics.vulnerability(
vulnerability = vuln,
failedProjects = selection.result.failedProjects,
affectedProjects = vulnerableDependencies.flatMap(dep => dep.projects.map(proj => (proj, dep))).groupBy(_._1).mapValues(_.map(_._2)),
vulnerableDependencies = vulnerableDependencies,
affectedLibraries = plainLibs,
@@ -196,7 +196,8 @@ class Statistics @Inject() (
Future.successful(Ok(views.html.statistics.vulnerableLibraries(
projectsWithSelection = selection.projectsWithSelection,
vulnerableDependencies = reports.vulnerableDependencies,
allDependenciesCount = reports.groupedDependencies.size
allDependenciesCount = reports.groupedDependencies.size,
reports = reports
)))
}
}
@@ -208,7 +209,8 @@ class Statistics @Inject() (
select(allResults, selectorOption).fold(Future.successful(notFound())){ selection =>
Future.successful(Ok(views.html.statistics.allLibraries(
projectsWithSelection = selection.projectsWithSelection,
allDependencies = selection.result.groupedDependencies
allDependencies = selection.result.groupedDependencies,
failedProjects = selection.result.failedProjects
)))
}
}

View File

@@ -0,0 +1,12 @@
@import com.ysoft.odc.statistics.FailedProjects
@(failedProjects: FailedProjects)
@if(failedProjects.nonEmpty){
<div class="alert alert-danger">
Some projects are excluded from the report, because there is some failure for them:
<ul>
@for(proj <- failedProjects.failedProjectsSet.toIndexedSeq.sortBy(_.projectName.toLowerCase)){
<li>@proj.projectName</li>
}
</ul>
</div>
}

View File

@@ -1,5 +1,5 @@
@(projects: Seq[ReportInfo], watchedProjects: Set[String])(implicit req: DefaultRequest)
@import helper._
@(projects: Seq[ReportInfo], watchedProjects: Set[String], failedReports: Set[String])(implicit req: DefaultRequest)
@button(action: Call)(label: String) = {
@form(action, 'style -> "display: inline-block"){
@CSRF.formField
@@ -13,6 +13,7 @@
@for(
isWatchedDirectly <- Some(watchedProjects contains project.fullId); // hack allowing one to define a variable
isWatchedByParent = project.isNotBare && (watchedProjects contains project.bare.fullId);
isFailed = failedReports contains project.fullId; // actually fullId should be equal bare id in such cases
watchedChildCount = subprojects.count(p => watchedProjects contains p.fullId);
hasWatchedChild = watchedChildCount > 0;
hasButtons = !subprojects.isEmpty;
@@ -31,6 +32,9 @@
</span>
}
@friendlyProjectName(project)
@if(isFailed){
(!)
}
@if(project.isBare){
@if(isWatchedDirectly){
<span class="badge">You watch this project with all subprojects.</span>

View File

@@ -1,12 +1,15 @@
@import com.ysoft.odc.statistics.FailedProjects
@(
projectsWithSelection: ProjectsWithSelection,
allDependencies: Seq[GroupedDependency]
allDependencies: Seq[GroupedDependency],
failedProjects: FailedProjects
)(implicit header: DefaultRequest)
@main(
title = s"All libraries for ${projectsWithSelection.projectNameText}",
projectsOption = Some((projectsWithSelection, routes.Statistics.allLibraries(_)))
){
@healthReport(failedProjects)
@dependencyList(
"all",
allDependencies.sortBy(_.identifiers.toIndexedSeq.sortBy(i => (i.confidence.id, i.identifierType, i.name)).mkString(", ")),

View File

@@ -42,7 +42,7 @@
headExtension = he,
projectsOption = Some((projectsWithSelection, routes.Statistics.basic(_)))
){
@healthReport(lds.failedProjects)
All dependencies: @parsedReports.groupedDependencies.size <br>
Vulnerable dependencies: @parsedReports.vulnerableDependencies.size <br>
Vulnerabilities: @parsedReports.vulnerableDependencies.flatMap(_.vulnerabilities.map(_.name)).toSet.size<br>

View File

@@ -8,6 +8,7 @@
title = s"details for ${projectsWithSelection.projectNameText}${tagOption.map(_._2.name).fold("")(" and tag "+_)}",
projectsOption = Some((projectsWithSelection, x => routes.Statistics.vulnerabilities(x, tagOption.map(_._1))))
){
@healthReport(statistics.failedProjects)
We have @statistics.vulnerabilitiesToDependencies.size vulnerabilities
of @statistics.vulnerabilitiesToDependencies.flatMap(_._2).toSet.size dependencies (@statistics.vulnerabilitiesToDependencies.flatMap(_._2.flatMap(_.plainLibraryIdentifiers)).toSet.size libraries).
@if(!projectsWithSelection.isProjectSpecified){

View File

@@ -1,16 +1,19 @@
@import com.ysoft.odc.statistics.FailedProjects
@(
projectsWithSelection: ProjectsWithSelection,
vulnerability: Vulnerability,
affectedProjects: Map[ReportInfo, Set[GroupedDependency]],
vulnerableDependencies: Set[GroupedDependency],
affectedLibraries: Set[PlainLibraryIdentifier],
issueOption: Option[(ExportedVulnerability[String], String)]
issueOption: Option[(ExportedVulnerability[String], String)],
failedProjects: FailedProjects
)(implicit header: DefaultRequest)
@section = @{views.html.genericSection("vuln")("h2") _}
@main(
title = s"vulnerability ${vulnerability.name} for ${projectsWithSelection.projectNameText}",
projectsOption = Some((projectsWithSelection, p => routes.Statistics.vulnerability(vulnerability.name, p)))
) {
@healthReport(failedProjects)
@if(projectsWithSelection.isProjectSpecified){
<div class="alert alert-warning">The vulnerability details are limited to some subset of projects.<br><a class="btn btn-default" href="@routes.Statistics.vulnerability(vulnerability.name, None)">Show it for all projects!</a></div>
}

View File

@@ -1,12 +1,15 @@
@import com.ysoft.odc.statistics.FailedProjects
@(
projectsWithSelection: ProjectsWithSelection,
name: String
name: String,
failedProjects: FailedProjects
)(implicit header: DefaultRequest)
@main(
title = s"Unknown vulnerability $name for ${projectsWithSelection.projectNameText}",
projectsOption = Some((projectsWithSelection, p => routes.Statistics.vulnerability(name, p)))
){
@healthReport(failedProjects)
<div class="alert alert-warning">Vulnerability <i>@name</i> is not found@if(projectsWithSelection.isProjectSpecified){ for selected project(s)}.</div>
<h2>Possible solutions</h2>
<ul class="solutions">

View File

@@ -1,13 +1,15 @@
@(
projectsWithSelection: ProjectsWithSelection,
vulnerableDependencies: Seq[GroupedDependency],
allDependenciesCount: Int
allDependenciesCount: Int,
reports: DependencyCheckReportsParser.Result
)(implicit header: DefaultRequest)
@main(
title = s"Vulnerable libraries for ${projectsWithSelection.projectNameText} (${vulnerableDependencies.size} deps, ${vulnerableDependencies.flatMap(_.cpeIdentifiers.map(_.toCpeIdentifierOption.get)).toSet.size} CPEs)",
projectsOption = Some((projectsWithSelection, routes.Statistics.vulnerableLibraries(_)))
){
@healthReport(reports.failedProjects)
<script type="text/javascript" src="@routes.Assets.versioned("lib/jqplot/jquery.jqplot.min.js")"></script>
<script type="text/javascript" src="@routes.Assets.versioned("lib/jqplot/plugins/jqplot.pieRenderer.min.js")"></script>
<h2>Plot</h2>