Fixed error when filter should contain a failed project

app/com/ysoft/odc/statistics/FailedProjects.scala

@@ -11,7 +11,7 @@ final class FailedProjects(val failedProjectsSet: Set[String]){
 }
 
 object FailedProjects {
-  private[statistics] def combineFails(failedReportDownloads: Map[String, Throwable], parsingFailures: Map[ReportInfo, Throwable]): FailedProjects = {
+  def combineFails(failedReportDownloads: Map[String, Throwable], parsingFailures: Map[ReportInfo, Throwable]): FailedProjects = {
     /*
     Fail can happen at multiple places:
     1. Build cannot be downloaded (auth error, connection error, …) or is failed (failedReportDownloads)

app/controllers/DependencyCheckReportsParser.scala

@@ -4,6 +4,7 @@ import java.net.URLEncoder
 
 import com.google.inject.Inject
 import com.ysoft.odc._
+import com.ysoft.odc.statistics.FailedProjects
 import controllers.DependencyCheckReportsParser.Result
 import models.PlainLibraryIdentifier
 import play.api.Logger
@@ -31,7 +32,12 @@ private final case class ProjectFilter(project: ReportInfo) extends Filter{
     val newFlatReports = f(r.flatReports)
     val newFailedAnalysises = f(r.failedAnalysises)
     if(newFlatReports.isEmpty && newFailedAnalysises.isEmpty) None
-    else Some(Result(bareFlatReports = newFlatReports, bareFailedAnalysises = newFailedAnalysises, projects = r.projects))
+    else Some(Result(
+      bareFlatReports = newFlatReports,
+      bareFailedAnalysises = newFailedAnalysises,
+      projects = r.projects,
+      failedReportDownloads = r.failedReportDownloads // TODO: consider filtering of failedReportDownloads
+    ))
   }
   override def selector = Some(s"project:${project.fullId}")
 }
@@ -44,14 +50,22 @@ private final case class TeamFilter(team: Team) extends Filter{
       case other => sys.error("some duplicate value: "+other)
     }.map(identity)
     val ProjectName = """^(.*): (.*)$""".r
-    val rootProjectReports = reportInfoByFriendlyProjectNameMap.collect{case (ProjectName(rootProject, subproject), v) => (rootProject, v)}.groupBy(_._1).mapValues(_.map(_._2))
+    val failedProjectsFriendlyNames = r.failedProjects.failedProjectsSet.map(r.projectsReportInfo.parseUnfriendlyName).map(_.projectName)
+    println(failedProjectsFriendlyNames)
+    val rootProjectReports = reportInfoByFriendlyProjectNameMap.collect{ case (ProjectName(rootProject, subproject), v) =>
+      (rootProject, v)
+    }.groupBy(_._1).mapValues(_.values).withDefault(name =>
+      if(failedProjectsFriendlyNames contains name) Seq()
+      else sys.error("Unknown project: "+name)
+    )
     def reportInfoByFriendlyProjectName(fpn: String) = reportInfoByFriendlyProjectNameMap.get(fpn).map(Set(_)).getOrElse(rootProjectReports(fpn.takeWhile(_ != ':')))
     val reportInfos = team.projectNames.flatMap(reportInfoByFriendlyProjectName)
     def submap[T](m: Map[String, T]) = reportInfos.toSeq.flatMap(ri => m.get(ri.fullId).map(ri.fullId -> _) ).toMap
     Some(Result(
       bareFlatReports = submap(r.bareFlatReports),
       bareFailedAnalysises = submap(r.bareFailedAnalysises),
-      projects = r.projects
+      projects = r.projects,
+      failedReportDownloads = r.failedReportDownloads // TODO: consider filtering of failedReportDownloads
     ))
   }
   override def descriptionHtml: Html = views.html.filters.team(team.id)
@@ -75,10 +89,11 @@ private final case class BadFilter(pattern: String) extends Filter{
 
 object DependencyCheckReportsParser{
   final case class ResultWithSelection(result: Result, projectsWithSelection: ProjectsWithSelection)
-  final case class Result(bareFlatReports: Map[String, Analysis], bareFailedAnalysises: Map[String, Throwable], projects: Projects /*TODO: maybe rename to rootProjects*/){
+  final case class Result(bareFlatReports: Map[String, Analysis], bareFailedAnalysises: Map[String, Throwable], projects: Projects /*TODO: maybe rename to rootProjects*/, failedReportDownloads: Map[String, Throwable]){
     lazy val projectsReportInfo = new ProjectsWithReports(projects, bareFlatReports.keySet ++ bareFailedAnalysises.keySet) // TODO: consider renaming to projectsWithReports
     lazy val flatReports: Map[ReportInfo, Analysis] = bareFlatReports.map{case (k, v) => projectsReportInfo.reportIdToReportInfo(k) -> v}
     lazy val failedAnalysises: Map[ReportInfo, Throwable] = bareFailedAnalysises.map{case (k, v) => projectsReportInfo.reportIdToReportInfo(k) -> v}
+    lazy val failedProjects = FailedProjects.combineFails(parsingFailures = failedAnalysises, failedReportDownloads = failedReportDownloads)
     lazy val allDependencies = flatReports.toSeq.flatMap(r => r._2.dependencies.map(_ -> r._1))
     lazy val groupedDependencies = allDependencies.groupBy(_._1.hashes).values.map(GroupedDependency(_)).toSeq
     lazy val groupedDependenciesByPlainLibraryIdentifier: Map[PlainLibraryIdentifier, Set[GroupedDependency]] =
@@ -109,7 +124,7 @@ object DependencyCheckReportsParser{
 
 final class DependencyCheckReportsParser @Inject() (cache: CacheApi, projects: Projects) {
 
-  def parseReports(successfulResults: Map[String, (Build, ArtifactItem, ArtifactFile)]) = {
+  def parseReports(successfulResults: Map[String, (Build, ArtifactItem, ArtifactFile)], failedReportDownloads: Map[String, Throwable]) = {
     val rid = math.random.toString  // for logging
     @volatile var parseFailedForSomeAnalysis = false
     val deepReportsTriesIterable: Iterable[Map[String, Try[Analysis]]] = for((k, (build, data, log)) <- successfulResults) yield {
@@ -140,7 +155,7 @@ final class DependencyCheckReportsParser @Inject() (cache: CacheApi, projects: P
     val failedAnalysises = deepReportsAndFailuresIterable.map(_._2).toSeq.flatten.toMap
     val flatReports = deepSuccessfulReports.flatten.toMap
     Logger.debug(s"[$rid] parse finished")
-    Result(flatReports, failedAnalysises, projects)
+    Result(flatReports, failedAnalysises, projects, failedReportDownloads = failedReportDownloads)
   }
 
 }

app/controllers/DependencyCheckReportsProcessor.scala

@@ -46,7 +46,7 @@ final class DependencyCheckReportsProcessor @Inject() (
     requiredVersions: Map[String, Int]
   )(implicit requestHeader: DefaultRequest, snoozesInfo: SnoozesInfo, executionContext: ExecutionContext) = try{
     for((successfulResults, failedResults) <- resultsFuture) yield{
-      val reportResult = dependencyCheckReportsParser.parseReports(successfulResults)
+      val reportResult = dependencyCheckReportsParser.parseReports(successfulResults, failedResults)
       import reportResult.{allDependencies, failedAnalysises, flatReports, groupedDependencies, vulnerableDependencies}
       val now = DateTime.now
       val oldReportThreshold = now - 1.day

app/controllers/Notifications.scala

@@ -42,7 +42,7 @@ class Notifications @Inject()(
       (successfulReports, failedReports) <- resultsFuture
       myWatches <- myWatchesFuture
     } yield {
-      val projects = dependencyCheckReportsParser.parseReports(successfulReports).projectsReportInfo.sortedReportsInfo
+      val projects = dependencyCheckReportsParser.parseReports(successfulReports, failedReports).projectsReportInfo.sortedReportsInfo
       Ok(views.html.notifications.index(projects, myWatches))
     }
   }
@@ -110,7 +110,7 @@ class Notifications @Inject()(
           // TODO: process failedReports, parsedReports.failedAnalysises and successfulResults.filter(x => x._2._1.state != "Successful" || x._2._1.buildState != "Successful")
           (successfulReports, failedReports) <- resultsFuture
           libraries <- librariesService.all
-          parsedReports = dependencyCheckReportsParser.parseReports(successfulReports)
+          parsedReports = dependencyCheckReportsParser.parseReports(successfulReports, failedReports)
           lds = LibDepStatistics(dependencies = parsedReports.groupedDependencies.toSet, libraries = libraries.toSet, failedReportDownloads = failedReports, parsedReports = parsedReports)
           failed = lds.failedProjects
           failedReportsExportFuture = Fut(()) // TODO: exportFailedReports(lds, failed)

app/controllers/Statistics.scala

@@ -5,6 +5,7 @@ import com.google.inject.Inject
 import com.google.inject.name.Named
 import com.ysoft.odc.statistics.{LibDepStatistics, TagStatistics}
 import com.ysoft.odc.{ArtifactFile, ArtifactItem}
+import controllers.DependencyCheckReportsParser.ResultWithSelection
 import models.LibraryTag
 import org.joda.time.DateTime
 import play.api.i18n.MessagesApi
@@ -39,7 +40,8 @@ class Statistics @Inject() (
   import secureRequestConversion._
 
 
-  private def select(successfulResults: Map[String, (Build, ArtifactItem, ArtifactFile)], selectorOption: Option[String]) = dependencyCheckReportsParser.parseReports(successfulResults).selection(selectorOption)
+  private def select(allResults: (Map[String, (Build, ArtifactItem, ArtifactFile)], Map[String, Throwable]), selectorOption: Option[String]): Option[ResultWithSelection] = select(allResults._1, allResults._2, selectorOption)
+  private def select(successfulResults: Map[String, (Build, ArtifactItem, ArtifactFile)], failedResults: Map[String, Throwable], selectorOption: Option[String]): Option[ResultWithSelection] = dependencyCheckReportsParser.parseReports(successfulResults, failedResults).selection(selectorOption)
 
   def searchVulnerableSoftware(versionlessCpes: Seq[String], versionOption: Option[String]) = ReadAction.async{ implicit req =>
     if(versionlessCpes.isEmpty){
@@ -71,26 +73,26 @@ class Statistics @Inject() (
     }
   }
 
-  def basic(projectOption: Option[String]) = ReadAction.async{ implicit req =>
+  def basic(selectorOption: Option[String]) = ReadAction.async{ implicit req =>
     val (lastRefreshTime, resultsFuture) = projectReportsProvider.resultsForVersions(versions)
-    resultsFuture flatMap { case (successfulResults, failedResults) =>
+    resultsFuture flatMap { allResults =>
-      select(successfulResults, projectOption).fold(Future.successful(notFound())){ selection =>
+      select(allResults, selectorOption).fold(Future.successful(notFound())){ selection =>
         val tagsFuture = tagsService.all
         val parsedReports = selection.result
         for{
-          tagStatistics <- statisticsForTags(parsedReports, failedResults, tagsFuture)
+          tagStatistics <- statisticsForTags(parsedReports, tagsFuture)
           libraries <- librariesService.all
         } yield Ok(views.html.statistics.basic(
           tagStatistics = tagStatistics,
           projectsWithSelection = selection.projectsWithSelection,
           parsedReports = parsedReports,
-          lds = LibDepStatistics(libraries.toSet, parsedReports.groupedDependencies.toSet, failedResults, parsedReports)
+          lds = LibDepStatistics(libraries.toSet, parsedReports.groupedDependencies.toSet, selection.result.failedReportDownloads, parsedReports)
         ))
       }
     }
   }
 
-  def statisticsForTags(parsedReports: DependencyCheckReportsParser.Result, failedReports: Map[String, Throwable], tagsFuture: Future[Seq[(Int, LibraryTag)]]): Future[Seq[TagStatistics]] = {
+  def statisticsForTags(parsedReports: DependencyCheckReportsParser.Result, tagsFuture: Future[Seq[(Int, LibraryTag)]]): Future[Seq[TagStatistics]] = {
     val librariesFuture = librariesService.byPlainLibraryIdentifiers(parsedReports.allDependencies.flatMap(_._1.plainLibraryIdentifiers).toSet)
     val libraryTagAssignmentsFuture = librariesFuture.flatMap{libraries => libraryTagAssignmentsService.forLibraries(libraries.values.map(_._1).toSet)}
     val tagsToLibrariesFuture = libraryTagAssignmentsService.tagsToLibraries(libraryTagAssignmentsFuture)
@@ -110,7 +112,7 @@ class Statistics @Inject() (
         stats = LibDepStatistics(
           libraries = tagLibraries,
           dependencies = tagDependencies,
-          failedReportDownloads = failedReports,
+          failedReportDownloads = parsedReports.failedReportDownloads,
           parsedReports = parsedReports
         )
       ))
@@ -119,8 +121,8 @@ class Statistics @Inject() (
 
   def vulnerabilities(projectOption: Option[String], tagIdOption: Option[Int]) = ReadAction.async {implicit req =>
     val (lastRefreshTime, resultsFuture) = projectReportsProvider.resultsForVersions(versions)
-    resultsFuture flatMap { case (successfulResults, failedResults) =>
+    resultsFuture flatMap { allResults =>
-      select(successfulResults, projectOption).fold(Future.successful(notFound())){ selection =>
+      select(allResults, projectOption).fold(Future.successful(notFound())){ selection =>
         val parsedReports = selection.result
         for{
           libraries <- librariesService.byPlainLibraryIdentifiers(parsedReports.allDependencies.flatMap(_._1.plainLibraryIdentifiers).toSet)
@@ -128,12 +130,12 @@ class Statistics @Inject() (
           statistics <- tagOption.fold(Future.successful(LibDepStatistics(
             dependencies = parsedReports.groupedDependencies.toSet,
             libraries = libraries.values.toSet,
-            failedReportDownloads = failedResults,
+            failedReportDownloads = selection.result.failedReportDownloads,
             parsedReports = parsedReports
           ))){ tag =>
-            statisticsForTags(parsedReports, failedResults, Future.successful(Seq(tag))).map{
+            statisticsForTags(parsedReports, Future.successful(Seq(tag))).map{
               case Seq(TagStatistics(_, stats)) => stats // statisticsForTags is designed for multiple tags, but we have just one…
-              case Seq() => LibDepStatistics(libraries = Set(), dependencies = Set(), failedReportDownloads = failedResults, parsedReports) // We don't want to crash when no dependencies are there…
+              case Seq() => LibDepStatistics(libraries = Set(), dependencies = Set(), failedReportDownloads = selection.result.failedReportDownloads, parsedReports) // We don't want to crash when no dependencies are there…
             }
           }
         } yield Ok(views.html.statistics.vulnerabilities(
@@ -145,10 +147,10 @@ class Statistics @Inject() (
     }
   }
 
-  def vulnerability(name: String, projectOption: Option[String]) = ReadAction.async { implicit req =>
+  def vulnerability(name: String, selectorOption: Option[String]) = ReadAction.async { implicit req =>
     val (lastRefreshTime, resultsFuture) = projectReportsProvider.resultsForVersions(versions)
-    resultsFuture flatMap { case (successfulResults, failedResults) =>
+    resultsFuture flatMap { allResults =>
-      select(successfulResults, projectOption).fold(Future.successful(notFound())){ selection =>
+      select(allResults, selectorOption).fold(Future.successful(notFound())){ selection =>
         val relevantReports = selection.result
         val vulns = relevantReports.vulnerableDependencies.flatMap(dep => dep.vulnerabilities.map(vuln => (vuln, dep))).groupBy(_._1.name).mapValues{case vulnsWithDeps =>
           val (vulnSeq, depSeq) = vulnsWithDeps.unzip
@@ -186,10 +188,10 @@ class Statistics @Inject() (
     }
   }
 
-  def vulnerableLibraries(project: Option[String]) = ReadAction.async { implicit req =>
+  def vulnerableLibraries(selectorOption: Option[String]) = ReadAction.async { implicit req =>
     val (lastRefreshTime, resultsFuture) = projectReportsProvider.resultsForVersions(versions)
-    resultsFuture flatMap { case (successfulResults, failedResults) =>
+    resultsFuture flatMap { allResults =>
-      select(successfulResults, project).fold(Future.successful(notFound())){selection =>
+      select(allResults, selectorOption).fold(Future.successful(notFound())){ selection =>
         val reports = selection.result
         Future.successful(Ok(views.html.statistics.vulnerableLibraries(
           projectsWithSelection = selection.projectsWithSelection,
@@ -200,10 +202,10 @@ class Statistics @Inject() (
     }
   }
 
-  def allLibraries(project: Option[String]) = ReadAction.async { implicit req =>
+  def allLibraries(selectorOption: Option[String]) = ReadAction.async { implicit req =>
     val (lastRefreshTime, resultsFuture) = projectReportsProvider.resultsForVersions(versions)
-    resultsFuture flatMap { case (successfulResults, failedResults) =>
+    resultsFuture flatMap { allResults =>
-      select(successfulResults, project).fold(Future.successful(notFound())){selection =>
+      select(allResults, selectorOption).fold(Future.successful(notFound())){ selection =>
         Future.successful(Ok(views.html.statistics.allLibraries(
           projectsWithSelection = selection.projectsWithSelection,
           allDependencies = selection.result.groupedDependencies
@@ -212,10 +214,10 @@ class Statistics @Inject() (
     }
   }
 
-  def allGavs(project: Option[String]) = ReadAction.async { implicit req =>
+  def allGavs(selectorOption: Option[String]) = ReadAction.async { implicit req =>
     val (lastRefreshTime, resultsFuture) = projectReportsProvider.resultsForVersions(versions)
-    resultsFuture flatMap { case (successfulResults, failedResults) =>
+    resultsFuture flatMap { allResults =>
-      select(successfulResults, project).fold(Future.successful(notFound())){selection =>
+      select(allResults, selectorOption).fold(Future.successful(notFound())){ selection =>
         Future.successful(Ok(Txt(
           selection.result.groupedDependencies.flatMap(_.mavenIdentifiers).toSet.toIndexedSeq.sortBy((id: Identifier) => (id.identifierType, id.name)).map(id => id.name.split(':') match {
             case Array(g, a, v) =>