Refactored LibDepStatistics to include FailedProjects

app/com/ysoft/odc/statistics/FailedProjects.scala

@@ -0,0 +1,23 @@
+package com.ysoft.odc.statistics
+
+import controllers.ReportInfo
+
+final class FailedProjects(val failedProjectsSet: Set[String]){
+  def isFailed(projectFullId: String): Boolean = {
+    val projectBareId = projectFullId.takeWhile(_ != '/')
+    failedProjectsSet contains projectBareId
+  }
+
+}
+
+object FailedProjects {
+  private[statistics] def combineFails(failedReportDownloads: Map[String, Throwable], parsingFailures: Map[ReportInfo, Throwable]): FailedProjects = {
+    /*
+    Fail can happen at multiple places:
+    1. Build cannot be downloaded (auth error, connection error, …) or is failed (failedReportDownloads)
+    2. Build is successful and can be downloaded, but it cannot be parsed (parsingFailures)
+    */
+    val failedProjectsSet = failedReportDownloads.keySet ++ parsingFailures.keySet.map(_.projectId)
+    new FailedProjects(failedProjectsSet)
+  }
+}

app/com/ysoft/odc/statistics/LibDepStatistics.scala

@@ -0,0 +1,34 @@
+package com.ysoft.odc.statistics
+
+import controllers.DependencyCheckReportsParser.Result
+import controllers._
+import models.Library
+
+case class LibDepStatistics(libraries: Set[(Int, Library)], dependencies: Set[GroupedDependency], failedProjects: FailedProjects){
+  def vulnerableRatio = vulnerableDependencies.size.toDouble / dependencies.size.toDouble
+  lazy val vulnerabilities: Set[Vulnerability] = dependencies.flatMap(_.vulnerabilities)
+  lazy val vulnerabilitiesByName = vulnerabilities.map(v => v.name -> v).toMap
+  lazy val vulnerabilityNames = vulnerabilities.map(_.name)
+  lazy val vulnerabilitiesToDependencies: Map[Vulnerability, Set[GroupedDependency]] = vulnerableDependencies.flatMap(dep =>
+    dep.vulnerabilities.map(vuln => (vuln, dep))
+  ).groupBy(_._1).mapValues(_.map(_._2)).map(identity)
+  vulnerableDependencies.flatMap(dep => dep.vulnerabilities.map(_ -> dep)).groupBy(_._1).mapValues(_.map(_._2)).map(identity)
+  vulnerableDependencies.flatMap(dep => dep.vulnerabilities.map(_ -> dep)).groupBy(_._1).mapValues(_.map(_._2)).map(identity)
+  lazy val vulnerableDependencies = dependencies.filter(_.isVulnerable)
+  lazy val (dependenciesWithCpe, dependenciesWithoutCpe) = dependencies.partition(_.hasCpe)
+  lazy val cpeRatio = dependenciesWithCpe.size.toDouble / dependencies.size.toDouble
+  lazy val weaknesses = vulnerabilities.flatMap(_.cweOption)
+  lazy val weaknessesFrequency = LibDepStatistics.computeWeaknessesFrequency(vulnerabilities)
+}
+
+object LibDepStatistics{
+  private def computeWeaknessesFrequency(vulnerabilities: Set[Vulnerability]) = vulnerabilities.toSeq.map(_.cweOption).groupBy(identity).mapValues(_.size).map(identity).withDefaultValue(0)
+  def apply(libraries: Set[(Int, Library)], dependencies: Set[GroupedDependency], failedReportDownloads: Map[String, Throwable], parsedReports: Result): LibDepStatistics = LibDepStatistics(
+    libraries = libraries,
+    dependencies = dependencies,
+    failedProjects = FailedProjects.combineFails(
+      failedReportDownloads = failedReportDownloads,
+      parsingFailures = parsedReports.failedAnalysises
+    )
+  )
+}

app/com/ysoft/odc/statistics/TagStatistics.scala

@@ -0,0 +1,8 @@
+package com.ysoft.odc.statistics
+
+import models.LibraryTag
+
+case class TagStatistics(tagRecord: (Int, LibraryTag), stats: LibDepStatistics){
+  def tag: LibraryTag = tagRecord._2
+  def tagId: Int = tagRecord._1
+}

app/controllers/Notifications.scala

@@ -4,8 +4,8 @@ import java.util.concurrent.atomic.AtomicBoolean
 import javax.inject.Inject
 
 import com.ysoft.concurrent.FutureLock._
+import com.ysoft.odc.statistics.{FailedProjects, LibDepStatistics}
 import com.ysoft.odc.{Absolutizer, ArtifactFile, ArtifactItem, SetDiff}
-import controllers.Statistics.LibDepStatistics
 import models.{EmailMessageId, ExportedVulnerability}
 import play.api.i18n.MessagesApi
 import play.api.libs.Crypto
@@ -91,29 +91,6 @@ class Notifications @Inject()(
     } yield (missingTickets, newTicketIds, projectUpdates.toSet: Set[Any])
   }
 
-  private final class FailedProjects(val failedProjectsSet: Set[String]){
-    def isFailed(projectFullId: String): Boolean = {
-      val projectBareId = projectFullId.takeWhile(_ != '/')
-      failedProjectsSet contains projectBareId
-    }
-
-  }
-
-  private object FailedProjects {
-    // TODO: Move elsewhere
-    def combineFails(failedReportDownloads: Map[String, Throwable], parsingFailures: Map[ReportInfo, Throwable]): FailedProjects = {
-      /*
-      Fail can happen at multiple places:
-      1. Build cannot be downloaded (auth error, connection error, …) or is failed (failedReportDownloads)
-      2. Build is successful and can be downloaded, but it cannot be parsed (parsingFailures)
-      */
-      val failedProjectsSet = failedReportDownloads.keySet ++ parsingFailures.keySet.map(_.projectId)
-      new FailedProjects(failedProjectsSet)
-    }
-  }
-
-  import FailedProjects.combineFails
-
   private def exportFailedReports(lds: LibDepStatistics, failed: FailedProjects): Future[Unit] = {
     if(failed.failedProjectsSet.nonEmpty){
       ???
@@ -134,8 +111,8 @@ class Notifications @Inject()(
           (successfulReports, failedReports) <- resultsFuture
           libraries <- librariesService.all
           parsedReports = dependencyCheckReportsParser.parseReports(successfulReports)
-          lds = LibDepStatistics(dependencies = parsedReports.groupedDependencies.toSet, libraries = libraries.toSet)
+          lds = LibDepStatistics(dependencies = parsedReports.groupedDependencies.toSet, libraries = libraries.toSet, failedReportDownloads = failedReports, parsedReports = parsedReports)
-          failed = combineFails(failedReports, parsedReports.failedAnalysises)
+          failed = lds.failedProjects
           failedReportsExportFuture = Fut(()) // TODO: exportFailedReports(lds, failed)
           issuesExportResultFuture = exportToIssueTracker(lds, failed, parsedReports.projectsReportInfo)
           diffDbExportResultFuture = exportToDiffDb(lds, failed, parsedReports.projectsReportInfo)

app/controllers/Statistics.scala

@@ -3,8 +3,9 @@ package controllers
 import com.github.nscala_time.time.Imports._
 import com.google.inject.Inject
 import com.google.inject.name.Named
+import com.ysoft.odc.statistics.{LibDepStatistics, TagStatistics}
 import com.ysoft.odc.{ArtifactFile, ArtifactItem}
-import models.{Library, LibraryTag}
+import models.LibraryTag
 import org.joda.time.DateTime
 import play.api.i18n.MessagesApi
 import play.twirl.api.Txt
@@ -13,34 +14,6 @@ import views.html.DefaultRequest
 
 import scala.concurrent.{ExecutionContext, Future}
 
-object Statistics{
-  case class LibDepStatistics(libraries: Set[(Int, Library)], dependencies: Set[GroupedDependency]){
-    def vulnerableRatio = vulnerableDependencies.size.toDouble / dependencies.size.toDouble
-    lazy val vulnerabilities: Set[Vulnerability] = dependencies.flatMap(_.vulnerabilities)
-    lazy val vulnerabilitiesByName = vulnerabilities.map(v => v.name -> v).toMap
-    lazy val vulnerabilityNames = vulnerabilities.map(_.name)
-    lazy val vulnerabilitiesToDependencies: Map[Vulnerability, Set[GroupedDependency]] = vulnerableDependencies.flatMap(dep =>
-      dep.vulnerabilities.map(vuln => (vuln, dep))
-    ).groupBy(_._1).mapValues(_.map(_._2)).map(identity)
-    vulnerableDependencies.flatMap(dep => dep.vulnerabilities.map(_ -> dep)).groupBy(_._1).mapValues(_.map(_._2)).map(identity)
-    vulnerableDependencies.flatMap(dep => dep.vulnerabilities.map(_ -> dep)).groupBy(_._1).mapValues(_.map(_._2)).map(identity)
-    lazy val vulnerableDependencies = dependencies.filter(_.isVulnerable)
-    lazy val (dependenciesWithCpe, dependenciesWithoutCpe) = dependencies.partition(_.hasCpe)
-    lazy val cpeRatio = dependenciesWithCpe.size.toDouble / dependencies.size.toDouble
-    lazy val weaknesses = vulnerabilities.flatMap(_.cweOption)
-    lazy val weaknessesFrequency = computeWeaknessesFrequency(vulnerabilities)
-  }
-  case class TagStatistics(tagRecord: (Int, LibraryTag), stats: LibDepStatistics){
-    def tag: LibraryTag = tagRecord._2
-    def tagId: Int = tagRecord._1
-  }
-
-  def computeWeaknessesFrequency(vulnerabilities: Set[Vulnerability]) = vulnerabilities.toSeq.map(_.cweOption).groupBy(identity).mapValues(_.size).map(identity).withDefaultValue(0)
-
-}
-
-import controllers.Statistics._
-
 class Statistics @Inject() (
   reportsParser: DependencyCheckReportsParser,
   reportsProcessor: DependencyCheckReportsProcessor,
@@ -105,17 +78,19 @@ class Statistics @Inject() (
         val tagsFuture = tagsService.all
         val parsedReports = selection.result
         for{
-          tagStatistics <- statisticsForTags(parsedReports, tagsFuture)
+          tagStatistics <- statisticsForTags(parsedReports, failedResults, tagsFuture)
+          libraries <- librariesService.all
         } yield Ok(views.html.statistics.basic(
           tagStatistics = tagStatistics,
           projectsWithSelection = selection.projectsWithSelection,
-          parsedReports = parsedReports
+          parsedReports = parsedReports,
+          lds = LibDepStatistics(libraries.toSet, parsedReports.groupedDependencies.toSet, failedResults, parsedReports)
         ))
       }
     }
   }
 
-  def statisticsForTags(parsedReports: DependencyCheckReportsParser.Result, tagsFuture: Future[Seq[(Int, LibraryTag)]]): Future[Seq[Statistics.TagStatistics]] = {
+  def statisticsForTags(parsedReports: DependencyCheckReportsParser.Result, failedReports: Map[String, Throwable], tagsFuture: Future[Seq[(Int, LibraryTag)]]): Future[Seq[TagStatistics]] = {
     val librariesFuture = librariesService.byPlainLibraryIdentifiers(parsedReports.allDependencies.flatMap(_._1.plainLibraryIdentifiers).toSet)
     val libraryTagAssignmentsFuture = librariesFuture.flatMap{libraries => libraryTagAssignmentsService.forLibraries(libraries.values.map(_._1).toSet)}
     val tagsToLibrariesFuture = libraryTagAssignmentsService.tagsToLibraries(libraryTagAssignmentsFuture)
@@ -130,7 +105,15 @@ class Statistics @Inject() (
       val tagDependencies: Set[GroupedDependency] = tagLibraries.flatMap{case (_, lib) => librariesToDependencies(lib.plainLibraryIdentifier)}
       // TODO: vulnerabilities in the past
       if(tagLibraries.isEmpty) None
-      else Some(TagStatistics(tagRecord = tagRecord, stats = LibDepStatistics(libraries = tagLibraries, dependencies = tagDependencies)))
+      else Some(TagStatistics(
+        tagRecord = tagRecord,
+        stats = LibDepStatistics(
+          libraries = tagLibraries,
+          dependencies = tagDependencies,
+          failedReportDownloads = failedReports,
+          parsedReports = parsedReports
+        )
+      ))
     }
   }
 
@@ -142,10 +125,15 @@ class Statistics @Inject() (
         for{
           libraries <- librariesService.byPlainLibraryIdentifiers(parsedReports.allDependencies.flatMap(_._1.plainLibraryIdentifiers).toSet)
           tagOption <- tagIdOption.fold[Future[Option[(Int, LibraryTag)]]](Future.successful(None))(tagId => tagsService.getById(tagId).map(Some(_)))
-          statistics <- tagOption.fold(Future.successful(LibDepStatistics(dependencies = parsedReports.groupedDependencies.toSet, libraries = libraries.values.toSet))){ tag =>
+          statistics <- tagOption.fold(Future.successful(LibDepStatistics(
-            statisticsForTags(parsedReports, Future.successful(Seq(tag))).map{
+            dependencies = parsedReports.groupedDependencies.toSet,
+            libraries = libraries.values.toSet,
+            failedReportDownloads = failedResults,
+            parsedReports = parsedReports
+          ))){ tag =>
+            statisticsForTags(parsedReports, failedResults, Future.successful(Seq(tag))).map{
               case Seq(TagStatistics(_, stats)) => stats // statisticsForTags is designed for multiple tags, but we have just one…
-              case Seq() => LibDepStatistics(libraries = Set(), dependencies = Set()) // We don't want to crash when no dependencies are there…
+              case Seq() => LibDepStatistics(libraries = Set(), dependencies = Set(), failedReportDownloads = failedResults, parsedReports) // We don't want to crash when no dependencies are there…
             }
           }
         } yield Ok(views.html.statistics.vulnerabilities(

app/controllers/package.scala

@@ -1,6 +1,6 @@
 import com.mohiva.play.silhouette.api.Environment
 import com.mohiva.play.silhouette.impl.authenticators.CookieAuthenticator
-import models.{User, SnoozeInfo}
+import models.{SnoozeInfo, User}
 
 /**
  * Created by user on 7/15/15.
@@ -16,6 +16,7 @@ package object controllers {
   type DateTime = org.joda.time.DateTime
   type SnoozesInfo = Map[String, SnoozeInfo]
   type AuthEnv = Environment[User, CookieAuthenticator]
+  type LibDepStatistics = com.ysoft.odc.statistics.LibDepStatistics
 
 
   val NormalUrlPattern = """^(http(s?)|ftp(s?))://.*""".r

app/views/statistics/basic.scala.html

@@ -1,10 +1,11 @@
+@import com.ysoft.odc.statistics.TagStatistics
 @(
     projectsWithSelection: ProjectsWithSelection,
-    tagStatistics: Seq[Statistics.TagStatistics],
+    lds: LibDepStatistics,
+    tagStatistics: Seq[TagStatistics],
     parsedReports: DependencyCheckReportsParser.Result
 )(implicit messagesApi: MessagesApi, requestHeader: DefaultRequest)
 @import com.ysoft.odc.CWE
-@import controllers.Statistics.TagStatistics
 @import play.api.libs.json.{JsNull, JsString}
 @import scala.language.implicitConversions
 @implicitTagStatistics(ts: TagStatistics) = @{ts.stats}
@@ -52,7 +53,7 @@
     }
 
 
-    <div id="weakness" data-data='@{plotData(Statistics.computeWeaknessesFrequency(parsedReports.groupedDependencies.flatMap(_.vulnerabilities).toSet))}'></div>
+    <div id="weakness" data-data='@{plotData(lds.weaknessesFrequency)}'></div>
     <script type="text/javascript">
 
     var WeaknessIdentifier = function(brief, verbose){

app/views/statistics/vulnerabilities.scala.html

@@ -1,7 +1,7 @@
 @(
     projectsWithSelection: ProjectsWithSelection,
     tagOption: Option[(Int, LibraryTag)],
-    statistics: Statistics.LibDepStatistics
+    statistics: LibDepStatistics
 )(implicit messagesApi: MessagesApi, requestHeader: DefaultRequest)
 
 @main(