diff --git a/app/com/ysoft/odc/statistics/FailedProjects.scala b/app/com/ysoft/odc/statistics/FailedProjects.scala new file mode 100644 index 0000000..829d205 --- /dev/null +++ b/app/com/ysoft/odc/statistics/FailedProjects.scala @@ -0,0 +1,23 @@ +package com.ysoft.odc.statistics + +import controllers.ReportInfo + +final class FailedProjects(val failedProjectsSet: Set[String]){ + def isFailed(projectFullId: String): Boolean = { + val projectBareId = projectFullId.takeWhile(_ != '/') + failedProjectsSet contains projectBareId + } + +} + +object FailedProjects { + private[statistics] def combineFails(failedReportDownloads: Map[String, Throwable], parsingFailures: Map[ReportInfo, Throwable]): FailedProjects = { + /* + Fail can happen at multiple places: + 1. Build cannot be downloaded (auth error, connection error, …) or is failed (failedReportDownloads) + 2. Build is successful and can be downloaded, but it cannot be parsed (parsingFailures) + */ + val failedProjectsSet = failedReportDownloads.keySet ++ parsingFailures.keySet.map(_.projectId) + new FailedProjects(failedProjectsSet) + } +} diff --git a/app/com/ysoft/odc/statistics/LibDepStatistics.scala b/app/com/ysoft/odc/statistics/LibDepStatistics.scala new file mode 100644 index 0000000..ed42ac3 --- /dev/null +++ b/app/com/ysoft/odc/statistics/LibDepStatistics.scala @@ -0,0 +1,34 @@ +package com.ysoft.odc.statistics + +import controllers.DependencyCheckReportsParser.Result +import controllers._ +import models.Library + +case class LibDepStatistics(libraries: Set[(Int, Library)], dependencies: Set[GroupedDependency], failedProjects: FailedProjects){ + def vulnerableRatio = vulnerableDependencies.size.toDouble / dependencies.size.toDouble + lazy val vulnerabilities: Set[Vulnerability] = dependencies.flatMap(_.vulnerabilities) + lazy val vulnerabilitiesByName = vulnerabilities.map(v => v.name -> v).toMap + lazy val vulnerabilityNames = vulnerabilities.map(_.name) + lazy val vulnerabilitiesToDependencies: Map[Vulnerability, Set[GroupedDependency]] = vulnerableDependencies.flatMap(dep => + dep.vulnerabilities.map(vuln => (vuln, dep)) + ).groupBy(_._1).mapValues(_.map(_._2)).map(identity) + vulnerableDependencies.flatMap(dep => dep.vulnerabilities.map(_ -> dep)).groupBy(_._1).mapValues(_.map(_._2)).map(identity) + vulnerableDependencies.flatMap(dep => dep.vulnerabilities.map(_ -> dep)).groupBy(_._1).mapValues(_.map(_._2)).map(identity) + lazy val vulnerableDependencies = dependencies.filter(_.isVulnerable) + lazy val (dependenciesWithCpe, dependenciesWithoutCpe) = dependencies.partition(_.hasCpe) + lazy val cpeRatio = dependenciesWithCpe.size.toDouble / dependencies.size.toDouble + lazy val weaknesses = vulnerabilities.flatMap(_.cweOption) + lazy val weaknessesFrequency = LibDepStatistics.computeWeaknessesFrequency(vulnerabilities) +} + +object LibDepStatistics{ + private def computeWeaknessesFrequency(vulnerabilities: Set[Vulnerability]) = vulnerabilities.toSeq.map(_.cweOption).groupBy(identity).mapValues(_.size).map(identity).withDefaultValue(0) + def apply(libraries: Set[(Int, Library)], dependencies: Set[GroupedDependency], failedReportDownloads: Map[String, Throwable], parsedReports: Result): LibDepStatistics = LibDepStatistics( + libraries = libraries, + dependencies = dependencies, + failedProjects = FailedProjects.combineFails( + failedReportDownloads = failedReportDownloads, + parsingFailures = parsedReports.failedAnalysises + ) + ) +} diff --git a/app/com/ysoft/odc/statistics/TagStatistics.scala b/app/com/ysoft/odc/statistics/TagStatistics.scala new file mode 100644 index 0000000..1a90e8d --- /dev/null +++ b/app/com/ysoft/odc/statistics/TagStatistics.scala @@ -0,0 +1,8 @@ +package com.ysoft.odc.statistics + +import models.LibraryTag + +case class TagStatistics(tagRecord: (Int, LibraryTag), stats: LibDepStatistics){ + def tag: LibraryTag = tagRecord._2 + def tagId: Int = tagRecord._1 +} diff --git a/app/controllers/Notifications.scala b/app/controllers/Notifications.scala index d3d113c..eaa4d65 100644 --- a/app/controllers/Notifications.scala +++ b/app/controllers/Notifications.scala @@ -4,8 +4,8 @@ import java.util.concurrent.atomic.AtomicBoolean import javax.inject.Inject import com.ysoft.concurrent.FutureLock._ +import com.ysoft.odc.statistics.{FailedProjects, LibDepStatistics} import com.ysoft.odc.{Absolutizer, ArtifactFile, ArtifactItem, SetDiff} -import controllers.Statistics.LibDepStatistics import models.{EmailMessageId, ExportedVulnerability} import play.api.i18n.MessagesApi import play.api.libs.Crypto @@ -91,29 +91,6 @@ class Notifications @Inject()( } yield (missingTickets, newTicketIds, projectUpdates.toSet: Set[Any]) } - private final class FailedProjects(val failedProjectsSet: Set[String]){ - def isFailed(projectFullId: String): Boolean = { - val projectBareId = projectFullId.takeWhile(_ != '/') - failedProjectsSet contains projectBareId - } - - } - - private object FailedProjects { - // TODO: Move elsewhere - def combineFails(failedReportDownloads: Map[String, Throwable], parsingFailures: Map[ReportInfo, Throwable]): FailedProjects = { - /* - Fail can happen at multiple places: - 1. Build cannot be downloaded (auth error, connection error, …) or is failed (failedReportDownloads) - 2. Build is successful and can be downloaded, but it cannot be parsed (parsingFailures) - */ - val failedProjectsSet = failedReportDownloads.keySet ++ parsingFailures.keySet.map(_.projectId) - new FailedProjects(failedProjectsSet) - } - } - - import FailedProjects.combineFails - private def exportFailedReports(lds: LibDepStatistics, failed: FailedProjects): Future[Unit] = { if(failed.failedProjectsSet.nonEmpty){ ??? @@ -134,8 +111,8 @@ class Notifications @Inject()( (successfulReports, failedReports) <- resultsFuture libraries <- librariesService.all parsedReports = dependencyCheckReportsParser.parseReports(successfulReports) - lds = LibDepStatistics(dependencies = parsedReports.groupedDependencies.toSet, libraries = libraries.toSet) - failed = combineFails(failedReports, parsedReports.failedAnalysises) + lds = LibDepStatistics(dependencies = parsedReports.groupedDependencies.toSet, libraries = libraries.toSet, failedReportDownloads = failedReports, parsedReports = parsedReports) + failed = lds.failedProjects failedReportsExportFuture = Fut(()) // TODO: exportFailedReports(lds, failed) issuesExportResultFuture = exportToIssueTracker(lds, failed, parsedReports.projectsReportInfo) diffDbExportResultFuture = exportToDiffDb(lds, failed, parsedReports.projectsReportInfo) diff --git a/app/controllers/Statistics.scala b/app/controllers/Statistics.scala index 1ffec1a..0d5e306 100644 --- a/app/controllers/Statistics.scala +++ b/app/controllers/Statistics.scala @@ -3,8 +3,9 @@ package controllers import com.github.nscala_time.time.Imports._ import com.google.inject.Inject import com.google.inject.name.Named +import com.ysoft.odc.statistics.{LibDepStatistics, TagStatistics} import com.ysoft.odc.{ArtifactFile, ArtifactItem} -import models.{Library, LibraryTag} +import models.LibraryTag import org.joda.time.DateTime import play.api.i18n.MessagesApi import play.twirl.api.Txt @@ -13,34 +14,6 @@ import views.html.DefaultRequest import scala.concurrent.{ExecutionContext, Future} -object Statistics{ - case class LibDepStatistics(libraries: Set[(Int, Library)], dependencies: Set[GroupedDependency]){ - def vulnerableRatio = vulnerableDependencies.size.toDouble / dependencies.size.toDouble - lazy val vulnerabilities: Set[Vulnerability] = dependencies.flatMap(_.vulnerabilities) - lazy val vulnerabilitiesByName = vulnerabilities.map(v => v.name -> v).toMap - lazy val vulnerabilityNames = vulnerabilities.map(_.name) - lazy val vulnerabilitiesToDependencies: Map[Vulnerability, Set[GroupedDependency]] = vulnerableDependencies.flatMap(dep => - dep.vulnerabilities.map(vuln => (vuln, dep)) - ).groupBy(_._1).mapValues(_.map(_._2)).map(identity) - vulnerableDependencies.flatMap(dep => dep.vulnerabilities.map(_ -> dep)).groupBy(_._1).mapValues(_.map(_._2)).map(identity) - vulnerableDependencies.flatMap(dep => dep.vulnerabilities.map(_ -> dep)).groupBy(_._1).mapValues(_.map(_._2)).map(identity) - lazy val vulnerableDependencies = dependencies.filter(_.isVulnerable) - lazy val (dependenciesWithCpe, dependenciesWithoutCpe) = dependencies.partition(_.hasCpe) - lazy val cpeRatio = dependenciesWithCpe.size.toDouble / dependencies.size.toDouble - lazy val weaknesses = vulnerabilities.flatMap(_.cweOption) - lazy val weaknessesFrequency = computeWeaknessesFrequency(vulnerabilities) - } - case class TagStatistics(tagRecord: (Int, LibraryTag), stats: LibDepStatistics){ - def tag: LibraryTag = tagRecord._2 - def tagId: Int = tagRecord._1 - } - - def computeWeaknessesFrequency(vulnerabilities: Set[Vulnerability]) = vulnerabilities.toSeq.map(_.cweOption).groupBy(identity).mapValues(_.size).map(identity).withDefaultValue(0) - -} - -import controllers.Statistics._ - class Statistics @Inject() ( reportsParser: DependencyCheckReportsParser, reportsProcessor: DependencyCheckReportsProcessor, @@ -105,17 +78,19 @@ class Statistics @Inject() ( val tagsFuture = tagsService.all val parsedReports = selection.result for{ - tagStatistics <- statisticsForTags(parsedReports, tagsFuture) + tagStatistics <- statisticsForTags(parsedReports, failedResults, tagsFuture) + libraries <- librariesService.all } yield Ok(views.html.statistics.basic( tagStatistics = tagStatistics, projectsWithSelection = selection.projectsWithSelection, - parsedReports = parsedReports + parsedReports = parsedReports, + lds = LibDepStatistics(libraries.toSet, parsedReports.groupedDependencies.toSet, failedResults, parsedReports) )) } } } - def statisticsForTags(parsedReports: DependencyCheckReportsParser.Result, tagsFuture: Future[Seq[(Int, LibraryTag)]]): Future[Seq[Statistics.TagStatistics]] = { + def statisticsForTags(parsedReports: DependencyCheckReportsParser.Result, failedReports: Map[String, Throwable], tagsFuture: Future[Seq[(Int, LibraryTag)]]): Future[Seq[TagStatistics]] = { val librariesFuture = librariesService.byPlainLibraryIdentifiers(parsedReports.allDependencies.flatMap(_._1.plainLibraryIdentifiers).toSet) val libraryTagAssignmentsFuture = librariesFuture.flatMap{libraries => libraryTagAssignmentsService.forLibraries(libraries.values.map(_._1).toSet)} val tagsToLibrariesFuture = libraryTagAssignmentsService.tagsToLibraries(libraryTagAssignmentsFuture) @@ -130,7 +105,15 @@ class Statistics @Inject() ( val tagDependencies: Set[GroupedDependency] = tagLibraries.flatMap{case (_, lib) => librariesToDependencies(lib.plainLibraryIdentifier)} // TODO: vulnerabilities in the past if(tagLibraries.isEmpty) None - else Some(TagStatistics(tagRecord = tagRecord, stats = LibDepStatistics(libraries = tagLibraries, dependencies = tagDependencies))) + else Some(TagStatistics( + tagRecord = tagRecord, + stats = LibDepStatistics( + libraries = tagLibraries, + dependencies = tagDependencies, + failedReportDownloads = failedReports, + parsedReports = parsedReports + ) + )) } } @@ -142,10 +125,15 @@ class Statistics @Inject() ( for{ libraries <- librariesService.byPlainLibraryIdentifiers(parsedReports.allDependencies.flatMap(_._1.plainLibraryIdentifiers).toSet) tagOption <- tagIdOption.fold[Future[Option[(Int, LibraryTag)]]](Future.successful(None))(tagId => tagsService.getById(tagId).map(Some(_))) - statistics <- tagOption.fold(Future.successful(LibDepStatistics(dependencies = parsedReports.groupedDependencies.toSet, libraries = libraries.values.toSet))){ tag => - statisticsForTags(parsedReports, Future.successful(Seq(tag))).map{ + statistics <- tagOption.fold(Future.successful(LibDepStatistics( + dependencies = parsedReports.groupedDependencies.toSet, + libraries = libraries.values.toSet, + failedReportDownloads = failedResults, + parsedReports = parsedReports + ))){ tag => + statisticsForTags(parsedReports, failedResults, Future.successful(Seq(tag))).map{ case Seq(TagStatistics(_, stats)) => stats // statisticsForTags is designed for multiple tags, but we have just one… - case Seq() => LibDepStatistics(libraries = Set(), dependencies = Set()) // We don't want to crash when no dependencies are there… + case Seq() => LibDepStatistics(libraries = Set(), dependencies = Set(), failedReportDownloads = failedResults, parsedReports) // We don't want to crash when no dependencies are there… } } } yield Ok(views.html.statistics.vulnerabilities( diff --git a/app/controllers/package.scala b/app/controllers/package.scala index 9b0a9b4..9f170b8 100644 --- a/app/controllers/package.scala +++ b/app/controllers/package.scala @@ -1,6 +1,6 @@ import com.mohiva.play.silhouette.api.Environment import com.mohiva.play.silhouette.impl.authenticators.CookieAuthenticator -import models.{User, SnoozeInfo} +import models.{SnoozeInfo, User} /** * Created by user on 7/15/15. @@ -16,6 +16,7 @@ package object controllers { type DateTime = org.joda.time.DateTime type SnoozesInfo = Map[String, SnoozeInfo] type AuthEnv = Environment[User, CookieAuthenticator] + type LibDepStatistics = com.ysoft.odc.statistics.LibDepStatistics val NormalUrlPattern = """^(http(s?)|ftp(s?))://.*""".r diff --git a/app/views/statistics/basic.scala.html b/app/views/statistics/basic.scala.html index 96242dc..9d870ac 100644 --- a/app/views/statistics/basic.scala.html +++ b/app/views/statistics/basic.scala.html @@ -1,10 +1,11 @@ +@import com.ysoft.odc.statistics.TagStatistics @( projectsWithSelection: ProjectsWithSelection, - tagStatistics: Seq[Statistics.TagStatistics], + lds: LibDepStatistics, + tagStatistics: Seq[TagStatistics], parsedReports: DependencyCheckReportsParser.Result )(implicit messagesApi: MessagesApi, requestHeader: DefaultRequest) @import com.ysoft.odc.CWE -@import controllers.Statistics.TagStatistics @import play.api.libs.json.{JsNull, JsString} @import scala.language.implicitConversions @implicitTagStatistics(ts: TagStatistics) = @{ts.stats} @@ -52,7 +53,7 @@ } -
+