mirror of
https://github.com/ysoftdevs/odc-analyzer.git
synced 2026-01-14 15:53:53 +01:00
236 lines
12 KiB
Scala
236 lines
12 KiB
Scala
package controllers
|
|
|
|
import com.github.nscala_time.time.Imports._
|
|
import com.google.inject.Inject
|
|
import com.google.inject.name.Named
|
|
import com.ysoft.odc.{ArtifactFile, ArtifactItem}
|
|
import models.{Library, LibraryTag}
|
|
import org.joda.time.DateTime
|
|
import play.api.i18n.MessagesApi
|
|
import play.twirl.api.Txt
|
|
import services._
|
|
import views.html.DefaultRequest
|
|
|
|
import scala.concurrent.{ExecutionContext, Future}
|
|
|
|
object Statistics{
|
|
case class LibDepStatistics(libraries: Set[(Int, Library)], dependencies: Set[GroupedDependency]){
|
|
def vulnerableRatio = vulnerableDependencies.size.toDouble / dependencies.size.toDouble
|
|
lazy val vulnerabilities: Set[Vulnerability] = dependencies.flatMap(_.vulnerabilities)
|
|
lazy val vulnerabilitiesByName = vulnerabilities.map(v => v.name -> v).toMap
|
|
lazy val vulnerabilityNames = vulnerabilities.map(_.name)
|
|
lazy val vulnerabilitiesToDependencies: Map[Vulnerability, Set[GroupedDependency]] = vulnerableDependencies.flatMap(dep =>
|
|
dep.vulnerabilities.map(vuln => (vuln, dep))
|
|
).groupBy(_._1).mapValues(_.map(_._2)).map(identity)
|
|
vulnerableDependencies.flatMap(dep => dep.vulnerabilities.map(_ -> dep)).groupBy(_._1).mapValues(_.map(_._2)).map(identity)
|
|
vulnerableDependencies.flatMap(dep => dep.vulnerabilities.map(_ -> dep)).groupBy(_._1).mapValues(_.map(_._2)).map(identity)
|
|
lazy val vulnerableDependencies = dependencies.filter(_.isVulnerable)
|
|
lazy val (dependenciesWithCpe, dependenciesWithoutCpe) = dependencies.partition(_.hasCpe)
|
|
lazy val cpeRatio = dependenciesWithCpe.size.toDouble / dependencies.size.toDouble
|
|
lazy val weaknesses = vulnerabilities.flatMap(_.cweOption)
|
|
lazy val weaknessesFrequency = computeWeaknessesFrequency(vulnerabilities)
|
|
}
|
|
case class TagStatistics(tagRecord: (Int, LibraryTag), stats: LibDepStatistics){
|
|
def tag: LibraryTag = tagRecord._2
|
|
def tagId: Int = tagRecord._1
|
|
}
|
|
|
|
def computeWeaknessesFrequency(vulnerabilities: Set[Vulnerability]) = vulnerabilities.toSeq.map(_.cweOption).groupBy(identity).mapValues(_.size).map(identity).withDefaultValue(0)
|
|
|
|
}
|
|
|
|
import controllers.Statistics._
|
|
|
|
class Statistics @Inject() (
|
|
reportsParser: DependencyCheckReportsParser,
|
|
reportsProcessor: DependencyCheckReportsProcessor,
|
|
projectReportsProvider: ProjectReportsProvider,
|
|
dependencyCheckReportsParser: DependencyCheckReportsParser,
|
|
librariesService: LibrariesService,
|
|
tagsService: TagsService,
|
|
odcService: OdcService,
|
|
libraryTagAssignmentsService: LibraryTagAssignmentsService,
|
|
@Named("missing-GAV-exclusions") missingGAVExclusions: MissingGavExclusions,
|
|
projects: Projects,
|
|
vulnerabilityNotificationService: VulnerabilityNotificationService,
|
|
issueTrackerServiceOption: Option[IssueTrackerService],
|
|
val env: AuthEnv
|
|
)(implicit val messagesApi: MessagesApi, executionContext: ExecutionContext) extends AuthenticatedController {
|
|
|
|
private val versions = Map[String, Int]()
|
|
|
|
private def notFound()(implicit req: DefaultRequest) = {
|
|
NotFound(views.html.defaultpages.notFound("GET", req.uri))
|
|
}
|
|
|
|
import secureRequestConversion._
|
|
|
|
|
|
private def select(successfulResults: Map[String, (Build, ArtifactItem, ArtifactFile)], selectorOption: Option[String]) = dependencyCheckReportsParser.parseReports(successfulResults).selection(selectorOption)
|
|
|
|
def searchVulnerableSoftware(versionlessCpes: Seq[String], versionOption: Option[String]) = ReadAction.async{ implicit req =>
|
|
if(versionlessCpes.isEmpty){
|
|
Future.successful(notFound())
|
|
}else{
|
|
val now = DateTime.now()
|
|
val oldDataThreshold = 2.days
|
|
val lastDbUpdateFuture = odcService.loadLastDbUpdate()
|
|
val isOldFuture = lastDbUpdateFuture.map{ lastUpdate => now - oldDataThreshold > lastUpdate}
|
|
versionOption match {
|
|
case Some(version) =>
|
|
for {
|
|
res1 <- Future.traverse(versionlessCpes) { versionlessCpe => odcService.findRelevantCpes(versionlessCpe, version) }
|
|
vulnIds = res1.flatten.map(_.vulnerabilityId).toSet
|
|
vulns <- Future.traverse(vulnIds)(id => odcService.getVulnerabilityDetails(id).map(_.get))
|
|
isOld <- isOldFuture
|
|
} yield Ok(views.html.statistics.vulnerabilitiesForLibrary(
|
|
vulnsAndVersionOption = Some((vulns, version)),
|
|
cpes = versionlessCpes,
|
|
isDbOld = isOld
|
|
))
|
|
case None =>
|
|
for(isOld <- isOldFuture) yield Ok(views.html.statistics.vulnerabilitiesForLibrary(
|
|
vulnsAndVersionOption = None,
|
|
cpes = versionlessCpes,
|
|
isDbOld = isOld
|
|
))
|
|
}
|
|
}
|
|
}
|
|
|
|
def basic(projectOption: Option[String]) = ReadAction.async{ implicit req =>
|
|
val (lastRefreshTime, resultsFuture) = projectReportsProvider.resultsForVersions(versions)
|
|
resultsFuture flatMap { case (successfulResults, failedResults) =>
|
|
select(successfulResults, projectOption).fold(Future.successful(notFound())){ selection =>
|
|
val tagsFuture = tagsService.all
|
|
val parsedReports = selection.result
|
|
for{
|
|
tagStatistics <- statisticsForTags(parsedReports, tagsFuture)
|
|
} yield Ok(views.html.statistics.basic(
|
|
tagStatistics = tagStatistics,
|
|
projectsWithSelection = selection.projectsWithSelection,
|
|
parsedReports = parsedReports
|
|
))
|
|
}
|
|
}
|
|
}
|
|
|
|
def statisticsForTags(parsedReports: DependencyCheckReportsParser.Result, tagsFuture: Future[Seq[(Int, LibraryTag)]]): Future[Seq[Statistics.TagStatistics]] = {
|
|
val librariesFuture = librariesService.byPlainLibraryIdentifiers(parsedReports.allDependencies.flatMap(_._1.plainLibraryIdentifiers).toSet)
|
|
val libraryTagAssignmentsFuture = librariesFuture.flatMap{libraries => libraryTagAssignmentsService.forLibraries(libraries.values.map(_._1).toSet)}
|
|
val tagsToLibrariesFuture = libraryTagAssignmentsService.tagsToLibraries(libraryTagAssignmentsFuture)
|
|
val librariesToDependencies = parsedReports.groupedDependenciesByPlainLibraryIdentifier
|
|
for{
|
|
librariesById <- librariesFuture.map(_.values.toMap)
|
|
tagsToLibraries <- tagsToLibrariesFuture
|
|
tags <- tagsFuture
|
|
} yield tags.flatMap{case tagRecord @ (tagId, tag) =>
|
|
val libraryAssignments = tagsToLibraries(tagId)
|
|
val tagLibraries = libraryAssignments.map(a => a.libraryId -> librariesById(a.libraryId))
|
|
val tagDependencies: Set[GroupedDependency] = tagLibraries.flatMap{case (_, lib) => librariesToDependencies(lib.plainLibraryIdentifier)}
|
|
// TODO: vulnerabilities in the past
|
|
if(tagLibraries.isEmpty) None
|
|
else Some(TagStatistics(tagRecord = tagRecord, stats = LibDepStatistics(libraries = tagLibraries, dependencies = tagDependencies)))
|
|
}
|
|
}
|
|
|
|
def vulnerabilities(projectOption: Option[String], tagIdOption: Option[Int]) = ReadAction.async {implicit req =>
|
|
val (lastRefreshTime, resultsFuture) = projectReportsProvider.resultsForVersions(versions)
|
|
resultsFuture flatMap { case (successfulResults, failedResults) =>
|
|
select(successfulResults, projectOption).fold(Future.successful(notFound())){ selection =>
|
|
val parsedReports = selection.result
|
|
for{
|
|
libraries <- librariesService.byPlainLibraryIdentifiers(parsedReports.allDependencies.flatMap(_._1.plainLibraryIdentifiers).toSet)
|
|
tagOption <- tagIdOption.fold[Future[Option[(Int, LibraryTag)]]](Future.successful(None))(tagId => tagsService.getById(tagId).map(Some(_)))
|
|
statistics <- tagOption.fold(Future.successful(LibDepStatistics(dependencies = parsedReports.groupedDependencies.toSet, libraries = libraries.values.toSet))){ tag =>
|
|
statisticsForTags(parsedReports, Future.successful(Seq(tag))).map{
|
|
case Seq(TagStatistics(_, stats)) => stats // statisticsForTags is designed for multiple tags, but we have just one…
|
|
case Seq() => LibDepStatistics(libraries = Set(), dependencies = Set()) // We don't want to crash when no dependencies are there…
|
|
}
|
|
}
|
|
} yield Ok(views.html.statistics.vulnerabilities(
|
|
projectsWithSelection = selection.projectsWithSelection,
|
|
tagOption = tagOption,
|
|
statistics = statistics
|
|
))
|
|
}
|
|
}
|
|
}
|
|
|
|
def vulnerability(name: String, projectOption: Option[String]) = ReadAction.async { implicit req =>
|
|
val (lastRefreshTime, resultsFuture) = projectReportsProvider.resultsForVersions(versions)
|
|
resultsFuture flatMap { case (successfulResults, failedResults) =>
|
|
select(successfulResults, projectOption).fold(Future.successful(notFound())){ selection =>
|
|
val relevantReports = selection.result
|
|
val vulns = relevantReports.vulnerableDependencies.flatMap(dep => dep.vulnerabilities.map(vuln => (vuln, dep))).groupBy(_._1.name).mapValues{case vulnsWithDeps =>
|
|
val (vulnSeq, depSeq) = vulnsWithDeps.unzip
|
|
val Seq(vuln) = vulnSeq.toSet.toSeq // Will fail when there are more different descriptions for one vulnerability…
|
|
vuln -> depSeq.toSet
|
|
}// .map(identity) // The .map(identity) materializes lazily mapped Map (because .mapValues is lazy). I am, however, unsure if this is a good idea. Probably not.
|
|
vulns.get(name).fold(Future.successful(Ok(views.html.statistics.vulnerabilityNotFound(
|
|
name = name,
|
|
projectsWithSelection = selection.projectsWithSelection
|
|
)))){ case (vuln, vulnerableDependencies) =>
|
|
for {
|
|
plainLibs <- librariesService.byPlainLibraryIdentifiers(vulnerableDependencies.flatMap(_.plainLibraryIdentifiers)).map(_.keySet)
|
|
ticketOption <- vulnerabilityNotificationService.issueTrackerExport.ticketForVulnerability(name)
|
|
} yield Ok(views.html.statistics.vulnerability(
|
|
vulnerability = vuln,
|
|
affectedProjects = vulnerableDependencies.flatMap(dep => dep.projects.map(proj => (proj, dep))).groupBy(_._1).mapValues(_.map(_._2)),
|
|
vulnerableDependencies = vulnerableDependencies,
|
|
affectedLibraries = plainLibs,
|
|
projectsWithSelection = selection.projectsWithSelection,
|
|
issueOption = for{
|
|
ticket <- ticketOption
|
|
issueTrackerService <- issueTrackerServiceOption
|
|
} yield ticket -> issueTrackerService.ticketLink(ticket)
|
|
))
|
|
}
|
|
|
|
}
|
|
}
|
|
}
|
|
|
|
def vulnerableLibraries(project: Option[String]) = ReadAction.async { implicit req =>
|
|
val (lastRefreshTime, resultsFuture) = projectReportsProvider.resultsForVersions(versions)
|
|
resultsFuture flatMap { case (successfulResults, failedResults) =>
|
|
select(successfulResults, project).fold(Future.successful(notFound())){selection =>
|
|
val reports = selection.result
|
|
Future.successful(Ok(views.html.statistics.vulnerableLibraries(
|
|
projectsWithSelection = selection.projectsWithSelection,
|
|
vulnerableDependencies = reports.vulnerableDependencies,
|
|
allDependenciesCount = reports.groupedDependencies.size
|
|
)))
|
|
}
|
|
}
|
|
}
|
|
|
|
def allLibraries(project: Option[String]) = ReadAction.async { implicit req =>
|
|
val (lastRefreshTime, resultsFuture) = projectReportsProvider.resultsForVersions(versions)
|
|
resultsFuture flatMap { case (successfulResults, failedResults) =>
|
|
select(successfulResults, project).fold(Future.successful(notFound())){selection =>
|
|
Future.successful(Ok(views.html.statistics.allLibraries(
|
|
projectsWithSelection = selection.projectsWithSelection,
|
|
allDependencies = selection.result.groupedDependencies
|
|
)))
|
|
}
|
|
}
|
|
}
|
|
|
|
def allGavs(project: Option[String]) = ReadAction.async { implicit req =>
|
|
val (lastRefreshTime, resultsFuture) = projectReportsProvider.resultsForVersions(versions)
|
|
resultsFuture flatMap { case (successfulResults, failedResults) =>
|
|
select(successfulResults, project).fold(Future.successful(notFound())){selection =>
|
|
Future.successful(Ok(Txt(
|
|
selection.result.groupedDependencies.flatMap(_.mavenIdentifiers).toSet.toIndexedSeq.sortBy((id: Identifier) => (id.identifierType, id.name)).map(id => id.name.split(':') match {
|
|
case Array(g, a, v) =>
|
|
s""""${id.identifierType}", "$g", "$a", "$v", "${id.url}" """
|
|
}).mkString("\n")
|
|
)))
|
|
}
|
|
}
|
|
}
|
|
|
|
|
|
}
|