Added new ODC scans for Java libraries. Those can scan even transitive dependencies and can be run before adding a new library to a project.

2026-04-30 12:24:26 +02:00 · 2017-07-31 12:09:23 +02:00
parent bb0089cd97
commit 2049759430
31 changed files with 824 additions and 200 deletions
--- a/app/controllers/Application.scala
+++ b/app/controllers/Application.scala
@@ -285,7 +285,8 @@ class Application @Inject() (
    Ok(
      JavaScriptReverseRouter("Routes")(
        routes.javascript.Application.setClassified,
-        routes.javascript.Application.addTag
+        routes.javascript.Application.addTag,
+        routes.javascript.LibraryAdvisor.scan
      )
    ).as("text/javascript")
  }
--- a/app/controllers/DependencyCheckReportsParser.scala
+++ b/app/controllers/DependencyCheckReportsParser.scala
@@ -113,6 +113,7 @@ private final case class BadFilter(pattern: String) extends Filter{
 }

 object DependencyCheckReportsParser{
+  def forAdHocScan(analysis: Analysis): Result = Result(Map(ReportInfo("adHocScan", "Ad hoc scan", "AHS", None) -> analysis), Map(), new ProjectsWithReports(new Projects(Map(), Map(), Map()), Set()), Map())
  final case class ResultWithSelection(result: Result, projectsWithSelection: ProjectsWithSelection)
  final case class Result(bareFlatReports: Map[ReportInfo, Analysis], bareFailedAnalysises: Map[ReportInfo, Throwable], projectsReportInfo: ProjectsWithReports/*TODO: maybe rename to rootProjects*/, failedReportDownloads: Map[ReportInfo, Throwable]){
    //lazy val projectsReportInfo = new ProjectsWithReports(projects, (bareFlatReports.keySet ++ bareFailedAnalysises.keySet ++ failedReportDownloads.keySet).map(_.fullId)) // TODO: consider renaming to projectsWithReports
@@ -126,6 +127,7 @@ object DependencyCheckReportsParser{
      groupedDependencies.toSet.flatMap((grDep: GroupedDependency) => grDep.plainLibraryIdentifiers.map(_ -> grDep)).groupBy(_._1).mapValues(_.map(_._2)).map(identity)
    lazy val groupedDependenciesByHashes: Map[Hashes, GroupedDependency] = groupedDependencies.map(gd => gd.hashes -> gd).toMap
    lazy val vulnerableDependencies = groupedDependencies.filter(_.vulnerabilities.nonEmpty)
+    lazy val nonVulnerableDependencies = groupedDependencies.filter(_.vulnerabilities.isEmpty)
    lazy val suppressedOnlyDependencies = groupedDependencies.filter(gd => gd.vulnerabilities.isEmpty && gd.suppressedIdentifiers.nonEmpty)

    private val ProjectSelectorPattern = """^project:(.*)$""".r
--- a/app/controllers/LibraryAdvisor.scala
+++ b/app/controllers/LibraryAdvisor.scala
@@ -0,0 +1,114 @@
+package controllers
+
+import java.net.URL
+import javax.inject.Inject
+
+import com.github.nscala_time.time.Imports._
+import com.ysoft.odc.SecureXml
+import modules.TemplateCustomization
+import org.joda.time.DateTime
+import play.api.Configuration
+import play.api.i18n.MessagesApi
+import play.api.mvc.{Action, AnyContent, Result}
+import play.twirl.api.Html
+import services.{DependencyNotFoundException, OdcDbService, OdcService, SingleLibraryScanResult}
+import views.html.DefaultRequest
+
+import scala.concurrent.{ExecutionContext, Future}
+import scala.util.Try
+
+class LibraryAdvisor @Inject() (
+                      config: Configuration,
+                      odcServiceOption: Option[OdcService],
+                      odcDbService: OdcDbService,
+                      val messagesApi: MessagesApi,
+                      val env: AuthEnv,
+                      val templateCustomization: TemplateCustomization
+                    ) (implicit ec: ExecutionContext) extends AuthenticatedController
+{
+
+  import secureRequestConversion._
+
+  private def withOdc(f: OdcService => Future[Result])(implicit defaultRequest: DefaultRequest) = {
+    odcServiceOption.fold(Future.successful(InternalServerError(views.html.libraryAdvisor.notEnabled())))(odcService =>
+      f(odcService)
+    )
+  }
+
+  private val InputParsers = Seq[(OdcService, String) => Option[Either[Future[SingleLibraryScanResult], String]]](
+    (odcService, xmlString) => {
+      val triedElem = Try {
+        SecureXml.loadString(xmlString)
+      }
+      triedElem.toOption.map{ xml =>
+        xml.label match {
+          case "dependency" =>
+            /*
+            Maven POM, e.g.:
+            <dependency>
+                <groupId>com.google.code.gson</groupId>
+                <artifactId>gson</artifactId>
+                <version>2.3.1</version>
+            </dependency>
+            */
+            val groupId = (xml \ "groupId").text
+            val artifactId = (xml \ "artifactId").text
+            val version = (xml \ "version").text
+            Left(odcService.scanMaven(groupId, artifactId, version))
+          case other =>
+            Right(s"Unknown root XML element: $other")
+        }
+      }
+    },
+    (odcService, urlString) => {
+      Try{new URL(urlString)}.toOption.map{url =>
+        url.getHost match {
+          case "www.mvnrepository.com" | "mvnrepository.com" =>
+            // https://www.mvnrepository.com/artifact/ch.qos.logback/logback-classic/0.9.10
+            // https://mvnrepository.com/artifact/ch.qos.logback/logback-classic/0.9.10
+            url.getPath.split('/') match {
+              case Array("", "artifact", groupId, artifactId, version) =>
+                Left(odcService.scanMaven(groupId, artifactId, version))
+              case _ =>
+                Right("Unknown path for mvnrepository.com: Expected https://mvnrepository.com/artifact/<groupId>/<artifactId>/<version>")
+            }
+        }
+      }
+    }
+  )
+
+  def index(dependency: Option[String]): Action[AnyContent] = ReadAction.async{ implicit req =>
+    withOdc{ odcService =>
+      Future.successful(Ok(views.html.libraryAdvisor.scanLibrary(dependency, Seq(
+        Html("&lt;dependency>…&lt;/dependency> – Maven POM format"),
+        Html("https://mvnrepository.com/artifact/<i>groupId</i>/<i>artifactId</i>/<i>version</i>")
+      ))))
+    }
+  }
+
+  //noinspection TypeAnnotation
+  def scan() = ReadAction.async(parse.json[String]){ implicit req =>
+    withOdc{ odcService =>
+      val now = DateTime.now()
+
+      val oldDataThreshold = 2.days
+      val lastDbUpdateFuture = odcDbService.loadLastDbUpdate()
+      val isOldFuture = lastDbUpdateFuture.map{ lastUpdate => now - oldDataThreshold > lastUpdate}
+
+      val response = InputParsers.toStream.map(_(odcService, req.body)).find(_.nonEmpty).flatten match{
+        case None => Future.successful(Ok(views.html.libraryAdvisor.scanInputError("Unknown input format")))
+        case Some(Right(message)) => Future.successful(Ok(views.html.libraryAdvisor.scanInputError(s"Unknown input format: $message")))
+        case Some(Left(resFuture)) =>
+          for{
+            res <- resFuture
+            isOld <- isOldFuture
+          } yield Ok(views.html.libraryAdvisor.scanResults(isOld, res))
+      }
+      response.recover{
+        case DependencyNotFoundException(dependency) =>
+          NotFound(views.html.libraryAdvisor.notFound(dependency))
+      }.map { _.withHeaders("Content-type" -> "text/plain; charset=utf-8")}
+    }
+  }
+
+}
--- a/app/controllers/Notifications.scala
+++ b/app/controllers/Notifications.scala
@@ -26,7 +26,7 @@ class Notifications @Inject()(
  dependencyCheckReportsParser: DependencyCheckReportsParser,
  issueTrackerServiceOption: Option[IssueTrackerService],
  emailExportServiceOption: Option[EmailExportService],
-  odcService: OdcService,
+  odcService: OdcDbService,
  absolutizer: Absolutizer,
  val env: AuthEnv,
  val templateCustomization: TemplateCustomization
--- a/app/controllers/Statistics.scala
+++ b/app/controllers/Statistics.scala
@@ -67,7 +67,8 @@ class Statistics @Inject()(
  dependencyCheckReportsParser: DependencyCheckReportsParser,
  librariesService: LibrariesService,
  tagsService: TagsService,
-  odcService: OdcService,
+  odcDbService: OdcDbService,
+  odcServiceOption: Option[OdcService],
  libraryTagAssignmentsService: LibraryTagAssignmentsService,
  @Named("missing-GAV-exclusions") missingGAVExclusions: MissingGavExclusions,
  projects: Projects,
@@ -98,14 +99,14 @@ class Statistics @Inject()(
    }else{
      val now = DateTime.now()
      val oldDataThreshold = 2.days
-      val lastDbUpdateFuture = odcService.loadLastDbUpdate()
+      val lastDbUpdateFuture = odcDbService.loadLastDbUpdate()
      val isOldFuture = lastDbUpdateFuture.map{ lastUpdate => now - oldDataThreshold > lastUpdate}
      versionOption match {
        case Some(version) =>
          for {
-            res1 <- Future.traverse(versionlessCpes) { versionlessCpe => odcService.findRelevantCpes(versionlessCpe, version) }
+            res1 <- Future.traverse(versionlessCpes) { versionlessCpe => odcDbService.findRelevantCpes(versionlessCpe, version) }
            vulnIds = res1.flatten.map(_.vulnerabilityId).toSet
-            vulns <- Future.traverse(vulnIds)(id => odcService.getVulnerabilityDetails(id).map(_.get))
+            vulns <- Future.traverse(vulnIds)(id => odcDbService.getVulnerabilityDetails(id).map(_.get))
            isOld <- isOldFuture
          } yield Ok(views.html.statistics.vulnerabilitiesForLibrary(
            vulnsAndVersionOption = Some((vulns, version)),
@@ -213,17 +214,27 @@ class Statistics @Inject()(
        }// .map(identity) // The .map(identity) materializes lazily mapped Map (because .mapValues is lazy). I am, however, unsure if this is a good idea. Probably not.
        vulns.get(name).fold{
          for{
-            vulnOption <- odcService.getVulnerabilityDetails(name)
+            vulnOption <- odcDbService.getVulnerabilityDetails(name)
            issueOption <- issueOptionFuture
-          } yield Ok(views.html.statistics.vulnerabilityNotFound( // TODO: the not found page might be replaced by some page explaining that there is no project affected by that vulnerability
-            name = name,
+          } yield vulnOption.fold(
+            Ok(views.html.statistics.vulnerabilityNotFound(
+              name = name,
+              projectsWithSelection = selection.projectsWithSelection,
+              failedProjects = selection.result.failedProjects,
+              issueOption = issueOption
+            ))
+          )(vuln => Ok(views.html.statistics.vulnerability(
            projectsWithSelection = selection.projectsWithSelection,
            failedProjects = selection.result.failedProjects,
-            issueOption = issueOption
-          ))
+            issueOption = issueOption,
+            vulnerability = vuln,
+            affectedProjects = Map(),
+            affectedLibraries = Set(),
+            vulnerableDependencies = Set()
+          )))
        }{ vulnerableDependencies =>
          for {
-            vulnOption <- odcService.getVulnerabilityDetails(name)
+            vulnOption <- odcDbService.getVulnerabilityDetails(name)
            plainLibs <- librariesService.byPlainLibraryIdentifiers(vulnerableDependencies.flatMap(_.plainLibraryIdentifiers)).map(_.keySet)
            issueOption <- issueOptionFuture
          } yield vulnOption.fold{
--- a/app/controllers/package.scala
+++ b/app/controllers/package.scala
@@ -1,6 +1,7 @@
 import com.mohiva.play.silhouette.api.Environment
 import com.mohiva.play.silhouette.impl.authenticators.CookieAuthenticator
 import models.{SnoozeInfo, User}
+import play.api.mvc.Call

 /**
 * Created by user on 7/15/15.
@@ -35,4 +36,30 @@ package object controllers {
  }*/
  def friendlyProjectNameString(reportInfo: ReportInfo) = reportInfo.subprojectNameOption.fold(reportInfo.projectName)(reportInfo.projectName+": "+_)

+  val severityOrdering: Ordering[GroupedDependency] = Ordering.by((d: GroupedDependency) => (
+    d.maxCvssScore.map(-_).getOrElse(0.0),                                          // maximum CVSS score is the king
+    if(d.maxCvssScore.isEmpty) Some(-d.dependencies.size) else None,                // more affected dependencies if no vulnerability has defined severity
+    -d.vulnerabilities.size,                                                        // more vulnerabilities
+    -d.projects.size,                                                               // more affected projects
+    d.cpeIdentifiers.map(_.toCpeIdentifierOption.get).toSeq.sorted.mkString(" "))   // at least make the order deterministic
+  )
+
+  def vulnerableSoftwareSearches(groupedDependency: GroupedDependency): Seq[(Call, String)] = {
+    val legacySearchOption = groupedDependency.cpeIdentifiers match {
+      case Seq() => None
+      case cpeIds => Some(
+        routes.Statistics.searchVulnerableSoftware(
+          cpeIds.map(_.name.split(':').take(4).mkString(":")).toSeq, None
+        ) -> "Search by CPE (legacy option)"
+      )
+    }
+    val mavenSearches = groupedDependency.mavenIdentifiers.map(_.name).toSeq.sorted.map{mavenIdentifier =>
+      val Array(groupId, artifactId, version) = mavenIdentifier.split(":", 3)
+      val identifierString = <dependency><groupId>{groupId}</groupId><artifactId>{artifactId}</artifactId><version>{version}</version></dependency>.toString()
+      routes.LibraryAdvisor.index(Some(identifierString)) -> s"Look for Maven dependency $mavenIdentifier"
+    }
+    mavenSearches ++ legacySearchOption
+
+  }
+
 }