Initial commit

This commit is contained in:
Šesták Vít
2016-01-10 17:31:07 +01:00
commit 4b87ced31f
104 changed files with 4870 additions and 0 deletions

View File

@@ -0,0 +1,166 @@
package com.ysoft.odc
import com.google.inject.Inject
import com.google.inject.name.Named
import org.ccil.cowan.tagsoup.jaxp.SAXFactoryImpl
import play.api.libs.ws.{WS, WSAuthScheme, WSClient, WSRequest}
import upickle.default._
import scala.concurrent.{ExecutionContext, Future}
import scala.util.{Failure, Success, Try}
import scala.xml.Node
final case class Link(
href: String,
rel: String
)
final case class Artifact(
name: String,
link: Link
//size: Option[Long]
){
def url: String = link.href
}
final case class Artifacts(
size: Int,
//`start-index`: Int,
//`max-result`: Int
artifact: Seq[Artifact]
)
final case class Build(
state: String,
//link: Link,
buildResultKey: String,
buildState: String,
projectName: String,
artifacts: Artifacts
) {
def resultLink(urlBase: String): String = s"$urlBase/browse/$buildResultKey/log"
}
sealed trait FlatArtifactItem{
def name: String
}
abstract sealed class ArtifactItem{
def name: String
final def flatFiles: Map[String, Array[Byte]] = flatFilesWithPrefix("")
def flatFilesWithPrefix(prefix: String): Map[String, Array[Byte]]
def toTree(indent: Int = 0): String
def toTree: String = toTree(0)
}
final case class ArtifactFile(name: String, data: Array[Byte]) extends ArtifactItem with FlatArtifactItem{
override def toTree(indent: Int): String = " "*indent + s"$name = $data"
override def flatFilesWithPrefix(prefix: String): Map[String, Array[Byte]] = Map(prefix + name -> data)
def dataString = new String(data, "utf-8")
}
final case class ArtifactDirectory(name: String, items: Map[String, ArtifactItem]) extends ArtifactItem{
override def toTree(indent: Int): String = " "*indent + s"$name:\n"+items.values.map(_.toTree(indent+2)).mkString("\n")
override def flatFilesWithPrefix(prefix: String): Map[String, Array[Byte]] = items.values.flatMap(_.flatFilesWithPrefix(s"$prefix$name/")).toMap
}
final case class FlatArtifactDirectory(name: String, items: Seq[(String, String)]) extends FlatArtifactItem{}
trait BambooAuthentication{
def addAuth(request: WSRequest): WSRequest
}
class SessionIdBambooAuthentication(sessionId: String) extends BambooAuthentication{
override def addAuth(request: WSRequest): WSRequest = request.withHeaders("Cookie" -> s"JSESSIONID=${sessionId.takeWhile(_.isLetterOrDigit)}")
}
class CredentialsBambooAuthentication(user: String, password: String) extends BambooAuthentication{
override def addAuth(request: WSRequest): WSRequest = request.withQueryString("os_authType" -> "basic").withAuth(user, password, WSAuthScheme.BASIC)
}
final class BambooDownloader @Inject() (@Named("bamboo-server-url") val server: String, auth: BambooAuthentication)(implicit executionContext: ExecutionContext, wSClient: WSClient) extends Downloader {
private object ArtifactKeys{
val BuildLog = "Build log"
val ResultsHtml = "Report results-HTML"
val ResultsXml = "Report results-XML"
}
private def downloadArtifact(artifactMap: Map[String, Artifact], key: String)(implicit wSClient: WSClient): Future[FlatArtifactItem] = {
val artifact = artifactMap(key)
downloadArtifact(artifact.url, artifact.name)
}
private def downloadArtifact(url: String, name: String)(implicit wSClient: WSClient): Future[FlatArtifactItem] = {
bambooUrl(url).get().map{response =>
response.header("Content-Disposition") match{
case Some(_) => ArtifactFile(name = name, data = response.bodyAsBytes)
case None =>
val html = response.body
val hpf = new SAXFactoryImpl
hpf.setFeature("http://xml.org/sax/features/external-general-entities", false)
//hpf.setFeature("http://apache.org/xml/features/disallow-doctype-decl", true)
hpf.setFeature("http://xml.org/sax/features/external-parameter-entities", false);
val HtmlParser = hpf.newSAXParser()
val Html = scala.xml.XML.withSAXParser(HtmlParser)
val xml = Html.loadString(html)
val tds = xml \\ "td"
val subdirs = tds flatMap { td =>
(td \ "img").headOption.flatMap{img =>
val suffix = img.attribute("alt").map(_.text) match { // suffix seems to be no longer needed, as we recognize directories elsehow
case Some("(dir)") => "/"
case Some("(file)") => ""
case other => sys.error(s"unexpected directory item type: $other")
}
(td \ "a").headOption.map{ link =>
val hrefAttribute: Option[Seq[Node]] = link.attribute("href")
link.text -> (hrefAttribute.getOrElse(sys.error(s"bad link $link at $url")).text+suffix) : (String, String)
} : Option[(String, String)]
} : Option[(String, String)]
}
FlatArtifactDirectory(name = name, items = subdirs)
}
}
}
private def downloadArtifactRecursively(artifactMap: Map[String, Artifact], key: String)(implicit wSClient: WSClient): Future[ArtifactItem] = {
val artifact = artifactMap(key)
downloadArtifactRecursively(url = artifact.url, name = artifact.name)
}
private def downloadArtifactRecursively(url: String, name: String/*artifactMap: Map[String, Artifact], key: String*/)(implicit wSClient: WSClient): Future[ArtifactItem] = {
downloadArtifact(url/*artifactMap, key*/, name).flatMap{
case directoryStructure: FlatArtifactDirectory =>
Future.traverse(directoryStructure.items){case (subName, urlString) =>
downloadArtifactRecursively(server+urlString, subName)
}.map{ items =>
ArtifactDirectory(name = directoryStructure.name, items = items.map(i => i.name->i).toMap)
}
case file: ArtifactFile => Future.successful(file)
}
}
override def downloadProjectReports(projects: Set[String], requiredVersions: Map[String, Int]): Future[(Map[String, (Build, ArtifactItem, ArtifactFile)], Map[String, Throwable])] = {
val resultsFuture = Future.traverse(projects){project =>
downloadProjectReport(project, requiredVersions.get(project))
}
resultsFuture.map{ results =>
val (successfulReportTries, failedReportTries) = results.partition(_._2.isSuccess)
val successfulReports = successfulReportTries.map{case (name, Success(data)) => name -> data; case _ => ???}.toMap
val failedReports = failedReportTries.map{case (name, Failure(data)) => name -> data; case _ => ???}.toMap
(successfulReports, failedReports)
}
}
private def bambooUrl(url: String) = auth.addAuth(WS.clientUrl(url))
private def downloadProjectReport(project: String, versionOption: Option[Int]): Future[(String, Try[(Build, ArtifactItem, ArtifactFile)])] = {
val url = s"$server/rest/api/latest/result/$project-${versionOption.getOrElse("latest")}.json?expand=artifacts"
val resultFuture = (bambooUrl(url).get().flatMap { response =>
val build = read[Build](response.body)
val artifactMap: Map[String, Artifact] = build.artifacts.artifact.map(x => x.name -> x).toMap
val logFuture = downloadArtifact(artifactMap, ArtifactKeys.BuildLog).map(_.asInstanceOf[ArtifactFile])
val reportsFuture: Future[ArtifactItem] = downloadArtifactRecursively(artifactMap, ArtifactKeys.ResultsXml)
for {
log <- logFuture
reports <- reportsFuture
} yield (build, reports, log)
}: Future[(Build, ArtifactItem, ArtifactFile)])
resultFuture.map(data => project -> Success(data)).recover{case e => project -> Failure(e)}
}
}

View File

@@ -0,0 +1,33 @@
package com.ysoft.odc
import controllers.WarningSeverity.WarningSeverity
import controllers.{IdentifiedWarning, ReportInfo, Warning}
import play.twirl.api.{Html, HtmlFormat}
object Checks {
def differentValues(id: String, name: String, severity: WarningSeverity)(f: Map[ReportInfo, Analysis] => Traversable[_]) = { (data: Map[ReportInfo, Analysis]) =>
val variants = f(data)
if(variants.size > 1){
Some(IdentifiedWarning(id, HtmlFormat.escape(s"different $name!"), severity))
}else{
None
}
}
def badValues(id: String, name: String, severity: WarningSeverity)(f: (ReportInfo, Analysis) => Option[Html]): Map[ReportInfo, Analysis] => Option[Warning] = { (data: Map[ReportInfo, Analysis]) =>
val badValues = data.collect(Function.unlift{case (analysisName, analysis) => f(analysisName, analysis).map(analysisName -> _)}).toSeq
if(badValues.size > 0) Some(IdentifiedWarning(id, views.html.warnings.badValues(name, badValues), severity))
else None
}
def badGroupedDependencies[C <: Traversable[_]](id: String, name: String, severity: WarningSeverity)(f: Seq[GroupedDependency] => C)(show: C => Traversable[_] = {(x: C) => x}, exclusions: Set[Exclusion] = Set()): (Seq[GroupedDependency] => Option[Warning]) = { (data: Seq[GroupedDependency]) =>
val badItems = f(data.filterNot(ds => exclusions.exists(_.matches(ds))))
if(badItems.size > 0){
Some(IdentifiedWarning(id, views.html.warnings.badGroupedDependencies(name, badItems.size, show(badItems)), severity))
}else{
None
}
}
}

View File

@@ -0,0 +1,10 @@
package com.ysoft.odc
import scala.concurrent.Future
/**
* Created by user on 10/30/15.
*/
trait Downloader {
def downloadProjectReports(projects: Set[String], requiredVersions: Map[String, Int]): Future[(Map[String, (Build, ArtifactItem, ArtifactFile)], Map[String, Throwable])]
}

View File

@@ -0,0 +1,17 @@
package com.ysoft.odc
import javax.inject.Named
import com.google.inject.Inject
import scala.concurrent.Future
class LocalFilesDownloader @Inject() (@Named("reports-path") path: String) extends Downloader{
override def downloadProjectReports(projects: Set[String], requiredVersions: Map[String, Int]): Future[(Map[String, (Build, ArtifactItem, ArtifactFile)], Map[String, Throwable])] = {
if(requiredVersions != Map()){
sys.error("Versions are not supported there")
}
projects.map{pn => ???}
???
}
}

View File

@@ -0,0 +1,293 @@
package com.ysoft.odc
import com.github.nscala_time.time.Imports._
import controllers.ReportInfo
import models.{LibraryType, PlainLibraryIdentifier}
import scala.xml._
final case class SerializableXml private (xmlString: String, @transient private val xmlData: NodeSeq) extends Serializable{
@transient lazy val xml = Option(xmlData).getOrElse(SecureXml.loadString(xmlString))
override def equals(obj: scala.Any): Boolean = obj match {
case SerializableXml(s, _) => s == this.xmlString
}
override def hashCode(): Int = 42+xmlString.hashCode
}
object SerializableXml{
def apply(xml: Node): SerializableXml = SerializableXml(xml.toString(), xml)
def apply(xml: NodeSeq): SerializableXml = SerializableXml(xml.toString(), xml)
}
final case class Analysis(scanInfo: SerializableXml, name: String, reportDate: DateTime, dependencies: Seq[Dependency])
final case class Hashes(sha1: String, md5: String){
override def toString: String = s"Hashes(sha1=$sha1, md5=$md5)"
}
final case class Exclusion(sha1: String) extends AnyVal {
def matches(dependency: Dependency): Boolean = dependency.sha1 == sha1
def matches(group: GroupedDependency): Boolean = group.sha1 == sha1
}
final case class Evidence(source: String, name: String, value: String, confidence: String, evidenceType: String)
final case class Dependency(
fileName: String,
filePath: String,
md5: String,
sha1: String,
description: String,
evidenceCollected: Set[Evidence],
identifiers: Seq[Identifier],
license: String,
vulnerabilities: Seq[Vulnerability],
suppressedVulnerabilities: Seq[Vulnerability],
relatedDependencies: SerializableXml
){
def hashes = Hashes(sha1 = sha1, md5 = md5)
def plainLibraryIdentifiers: Set[PlainLibraryIdentifier] = identifiers.flatMap(_.toLibraryIdentifierOption).toSet
/*
Method equals seems to be a CPU hog there. I am not sure if we can do something reasonable about it.
We can compare by this.hashes, but, in such case, dependencies that differ in evidence will be considered the same if their JAR hashes are the same, which would break some sanity checks.
*/
}
/**
* A group of dependencies having the same fingerprints
* @param dependencies
*/
final case class GroupedDependency(dependencies: Map[Dependency, Set[ReportInfo]]) {
def parsedDescriptions: Seq[Seq[Seq[String]]] = descriptions.toSeq.sorted.map(_.split("\n\n").toSeq.map(_.split("\n").toSeq))
def isVulnerable: Boolean = vulnerabilities.nonEmpty
def maxCvssScore = (Seq(None) ++ vulnerabilities.map(_.cvssScore)).max
def ysdssScore = maxCvssScore.map(_ * projects.size)
def descriptions = dependencies.keySet.map(_.description)
def projects = dependencies.values.flatten.toSet
def fileNames = dependencies.keySet.map(_.fileName)
def hashes = dependencies.keys.head.hashes // valid since all deps in a group have the same hashes
val sha1 = hashes.sha1
def identifiers: Set[Identifier] = dependencies.keySet.flatMap(_.identifiers)
def mavenIdentifiers = identifiers.filter(_.identifierType == "maven")
def cpeIdentifiers = identifiers.filter(_.identifierType == "cpe")
def vulnerabilities: Set[Vulnerability] = dependencies.keySet.flatMap(_.vulnerabilities)
def plainLibraryIdentifiers: Set[PlainLibraryIdentifier] = identifiers.flatMap(_.toLibraryIdentifierOption)
def hasCpe: Boolean = cpeIdentifiers.nonEmpty
}
object GroupedDependency{
def apply(deps: Seq[(Dependency, ReportInfo)]): GroupedDependency = GroupedDependency(deps.groupBy(_._1).mapValues(_.map(_._2).toSet)) // TODO: the groupBy seems to be a CPU hog (because of GroupedDependency.equals); The mapValues is lazy, so its repeated might also be a performance hog, but I doubt that values are used frequently.
}
object Confidence extends Enumeration {
type Confidence = Value
// Order is important
val Low = Value("LOW")
val Medium = Value("MEDIUM")
val High = Value("HIGH")
val Highest = Value("HIGHEST")
}
final case class Reference(source: String, url: String, name: String)
final case class VulnerableSoftware(allPreviousVersion: Boolean, name: String)
final case class CvssRating(score: Option[Double], authenticationr: Option[String], availabilityImpact: Option[String], accessVector: Option[String], integrityImpact: Option[String], accessComplexity: Option[String], confidentialImpact: Option[String])
final case class CWE(name: String) extends AnyVal{
override def toString = name
def brief = name.takeWhile(_ != ' ')
def numberOption: Option[Int] = if(brief startsWith "CWE-") try {
Some(brief.substring(4).toInt)
} catch {
case _: NumberFormatException => None
} else None
}
final case class Vulnerability(name: String, cweOption: Option[CWE], cvss: CvssRating, description: String, vulnerableSoftware: Seq[VulnerableSoftware], references: Seq[Reference]){
def cvssScore = cvss.score
def ysvssScore(affectedDeps: Set[GroupedDependency]) = cvssScore.map(_ * affectedDeps.flatMap(_.projects).toSet.size)
}
final case class Identifier(name: String, confidence: Confidence.Confidence, url: String, identifierType: String) {
def toLibraryIdentifierOption: Option[PlainLibraryIdentifier] = {
if(identifierType == "maven"){
val groupId::artifactId::_ = name.split(':').toList
Some(PlainLibraryIdentifier(libraryType = LibraryType.Maven, libraryIdentifier = s"$groupId:$artifactId"))
}else{
None
}
}
def toCpeIdentifierOption: Option[String] = identifierType match {
case "cpe" => Some(name)
case _ => None
}
//def isClassifiedInSet(set: Set[PlainLibraryIdentifier]): Boolean = toLibraryIdentifierOption.exists(set contains _)
}
object OdcParser {
def filterWhitespace(node: Node) = node.nonEmptyChildren.filter{
case t: scala.xml.Text if t.text.trim == "" => false
case t: scala.xml.PCData if t.text.trim == "" => false
case _ => true
}
def checkElements(node: Node, knownElements: Set[String]) {
val subelementNames = filterWhitespace(node).map(_.label).toSet
val unknownElements = subelementNames -- knownElements
if(unknownElements.nonEmpty){
sys.error("Unknown elements for "+node.label+": "+unknownElements)
}
}
private def getAttributes(data: MetaData): List[String] = data match {
case Null => Nil
case Attribute(key, _, next) => key :: getAttributes(next)
}
def checkParams(node: Node, knownParams: Set[String]) {
val paramNames = getAttributes(node.attributes).toSet
val unknownParams = paramNames -- knownParams
if(unknownParams.nonEmpty){
sys.error("Unknown params for "+node.label+": "+unknownParams)
}
}
def parseVulnerableSoftware(node: Node): VulnerableSoftware = {
checkElements(node, Set("#PCDATA"))
checkParams(node, Set("allPreviousVersion"))
if(node.label != "software"){
sys.error(s"Unexpected element for vulnerableSoftware: ${node.label}")
}
VulnerableSoftware(
name = node.text,
allPreviousVersion = node.attribute("allPreviousVersion").map(_.text).map(Map("true"->true, "false"->false)).getOrElse(false)
)
}
def parseReference(node: Node): Reference = {
checkElements(node, Set("source", "url", "name"))
checkParams(node, Set())
if(node.label != "reference"){
sys.error(s"Unexpected element for reference: ${node.label}")
}
Reference(
source = (node \ "source").text,
url = (node \ "url").text,
name = (node \ "name").text
)
}
def parseVulnerability(node: Node, expectedLabel: String = "vulnerability"): Vulnerability = {
checkElements(node, Set("name", "severity", "cwe", "cvssScore", "description", "references", "vulnerableSoftware", "cvssAuthenticationr", "cvssAvailabilityImpact", "cvssAccessVector", "cvssIntegrityImpact", "cvssAccessComplexity", "cvssConfidentialImpact"))
if(node.label != expectedLabel){
sys.error(s"Unexpected element for vuln: ${node.label}")
}
def t(ns: NodeSeq) = {
ns match {
case Seq() => None
case Seq(one) =>
one.attributes match {
case Null =>
one.child match {
case Seq(hopefullyTextChild) =>
hopefullyTextChild match {
case Text(data) => Some(data)
}
}
}
}
}
Vulnerability(
name = (node \ "name").text,
//severity = (node \ "severity"), <- severity is useless, as it is computed from cvssScore :D
cweOption = (node \ "cwe").headOption.map(_.text).map(CWE),
description = (node \ "description").text,
cvss = CvssRating(
score = (node \ "cvssScore").headOption.map(_.text.toDouble),
authenticationr = t(node \ "cvssAuthenticationr"),
availabilityImpact = t(node \ "cvssAvailabilityImpact"),
accessVector = t(node \ "cvssAccessVector"),
integrityImpact = t(node \ "cvssIntegrityImpact"),
accessComplexity = t(node \ "cvssAccessComplexity"),
confidentialImpact = t(node \ "cvssConfidentialImpact")
),
references = (node \ "references").flatMap(filterWhitespace).map(parseReference(_)),
vulnerableSoftware = (node \ "vulnerableSoftware").flatMap(filterWhitespace).map(parseVulnerableSoftware)
)
}
def parseIdentifier(node: Node): Identifier = {
checkElements(node, Set("name", "url"))
checkParams(node, Set("type", "confidence"))
val ExtractPattern = """\((.*)\)""".r
Identifier(
name = (node \ "name").text match {
case ExtractPattern(text) => text
},
url = (node \ "url").text,
identifierType = node.attribute("type").get.text,
confidence = Confidence.withName(node.attribute("confidence").get.text)
)
}
def parseIdentifiers(seq: Node): Seq[Identifier] = {
filterWhitespace(seq.head).map(parseIdentifier(_))
}
def parseDependency(node: Node): Dependency = {
checkElements(node, Set("fileName", "filePath", "md5", "sha1", "description", "evidenceCollected", "identifiers", "license", "vulnerabilities", "relatedDependencies"))
checkParams(node, Set())
val (vulnerabilities: Seq[Node], suppressedVulnerabilities: Seq[Node]) = (node \ "vulnerabilities").headOption.map(filterWhitespace).getOrElse(Seq()).partition(_.label == "vulnerability")
Dependency(
fileName = (node \ "fileName").text,
filePath = (node \ "filePath").text,
md5 = (node \ "md5").text,
sha1 = (node \ "sha1").text,
description = (node \ "description").text,
evidenceCollected = filterWhitespace((node \ "evidenceCollected").head).map(parseEvidence).toSet,
identifiers = (node \ "identifiers").headOption.map(parseIdentifiers).getOrElse(Seq()),
license = (node \ "license").text,
vulnerabilities = vulnerabilities.map(parseVulnerability(_)),
suppressedVulnerabilities = suppressedVulnerabilities.map(parseVulnerability(_, "suppressedVulnerability")),
relatedDependencies = SerializableXml(node \ "relatedDependencies")
)
}
def parseEvidence(node: Node): Evidence = {
if(node.label != "evidence"){
sys.error(s"Unexpected element for evidence: ${node.label}")
}
checkElements(node, Set("source", "name", "value"))
checkParams(node, Set("confidence", "type"))
Evidence(
source = (node \ "source").text,
name = (node \ "name").text,
value = (node \ "value").text,
confidence = node.attribute("confidence").map(_.text).get,
evidenceType = node.attribute("type").map(_.text).get
)
}
def parseDependencies(nodes: NodeSeq): Seq[Dependency] = nodes.map(parseDependency(_))
def parseXmlReport(data: Array[Byte]) = {
val xml = SecureXml.loadString(new String(data, "utf-8"))
Analysis(
scanInfo = SerializableXml((xml \ "scanInfo").head),
name = (xml \ "projectInfo" \ "name").text,
reportDate = DateTime.parse((xml \ "projectInfo" \ "reportDate").text),
dependencies = parseDependencies(xml \ "dependencies" \ "dependency").toIndexedSeq
)
}
}

View File

@@ -0,0 +1,17 @@
package com.ysoft.odc
import javax.xml.parsers.SAXParserFactory
import scala.xml.{Elem, XML}
// copied from https://github.com/scala/scala-xml/issues/17 and slightly modified
object SecureXml {
def loadString(xml: String): Elem = {
val spf = SAXParserFactory.newInstance()
spf.setFeature("http://xml.org/sax/features/external-general-entities", false)
spf.setFeature("http://apache.org/xml/features/disallow-doctype-decl", true)
val saxParser = spf.newSAXParser()
XML.withSAXParser(saxParser).loadString(xml)
}
}