diff --git a/app/com/ysoft/debug/KnownObjects.scala b/app/com/ysoft/debug/KnownObjects.scala new file mode 100644 index 0000000..75336e0 --- /dev/null +++ b/app/com/ysoft/debug/KnownObjects.scala @@ -0,0 +1,25 @@ +package com.ysoft.debug + +import java.util + +import com.google.caliper.memory.ObjectVisitor.Traversal +import play.api.Logger + +// We use Java collections because they can have the initial size configured +case class KnownObjects( + objSet: java.util.HashSet[Any] = new util.HashSet[Any](), + identitiesSet: java.util.Set[Any] = java.util.Collections.newSetFromMap(new util.IdentityHashMap[Any, java.lang.Boolean]()) +){ + def visit(obj: AnyRef) = { + val seen = !identitiesSet.add(obj) + if(seen){ + Traversal.SKIP + }else{ + objSet.add(obj) + Traversal.EXPLORE + } + } + + def stats = (identitiesSet.size, objSet.size) + +} diff --git a/app/com/ysoft/debug/ObjectGraphDuplicityMeasurer.scala b/app/com/ysoft/debug/ObjectGraphDuplicityMeasurer.scala new file mode 100644 index 0000000..0e3a3fa --- /dev/null +++ b/app/com/ysoft/debug/ObjectGraphDuplicityMeasurer.scala @@ -0,0 +1,44 @@ +package com.ysoft.debug + +import java.util + +import com.google.caliper.memory.ObjectVisitor.Traversal +import com.google.caliper.memory.{Chain, ObjectExplorer, ObjectVisitor} + +import scala.collection.mutable + +object ObjectGraphDuplicityMeasurer { + + def measureUnique(obj: AnyRef) = { + ObjectExplorer.exploreObject(obj, new ObjectVisitor[((Int, Int), Map[Class[_], (Int, Int)])](){ + val all = KnownObjects( + objSet = new util.HashSet[Any](), + identitiesSet = java.util.Collections.newSetFromMap(new util.IdentityHashMap[Any, java.lang.Boolean]()) + ) + + val classMap = mutable.Map[Class[_], KnownObjects]() + def forClass(cl: Class[_]) = classMap.contains(cl) match{ + case true => classMap(cl) + case false => + val kn = KnownObjects() + classMap(cl) = kn + kn + } + + override def visit(chain: Chain): Traversal = { + val value = chain.getValue + if(chain.isPrimitive || value == null || classOf[Enum[_]].isAssignableFrom(chain.getValueType) || value.isInstanceOf[Class[_]] ){ + Traversal.SKIP + }else{ + val res = all.visit(value) + forClass(value.getClass).visit(value) + res + } + } + + override def result() = (all.stats, classMap.toMap.mapValues(_.stats).map(identity)) + + }) + } + +} diff --git a/app/com/ysoft/memory/ObjectPool.scala b/app/com/ysoft/memory/ObjectPool.scala new file mode 100644 index 0000000..d890e94 --- /dev/null +++ b/app/com/ysoft/memory/ObjectPool.scala @@ -0,0 +1,24 @@ +package com.ysoft.memory + +import java.lang.ref.{WeakReference => JWeakReference} +import java.util + +class ObjectPool{ + + private val objects = new util.WeakHashMap[Any, JWeakReference[Any]]()//new MapMaker().concurrencyLevel(1).weakKeys().weakValues().makeMap[Any, Any]() + + def apply[T](obj: T): T = synchronized{ + // The code is intentionally low-level for performance reasons. No Option[_] used for performance reasons, no scala.ref._ wrapper is used for memory overhead reasons. + val res = objects.get(obj) match { + case null => null + case weakObj => weakObj.get() + } + if(res == null){ + objects.put(obj, new JWeakReference[Any](obj)) + obj + }else{ + res.asInstanceOf[T] + } + } + +} diff --git a/app/com/ysoft/odc/OdcParser.scala b/app/com/ysoft/odc/OdcParser.scala index 9f0f5f3..8b254ff 100644 --- a/app/com/ysoft/odc/OdcParser.scala +++ b/app/com/ysoft/odc/OdcParser.scala @@ -1,16 +1,19 @@ package com.ysoft.odc import com.github.nscala_time.time.Imports._ +import com.ysoft.memory.ObjectPool import controllers.ReportInfo import models.{LibraryType, PlainLibraryIdentifier} import scala.xml._ -final case class SerializableXml private (xmlString: String, @transient private val xmlData: NodeSeq) extends Serializable{ - @transient lazy val xml = Option(xmlData).getOrElse(SecureXml.loadString(xmlString)) + +final case class SerializableXml private (xmlString: String) extends Serializable{ + def xml = SecureXml.loadString(xmlString) // TODO: cache override def equals(obj: scala.Any): Boolean = obj match { - case SerializableXml(s, _) => s == this.xmlString + case SerializableXml(s/*, _*/) => s == this.xmlString + case other => false } override def hashCode(): Int = 42+xmlString.hashCode @@ -18,8 +21,8 @@ final case class SerializableXml private (xmlString: String, @transient private } object SerializableXml{ - def apply(xml: Node): SerializableXml = SerializableXml(xml.toString(), xml) - def apply(xml: NodeSeq): SerializableXml = SerializableXml(xml.toString(), xml) + def apply(xml: Node): SerializableXml = SerializableXml(xml.toString()) + def apply(xml: NodeSeq): SerializableXml = SerializableXml(xml.toString()) } final case class Analysis(scanInfo: SerializableXml, name: String, reportDate: DateTime, dependencies: Seq[Dependency]) @@ -61,6 +64,7 @@ final case class Dependency( /** * A group of dependencies having the same fingerprints + * * @param dependencies */ final case class GroupedDependency(dependencies: Map[Dependency, Set[ReportInfo]]) { @@ -82,7 +86,10 @@ final case class GroupedDependency(dependencies: Map[Dependency, Set[ReportInfo] } object GroupedDependency{ - def apply(deps: Seq[(Dependency, ReportInfo)]): GroupedDependency = GroupedDependency(deps.groupBy(_._1).mapValues(_.map(_._2).toSet)) // TODO: the groupBy seems to be a CPU hog (because of GroupedDependency.equals); The mapValues is lazy, so its repeated might also be a performance hog, but I doubt that values are used frequently. + private val groupToSet = (_: Seq[(Dependency, ReportInfo)]).map(_._2).toSet // reduces number of lambda instances + def apply(deps: Seq[(Dependency, ReportInfo)]): GroupedDependency = { + GroupedDependency(deps.groupBy(_._1).mapValues(groupToSet)) + } // TODO: the groupBy seems to be a CPU hog (because of GroupedDependency.equals); The mapValues is lazy, so its repeated might also be a performance hog, but I doubt that values are used frequently. } object Confidence extends Enumeration { @@ -101,7 +108,7 @@ final case class VulnerableSoftware(allPreviousVersion: Boolean, name: String) final case class CvssRating(score: Option[Double], authenticationr: Option[String], availabilityImpact: Option[String], accessVector: Option[String], integrityImpact: Option[String], accessComplexity: Option[String], confidentialImpact: Option[String]) -final case class CWE(name: String) extends AnyVal{ +final case class CWE private(name: String) /*extends AnyVal*/{ // extends AnyVal prevents pooling override def toString = name def brief = name.takeWhile(_ != ' ') def numberOption: Option[Int] = if(brief startsWith "CWE-") try { @@ -111,6 +118,11 @@ final case class CWE(name: String) extends AnyVal{ } else None } +object CWE{ + private val cwePool = new ObjectPool() + def forIdentifierWithDescription(name: String) = cwePool(new CWE(name)) +} + final case class Vulnerability(name: String, cweOption: Option[CWE], cvss: CvssRating, description: String, vulnerableSoftware: Seq[VulnerableSoftware], references: Seq[Reference]){ def cvssScore = cvss.score def ysvssScore(affectedDeps: Set[GroupedDependency]) = cvssScore.map(_ * affectedDeps.flatMap(_.projects).toSet.size) @@ -134,6 +146,12 @@ final case class Identifier(name: String, confidence: Confidence.Confidence, url object OdcParser { + private val vulnPool = new ObjectPool() + private val evidencePool = new ObjectPool() + private val dependencyPool = new ObjectPool() + private val identifierPool = new ObjectPool() + private val vulnerableSoftwarePool = new ObjectPool() + def filterWhitespace(node: Node) = node.nonEmptyChildren.filter{ case t: scala.xml.Text if t.text.trim == "" => false case t: scala.xml.PCData if t.text.trim == "" => false @@ -168,10 +186,10 @@ object OdcParser { if(node.label != "software"){ sys.error(s"Unexpected element for vulnerableSoftware: ${node.label}") } - VulnerableSoftware( + vulnerableSoftwarePool(VulnerableSoftware( name = node.text, allPreviousVersion = node.attribute("allPreviousVersion").map(_.text).map(Map("true"->true, "false"->false)).getOrElse(false) - ) + )) } def parseReference(node: Node): Reference = { @@ -207,10 +225,10 @@ object OdcParser { } } } - Vulnerability( + vulnPool(Vulnerability( name = (node \ "name").text, //severity = (node \ "severity"), <- severity is useless, as it is computed from cvssScore :D - cweOption = (node \ "cwe").headOption.map(_.text).map(CWE), + cweOption = (node \ "cwe").headOption.map(_.text).map(CWE.forIdentifierWithDescription), description = (node \ "description").text, cvss = CvssRating( score = (node \ "cvssScore").headOption.map(_.text.toDouble), @@ -223,21 +241,21 @@ object OdcParser { ), references = (node \ "references").flatMap(filterWhitespace).map(parseReference(_)), vulnerableSoftware = (node \ "vulnerableSoftware").flatMap(filterWhitespace).map(parseVulnerableSoftware) - ) + )) } def parseIdentifier(node: Node): Identifier = { checkElements(node, Set("name", "url")) checkParams(node, Set("type", "confidence")) val ExtractPattern = """\((.*)\)""".r - Identifier( + identifierPool(Identifier( name = (node \ "name").text match { case ExtractPattern(text) => text }, url = (node \ "url").text, identifierType = node.attribute("type").get.text, confidence = Confidence.withName(node.attribute("confidence").get.text) - ) + )) } def parseIdentifiers(seq: Node): Seq[Identifier] = { @@ -248,7 +266,7 @@ object OdcParser { checkElements(node, Set("fileName", "filePath", "md5", "sha1", "description", "evidenceCollected", "identifiers", "license", "vulnerabilities", "relatedDependencies")) checkParams(node, Set()) val (vulnerabilities: Seq[Node], suppressedVulnerabilities: Seq[Node]) = (node \ "vulnerabilities").headOption.map(filterWhitespace).getOrElse(Seq()).partition(_.label == "vulnerability") - Dependency( + dependencyPool(Dependency( fileName = (node \ "fileName").text, filePath = (node \ "filePath").text, md5 = (node \ "md5").text, @@ -260,7 +278,7 @@ object OdcParser { vulnerabilities = vulnerabilities.map(parseVulnerability(_)), suppressedVulnerabilities = suppressedVulnerabilities.map(parseVulnerability(_, "suppressedVulnerability")), relatedDependencies = SerializableXml(node \ "relatedDependencies") - ) + )) } def parseEvidence(node: Node): Evidence = { @@ -269,13 +287,13 @@ object OdcParser { } checkElements(node, Set("source", "name", "value")) checkParams(node, Set("confidence", "type")) - Evidence( + evidencePool(Evidence( source = (node \ "source").text, name = (node \ "name").text, value = (node \ "value").text, confidence = node.attribute("confidence").map(_.text).get, evidenceType = node.attribute("type").map(_.text).get - ) + )) } def parseDependencies(nodes: NodeSeq): Seq[Dependency] = nodes.map(parseDependency(_)) diff --git a/app/controllers/Application.scala b/app/controllers/Application.scala index 44531cc..f69b8f4 100644 --- a/app/controllers/Application.scala +++ b/app/controllers/Application.scala @@ -11,7 +11,7 @@ import play.api.data.Forms._ import play.api.data._ import play.api.db.slick.{DatabaseConfigProvider, HasDatabaseConfigProvider} import play.api.http.ContentTypes -import play.api.i18n.{I18nSupport, MessagesApi} +import play.api.i18n.MessagesApi import play.api.libs.json._ import play.api.mvc._ import play.api.routing.JavaScriptReverseRouter @@ -62,7 +62,6 @@ class Application @Inject() ( import dbConfig.driver.api._ import models.tables.snoozesTable import reportsProcessor.processResults - import secureRequestConversion._ val dateFormatter = DateTimeFormat.forPattern("dd-MM-yyyy") @@ -126,6 +125,11 @@ class Application @Inject() ( lastRefreshTime <- lastRefreshTimeFuture } yield { Logger.debug("indexPage: Got all ingredients") + /*val (global, classes) = ObjectGraphDuplicityMeasurer.measureUnique((vulnerableDependencies, allWarnings, groupedDependencies)) + Logger.debug("(all,unique): "+global) + Logger.debug(classes.toIndexedSeq.sortBy(x => (x._2, x._1.getName)).mkString("\n")) + Logger.debug("footprint: "+ObjectGraphMeasurer.measure((vulnerableDependencies, allWarnings, groupedDependencies))) + //Logger.debug("footprint: "+ObjectGraphMeasurer.measure(Array((vulnerableDependencies, allWarnings, groupedDependencies))))*/ Ok(views.html.index( vulnerableDependencies = vulnerableDependencies, warnings = allWarnings, diff --git a/app/controllers/DependencyCheckReportsProcessor.scala b/app/controllers/DependencyCheckReportsProcessor.scala index 7669bf9..838ce2d 100644 --- a/app/controllers/DependencyCheckReportsProcessor.scala +++ b/app/controllers/DependencyCheckReportsProcessor.scala @@ -96,7 +96,7 @@ final class DependencyCheckReportsProcessor @Inject() ( // TODO: log analysis // TODO: related dependencies - (vulnerableDependencies, allWarnings, groupedDependencies) + (vulnerableDependencies, allWarnings.map(_.optimize), groupedDependencies) } }finally{ Logger.debug("Reports processed") diff --git a/app/controllers/warnings.scala b/app/controllers/warnings.scala index d78657a..671f90b 100644 --- a/app/controllers/warnings.scala +++ b/app/controllers/warnings.scala @@ -12,10 +12,13 @@ object WarningSeverity extends Enumeration { } sealed abstract class Warning { + def optimize: Warning def html: Html def id: String def allowSnoozes = true def severity: WarningSeverity } -final case class IdentifiedWarning(id: String, html: Html, severity: WarningSeverity) extends Warning \ No newline at end of file +final case class IdentifiedWarning(id: String, html: Html, severity: WarningSeverity) extends Warning{ + def optimize = copy(html = Html(html.body)) +} \ No newline at end of file diff --git a/app/models/odc/Vulnerability.scala b/app/models/odc/Vulnerability.scala index 50a3873..9e0b480 100644 --- a/app/models/odc/Vulnerability.scala +++ b/app/models/odc/Vulnerability.scala @@ -19,7 +19,7 @@ class Vulnerabilities(tag: Tag) extends Table[(Int, Vulnerability)](tag, "vulner def cvssConfidentialityImpact = column[String]("cvssConfidentialityImpact").? def cvssRating = (cvssScore, authentication, availabilityImpact, accessVector, integrityImpact, cvssAccessComplexity, cvssConfidentialityImpact) <> (CvssRating.tupled, CvssRating.unapply) - def cweOptionMapped = cweOption <> ((_: Option[String]).map(CWE.apply), (_: Option[CWE]).map(CWE.unapply)) + def cweOptionMapped = cweOption <> ((_: Option[String]).map(CWE.forIdentifierWithDescription), (_: Option[CWE]).map(CWE.unapply)) def base = (cve, description, cweOptionMapped, cvssRating) <> (Vulnerability.tupled, Vulnerability.unapply) def * = (id, base) diff --git a/build.sbt b/build.sbt index d0c18d6..3397ae0 100644 --- a/build.sbt +++ b/build.sbt @@ -77,6 +77,8 @@ libraryDependencies += "org.owasp" % "dependency-check-core" % "1.3.0" libraryDependencies += "com.typesafe.play" %% "play-mailer" % "3.0.1" +libraryDependencies += "com.google.caliper" % "caliper" % "1.0-beta-2" + routesImport += "binders.QueryBinders._" // Uncomment to use Akka