Drastically reduced memory usage, mostly using deduplication.

app/com/ysoft/debug/KnownObjects.scala

@@ -0,0 +1,25 @@
+package com.ysoft.debug
+
+import java.util
+
+import com.google.caliper.memory.ObjectVisitor.Traversal
+import play.api.Logger
+
+// We use Java collections because they can have the initial size configured
+case class KnownObjects(
+  objSet: java.util.HashSet[Any] = new util.HashSet[Any](),
+  identitiesSet: java.util.Set[Any] = java.util.Collections.newSetFromMap(new util.IdentityHashMap[Any, java.lang.Boolean]())
+){
+  def visit(obj: AnyRef) = {
+    val seen = !identitiesSet.add(obj)
+    if(seen){
+      Traversal.SKIP
+    }else{
+      objSet.add(obj)
+      Traversal.EXPLORE
+    }
+  }
+
+  def stats = (identitiesSet.size, objSet.size)
+
+}

app/com/ysoft/debug/ObjectGraphDuplicityMeasurer.scala

@@ -0,0 +1,44 @@
+package com.ysoft.debug
+
+import java.util
+
+import com.google.caliper.memory.ObjectVisitor.Traversal
+import com.google.caliper.memory.{Chain, ObjectExplorer, ObjectVisitor}
+
+import scala.collection.mutable
+
+object ObjectGraphDuplicityMeasurer {
+
+  def measureUnique(obj: AnyRef) = {
+    ObjectExplorer.exploreObject(obj, new ObjectVisitor[((Int, Int), Map[Class[_], (Int, Int)])](){
+      val all = KnownObjects(
+        objSet = new util.HashSet[Any](),
+        identitiesSet = java.util.Collections.newSetFromMap(new util.IdentityHashMap[Any, java.lang.Boolean]())
+      )
+
+      val classMap = mutable.Map[Class[_], KnownObjects]()
+      def forClass(cl: Class[_]) = classMap.contains(cl) match{
+        case true => classMap(cl)
+        case false =>
+          val kn = KnownObjects()
+          classMap(cl) = kn
+          kn
+      }
+
+      override def visit(chain: Chain): Traversal = {
+        val value = chain.getValue
+        if(chain.isPrimitive || value == null || classOf[Enum[_]].isAssignableFrom(chain.getValueType) || value.isInstanceOf[Class[_]] ){
+          Traversal.SKIP
+        }else{
+          val res = all.visit(value)
+          forClass(value.getClass).visit(value)
+          res
+        }
+      }
+
+      override def result() = (all.stats, classMap.toMap.mapValues(_.stats).map(identity))
+
+    })
+  }
+
+}

app/com/ysoft/memory/ObjectPool.scala

@@ -0,0 +1,24 @@
+package com.ysoft.memory
+
+import java.lang.ref.{WeakReference => JWeakReference}
+import java.util
+
+class ObjectPool{
+
+  private val objects = new util.WeakHashMap[Any, JWeakReference[Any]]()//new MapMaker().concurrencyLevel(1).weakKeys().weakValues().makeMap[Any, Any]()
+
+  def apply[T](obj: T): T = synchronized{
+    // The code is intentionally low-level for performance reasons. No Option[_] used for performance reasons, no scala.ref._ wrapper is used for memory overhead reasons.
+    val res = objects.get(obj) match {
+      case null => null
+      case weakObj => weakObj.get()
+    }
+    if(res == null){
+      objects.put(obj, new JWeakReference[Any](obj))
+      obj
+    }else{
+      res.asInstanceOf[T]
+    }
+  }
+
+}

app/com/ysoft/odc/OdcParser.scala

@@ -1,16 +1,19 @@
 package com.ysoft.odc
 
 import com.github.nscala_time.time.Imports._
+import com.ysoft.memory.ObjectPool
 import controllers.ReportInfo
 import models.{LibraryType, PlainLibraryIdentifier}
 
 import scala.xml._
 
-final case class SerializableXml private (xmlString: String, @transient private val xmlData: NodeSeq) extends Serializable{
-  @transient lazy val xml = Option(xmlData).getOrElse(SecureXml.loadString(xmlString))
+
+final case class SerializableXml private (xmlString: String) extends Serializable{
+  def xml = SecureXml.loadString(xmlString) // TODO: cache
 
   override def equals(obj: scala.Any): Boolean = obj match {
-    case SerializableXml(s, _) => s == this.xmlString
+    case SerializableXml(s/*, _*/) => s == this.xmlString
+    case other => false
   }
 
   override def hashCode(): Int = 42+xmlString.hashCode
@@ -18,8 +21,8 @@ final case class SerializableXml private (xmlString: String, @transient private
 }
 
 object SerializableXml{
-  def apply(xml: Node): SerializableXml = SerializableXml(xml.toString(), xml)
-  def apply(xml: NodeSeq): SerializableXml = SerializableXml(xml.toString(), xml)
+  def apply(xml: Node): SerializableXml = SerializableXml(xml.toString())
+  def apply(xml: NodeSeq): SerializableXml = SerializableXml(xml.toString())
 }
 
 final case class Analysis(scanInfo: SerializableXml, name: String, reportDate: DateTime, dependencies: Seq[Dependency])
@@ -61,6 +64,7 @@ final case class Dependency(
 
 /**
  * A group of dependencies having the same fingerprints
+ *
  * @param dependencies
  */
 final case class GroupedDependency(dependencies: Map[Dependency, Set[ReportInfo]]) {
@@ -82,7 +86,10 @@ final case class GroupedDependency(dependencies: Map[Dependency, Set[ReportInfo]
 }
 
 object GroupedDependency{
-  def apply(deps: Seq[(Dependency, ReportInfo)]): GroupedDependency = GroupedDependency(deps.groupBy(_._1).mapValues(_.map(_._2).toSet)) // TODO: the groupBy seems to be a CPU hog (because of GroupedDependency.equals); The mapValues is lazy, so its repeated might also be a performance hog, but I doubt that values are used frequently.
+  private val groupToSet = (_: Seq[(Dependency, ReportInfo)]).map(_._2).toSet // reduces number of lambda instances
+  def apply(deps: Seq[(Dependency, ReportInfo)]): GroupedDependency = {
+    GroupedDependency(deps.groupBy(_._1).mapValues(groupToSet))
+  } // TODO: the groupBy seems to be a CPU hog (because of GroupedDependency.equals); The mapValues is lazy, so its repeated might also be a performance hog, but I doubt that values are used frequently.
 }
 
 object Confidence extends Enumeration {
@@ -101,7 +108,7 @@ final case class VulnerableSoftware(allPreviousVersion: Boolean, name: String)
 
 final case class CvssRating(score: Option[Double], authenticationr: Option[String], availabilityImpact: Option[String], accessVector: Option[String], integrityImpact: Option[String], accessComplexity: Option[String], confidentialImpact: Option[String])
 
-final case class CWE(name: String) extends AnyVal{
+final case class CWE private(name: String) /*extends AnyVal*/{ // extends AnyVal prevents pooling
   override def toString = name
   def brief = name.takeWhile(_ != ' ')
   def numberOption: Option[Int] = if(brief startsWith "CWE-") try {
@@ -111,6 +118,11 @@ final case class CWE(name: String) extends AnyVal{
   } else None
 }
 
+object CWE{
+  private val cwePool = new ObjectPool()
+  def forIdentifierWithDescription(name: String) = cwePool(new CWE(name))
+}
+
 final case class Vulnerability(name: String, cweOption: Option[CWE], cvss: CvssRating, description: String, vulnerableSoftware: Seq[VulnerableSoftware], references: Seq[Reference]){
   def cvssScore = cvss.score
   def ysvssScore(affectedDeps: Set[GroupedDependency]) = cvssScore.map(_ * affectedDeps.flatMap(_.projects).toSet.size)
@@ -134,6 +146,12 @@ final case class Identifier(name: String, confidence: Confidence.Confidence, url
 
 object OdcParser {
 
+  private val vulnPool = new ObjectPool()
+  private val evidencePool = new ObjectPool()
+  private val dependencyPool = new ObjectPool()
+  private val identifierPool = new ObjectPool()
+  private val vulnerableSoftwarePool = new ObjectPool()
+
   def filterWhitespace(node: Node) = node.nonEmptyChildren.filter{
     case t: scala.xml.Text if t.text.trim == "" => false
     case t: scala.xml.PCData if t.text.trim == "" => false
@@ -168,10 +186,10 @@ object OdcParser {
     if(node.label != "software"){
       sys.error(s"Unexpected element for vulnerableSoftware: ${node.label}")
     }
-    VulnerableSoftware(
+    vulnerableSoftwarePool(VulnerableSoftware(
       name = node.text,
       allPreviousVersion = node.attribute("allPreviousVersion").map(_.text).map(Map("true"->true, "false"->false)).getOrElse(false)
-    )
+    ))
   }
 
   def parseReference(node: Node): Reference = {
@@ -207,10 +225,10 @@ object OdcParser {
           }
       }
     }
-    Vulnerability(
+    vulnPool(Vulnerability(
       name = (node \ "name").text,
       //severity = (node \ "severity"), <- severity is useless, as it is computed from cvssScore :D
-      cweOption = (node \ "cwe").headOption.map(_.text).map(CWE),
+      cweOption = (node \ "cwe").headOption.map(_.text).map(CWE.forIdentifierWithDescription),
       description = (node \ "description").text,
       cvss = CvssRating(
         score = (node \ "cvssScore").headOption.map(_.text.toDouble),
@@ -223,21 +241,21 @@ object OdcParser {
       ),
       references = (node \ "references").flatMap(filterWhitespace).map(parseReference(_)),
       vulnerableSoftware = (node \ "vulnerableSoftware").flatMap(filterWhitespace).map(parseVulnerableSoftware)
-    )
+    ))
   }
 
   def parseIdentifier(node: Node): Identifier = {
     checkElements(node, Set("name", "url"))
     checkParams(node, Set("type", "confidence"))
     val ExtractPattern = """\((.*)\)""".r
-    Identifier(
+    identifierPool(Identifier(
       name = (node \ "name").text match {
         case ExtractPattern(text) => text
       },
       url = (node \ "url").text,
       identifierType = node.attribute("type").get.text,
       confidence = Confidence.withName(node.attribute("confidence").get.text)
-    )
+    ))
   }
 
   def parseIdentifiers(seq: Node): Seq[Identifier] = {
@@ -248,7 +266,7 @@ object OdcParser {
     checkElements(node, Set("fileName", "filePath", "md5", "sha1", "description", "evidenceCollected", "identifiers", "license", "vulnerabilities", "relatedDependencies"))
     checkParams(node, Set())
     val (vulnerabilities: Seq[Node], suppressedVulnerabilities: Seq[Node]) = (node \ "vulnerabilities").headOption.map(filterWhitespace).getOrElse(Seq()).partition(_.label == "vulnerability")
-    Dependency(
+    dependencyPool(Dependency(
       fileName = (node \ "fileName").text,
       filePath = (node \ "filePath").text,
       md5 = (node \ "md5").text,
@@ -260,7 +278,7 @@ object OdcParser {
       vulnerabilities = vulnerabilities.map(parseVulnerability(_)),
       suppressedVulnerabilities = suppressedVulnerabilities.map(parseVulnerability(_, "suppressedVulnerability")),
       relatedDependencies = SerializableXml(node \ "relatedDependencies")
-    )
+    ))
   }
 
   def parseEvidence(node: Node): Evidence = {
@@ -269,13 +287,13 @@ object OdcParser {
     }
     checkElements(node, Set("source", "name", "value"))
     checkParams(node, Set("confidence", "type"))
-    Evidence(
+    evidencePool(Evidence(
       source = (node \ "source").text,
       name = (node \ "name").text,
       value = (node \ "value").text,
       confidence = node.attribute("confidence").map(_.text).get,
       evidenceType = node.attribute("type").map(_.text).get
-    )
+    ))
   }
 
   def parseDependencies(nodes: NodeSeq): Seq[Dependency] = nodes.map(parseDependency(_))

app/controllers/Application.scala

@@ -11,7 +11,7 @@ import play.api.data.Forms._
 import play.api.data._
 import play.api.db.slick.{DatabaseConfigProvider, HasDatabaseConfigProvider}
 import play.api.http.ContentTypes
-import play.api.i18n.{I18nSupport, MessagesApi}
+import play.api.i18n.MessagesApi
 import play.api.libs.json._
 import play.api.mvc._
 import play.api.routing.JavaScriptReverseRouter
@@ -62,7 +62,6 @@ class Application @Inject() (
   import dbConfig.driver.api._
   import models.tables.snoozesTable
   import reportsProcessor.processResults
-
   import secureRequestConversion._
 
   val dateFormatter = DateTimeFormat.forPattern("dd-MM-yyyy")
@@ -126,6 +125,11 @@ class Application @Inject() (
         lastRefreshTime <- lastRefreshTimeFuture
       } yield {
         Logger.debug("indexPage: Got all ingredients")
+        /*val (global, classes) = ObjectGraphDuplicityMeasurer.measureUnique((vulnerableDependencies, allWarnings, groupedDependencies))
+        Logger.debug("(all,unique): "+global)
+        Logger.debug(classes.toIndexedSeq.sortBy(x => (x._2, x._1.getName)).mkString("\n"))
+        Logger.debug("footprint: "+ObjectGraphMeasurer.measure((vulnerableDependencies, allWarnings, groupedDependencies)))
+        //Logger.debug("footprint: "+ObjectGraphMeasurer.measure(Array((vulnerableDependencies, allWarnings, groupedDependencies))))*/
         Ok(views.html.index(
           vulnerableDependencies = vulnerableDependencies,
           warnings = allWarnings,

app/controllers/DependencyCheckReportsProcessor.scala

@@ -96,7 +96,7 @@ final class DependencyCheckReportsProcessor @Inject() (
 
       // TODO: log analysis
       // TODO: related dependencies
-      (vulnerableDependencies, allWarnings, groupedDependencies)
+      (vulnerableDependencies, allWarnings.map(_.optimize), groupedDependencies)
     }
   }finally{
     Logger.debug("Reports processed")

app/controllers/warnings.scala

@@ -12,10 +12,13 @@ object WarningSeverity extends Enumeration {
 }
 
 sealed abstract class Warning {
+  def optimize: Warning
   def html: Html
   def id: String
   def allowSnoozes = true
   def severity: WarningSeverity
 }
 
-final case class IdentifiedWarning(id: String, html: Html, severity: WarningSeverity) extends Warning
+final case class IdentifiedWarning(id: String, html: Html, severity: WarningSeverity) extends Warning{
+  def optimize = copy(html = Html(html.body))
+}

app/models/odc/Vulnerability.scala

@@ -19,7 +19,7 @@ class Vulnerabilities(tag: Tag) extends Table[(Int, Vulnerability)](tag, "vulner
   def cvssConfidentialityImpact = column[String]("cvssConfidentialityImpact").?
 
   def cvssRating = (cvssScore, authentication, availabilityImpact, accessVector, integrityImpact, cvssAccessComplexity, cvssConfidentialityImpact) <> (CvssRating.tupled, CvssRating.unapply)
-  def cweOptionMapped = cweOption <> ((_: Option[String]).map(CWE.apply), (_: Option[CWE]).map(CWE.unapply))
+  def cweOptionMapped = cweOption <> ((_: Option[String]).map(CWE.forIdentifierWithDescription), (_: Option[CWE]).map(CWE.unapply))
   def base = (cve, description, cweOptionMapped, cvssRating) <> (Vulnerability.tupled, Vulnerability.unapply)
 
   def * = (id, base)

build.sbt

@@ -77,6 +77,8 @@ libraryDependencies += "org.owasp" % "dependency-check-core" % "1.3.0"
 
 libraryDependencies += "com.typesafe.play" %% "play-mailer" % "3.0.1"
 
+libraryDependencies += "com.google.caliper" % "caliper" % "1.0-beta-2"
+
 routesImport += "binders.QueryBinders._"
 
 // Uncomment to use Akka