From b4e7bd251e969bdba7e79c3ec750ca1d5f7e9886 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C5=A0est=C3=A1k=20V=C3=ADt?= <Vit.Sestak@ysoft.com>
Date: Mon, 5 Dec 2016 09:29:49 +0100
Subject: [PATCH] Added support for finding log smells using custom regular
 expressions.

---
 .../DependencyCheckReportsProcessor.scala         | 15 +++++++++++----
 app/modules/ConfigModule.scala                    | 12 ++++++++++--
 conf/application.conf.-example                    |  8 ++++++++
 3 files changed, 29 insertions(+), 6 deletions(-)

diff --git a/app/controllers/DependencyCheckReportsProcessor.scala b/app/controllers/DependencyCheckReportsProcessor.scala
index 216efa0..6799607 100644
--- a/app/controllers/DependencyCheckReportsProcessor.scala
+++ b/app/controllers/DependencyCheckReportsProcessor.scala
@@ -6,11 +6,12 @@ import com.google.inject.name.Named
 import com.ysoft.odc.Checks._
 import com.ysoft.odc._
 import com.ysoft.odc.statistics.FailedProjects
+import modules.{LogSmell, LogSmellChecks}
 import org.joda.time.DateTimeConstants
 import play.api.Logger
 import play.api.i18n.{I18nSupport, MessagesApi}
 import play.api.mvc.RequestHeader
-import play.twirl.api.Html
+import play.twirl.api.{Html, HtmlFormat}
 import views.html.DefaultRequest
 
 import scala.concurrent.{ExecutionContext, Future}
@@ -23,6 +24,7 @@ final class DependencyCheckReportsProcessor @Inject() (
                                                   @Named("bamboo-server-url") val server: String,
                                                   dependencyCheckReportsParser: DependencyCheckReportsParser,
                                                   @Named("missing-GAV-exclusions") missingGAVExclusions: MissingGavExclusions,
+                                                  @Named("log-smells") logSmells: LogSmellChecks,
                                                   val messagesApi: MessagesApi
                                                   ) extends I18nSupport {
 
@@ -80,10 +82,15 @@ final class DependencyCheckReportsProcessor @Inject() (
           log => log.lines.exists(l => (l.toLowerCase startsWith "error") || (l.toLowerCase contains "[error]")),
           ProjectWarningBuilder("results-with-error-messages", views.html.warnings.resultsWithErrorMessages(), WarningSeverity.Error)
         )
-      )
+      ) ++ logSmells.checks.toSeq.map { case (id, s) =>
+        (
+          (log: String) => log.lines.exists(l => s.regex.pattern.matcher(l).find),
+          ProjectWarningBuilder(id, HtmlFormat.escape(s.message), s.severity)
+        )
+      }
       val logWarnings: Seq[Warning] = logChecks.flatMap{case (logCheck, warningBuilder) =>
-        val resultsWithErrorMessages = successfulResults.filter{case (k, (_, _, log)) => logCheck(log.dataString)}
-        if(resultsWithErrorMessages.nonEmpty) Some(warningBuilder.forProjects(new FailedProjects(resultsWithErrorMessages.keys.map(projectsReportInfo.reportIdToReportInfo).toSet), buildLink)) else None
+        val resultsWithErrorMessages = successfulResults.par.filter{case (k, (_, _, log)) => logCheck(log.dataString)}
+        if(resultsWithErrorMessages.nonEmpty) Some(warningBuilder.forProjects(new FailedProjects(resultsWithErrorMessages.keys.map(projectsReportInfo.reportIdToReportInfo).seq.toSet), buildLink)) else None
       }
       val extraWarnings = Seq[Option[Warning]](
         if(unknownIdentifierTypes.size > 0) Some(IdentifiedWarning("unknown-identifier-types", views.html.warnings.unknownIdentifierType(unknownIdentifierTypes), WarningSeverity.Info)) else None,
diff --git a/app/modules/ConfigModule.scala b/app/modules/ConfigModule.scala
index cf3eb09..359c119 100644
--- a/app/modules/ConfigModule.scala
+++ b/app/modules/ConfigModule.scala
@@ -7,7 +7,7 @@ import java.util.concurrent.Executors
 
 import akka.util.ClassLoaderObjectInputStream
 import com.ysoft.odc._
-import controllers.MissingGavExclusions
+import controllers.{MissingGavExclusions, WarningSeverity}
 import net.ceedubs.ficus.Ficus._
 import net.ceedubs.ficus.readers.ArbitraryTypeReader._
 import play.api.cache.CacheApi
@@ -99,7 +99,8 @@ class ConfigModule extends Module {
     bind[MissingGavExclusions].qualifiedWith("missing-GAV-exclusions").toInstance(MissingGavExclusions(
       configuration.getStringSeq("yssdc.exclusions.missingGAV.bySha1").getOrElse(Seq()).toSet.map(Exclusion))
     ),
-    bind[ExecutionContext].qualifiedWith("email-sending").toInstance(ExecutionContext.fromExecutor(Executors.newSingleThreadExecutor()))
+    bind[ExecutionContext].qualifiedWith("email-sending").toInstance(ExecutionContext.fromExecutor(Executors.newSingleThreadExecutor())),
+    bind[LogSmellChecks].qualifiedWith("log-smells").toInstance(LogSmellChecks(configuration.underlying.getAs[Map[String, LogSmell]]("yssdc.logSmells").getOrElse(Map())))
   ) ++
     configuration.underlying.getAs[Absolutizer]("app").map(a => bind[Absolutizer].toInstance(a)) ++
     configuration.getString("play.cache.path").map(cachePath => bind[CacheApi].toInstance(new FileCacheApi(Paths.get(cachePath)))) ++
@@ -108,3 +109,10 @@ class ConfigModule extends Module {
     configuration.getString("yssdc.reports.path").map{s => bind[String].qualifiedWith("reports-path").toInstance(s)}
 
 }
+
+case class LogSmellChecks(checks: Map[String, LogSmell])
+
+case class LogSmell(pattern: String, message: String){
+  val regex = pattern.r
+  def severity = WarningSeverity.Warning
+}
diff --git a/conf/application.conf.-example b/conf/application.conf.-example
index 487b400..a08f6dc 100644
--- a/conf/application.conf.-example
+++ b/conf/application.conf.-example
@@ -68,6 +68,14 @@ yssdc{
       //optional: type = "digest" or type="vulnerabilities" (default); Digest is WIP.
     }
   }
+  logSmells {
+    // An example of analysis of logs. This one if for Maven and requires -X to be used for Maven scans.
+    //centralLookup {
+    //  patternType = "regex"
+    //  pattern = "Searching Central url http(s?)://search\\.maven\\.org"
+    //  message = "Maven Central lookup used. Enable usage of local repository, please."
+    //}
+  }
   projects = {jobId:humanReadableName, …}
   teams = […]
   exclusions{