Improve testing with stats and errors per test section (#498)

* Emojis are moves to the left to be aligned
* A summary line is added with test counts
* Facts and Examples are grouped under their own section

pkl-gradle/src/test/kotlin/org/pkl/gradle/TestsTest.kt

@@ -30,7 +30,7 @@ class TestsTest : AbstractTest() {
     writePklFile()
 
     val res = runTask("evalTest")
-    assertThat(res.output).contains("should pass ✅")
+    assertThat(res.output).contains("✅ should pass")
   }
 
   @Test
@@ -49,9 +49,9 @@ class TestsTest : AbstractTest() {
     )
 
     val res = runTask("evalTest", expectFailure = true)
-    assertThat(res.output).contains("should fail ❌")
-    assertThat(res.output).contains("1 == 3 ❌")
-    assertThat(res.output).contains(""""foo" == "bar" ❌""")
+    assertThat(res.output).contains("❌ should fail")
+    assertThat(res.output).contains("1 == 3")
+    assertThat(res.output).contains(""""foo" == "bar"""")
   }
 
   @Test
@@ -68,22 +68,28 @@ class TestsTest : AbstractTest() {
           .trimIndent()
     )
 
-    val output = runTask("evalTest", expectFailure = true).output.stripFilesAndLines()
+    val output =
+      runTask("evalTest", expectFailure = true)
+        .output
+        .stripFilesAndLines()
+        .lineSequence()
+        .joinToString("\n")
 
     assertThat(output)
       .containsIgnoringNewLines(
         """
       > Task :evalTest FAILED
-      module test (file:///file, line x)
-        should pass ✅
-        error ❌
-          Error:
-              –– Pkl Error ––
-              exception
-              
-              9 | throw("exception")
-                  ^^^^^^^^^^^^^^^^^^
-              at test#facts["error"][#1] (file:///file, line x)
+      module test
+        facts
+          ✅ should pass
+          ❌ error
+             –– Pkl Error ––
+             exception
+
+             9 | throw("exception")
+                 ^^^^^^^^^^^^^^^^^^
+             at test#facts["error"][#1] (file:///file, line x)
+      ❌ 50.0% tests pass [1/2 failed], 66.7% asserts pass [1/3 failed]
       """
           .trimIndent()
       )
@@ -96,41 +102,41 @@ class TestsTest : AbstractTest() {
 
     writeBuildFile()
 
-    val output = runTask("evalTest", expectFailure = true).output.stripFilesAndLines()
+    val output =
+      runTask("evalTest", expectFailure = true)
+        .output
+        .stripFilesAndLines()
+        .lineSequence()
+        .joinToString("\n")
 
     assertThat(output.trimStart())
-      .contains(
+      .startsWith(
         """
-      module test (file:///file, line x)
-        sum numbers ✅
-        divide numbers ✅
-        fail ❌
-          4 == 9 ❌ (file:///file, line x)
-          "foo" == "bar" ❌ (file:///file, line x)
-        user 0 ✅
-        user 1 ❌
-          (file:///file, line x)
-          Expected: (file:///file, line x)
-          new {
-            name = "Pigeon"
-            age = 40
-          }
-          Actual: (file:///file, line x)
-          new {
-            name = "Pigeon"
-            age = 41
-          }
-          (file:///file, line x)
-          Expected: (file:///file, line x)
-          new {
-            name = "Parrot"
-            age = 35
-          }
-          Actual: (file:///file, line x)
-          new {
-            name = "Welma"
-            age = 35
-          }
+      > Task :evalTest FAILED
+      pkl: TRACE: 8 = 8 (file:///file, line x)
+      module test
+        facts
+          ✅ sum numbers
+          ✅ divide numbers
+          ❌ fail
+             4 == 9 (file:///file, line x)
+             "foo" == "bar" (file:///file, line x)
+        examples
+          ✅ user 0
+          ✅ user 1 #0
+          ❌ user 1 #1
+             (file:///file, line x)
+             Expected: (file:///file, line x)
+             new {
+               name = "Parrot"
+               age = 35
+             }
+             Actual: (file:///file, line x)
+             new {
+               name = "Welma"
+               age = 35
+             }
+      ❌ 66.7% tests pass [2/6 failed], 66.7% asserts pass [3/9 failed]
     """
           .trimIndent()
       )
@@ -138,28 +144,7 @@ class TestsTest : AbstractTest() {
 
   @Test
   fun `overwrite expected examples`() {
-    writePklFile(
-      additionalExamples =
-        """
-      ["user 0"] {
-        new {
-          name = "Cool"
-          age = 11
-        }
-      }
-      ["user 1"] {
-        new {
-          name = "Pigeon"
-          age = 41
-        }
-        new {
-          name = "Welma"
-          age = 35
-        }
-      }
-    """
-          .trimIndent()
-    )
+    writePklFile(additionalExamples = examples)
     writeFile("test.pkl-expected.pcf", bigTestExpected)
 
     writeBuildFile("overwrite = true")
@@ -170,6 +155,64 @@ class TestsTest : AbstractTest() {
     assertThat(output).contains("user 1 ✍️")
   }
 
+  @Test
+  fun `full example with error`() {
+    writeBuildFile()
+
+    writePklFile(
+      additionalFacts =
+        """
+      ["error"] {
+        throw("exception")
+      }
+    """
+          .trimIndent(),
+      additionalExamples = examples
+    )
+    writeFile("test.pkl-expected.pcf", bigTestExpected)
+
+    val output =
+      runTask("evalTest", expectFailure = true)
+        .output
+        .stripFilesAndLines()
+        .lineSequence()
+        .joinToString("\n")
+
+    assertThat(output.trimStart())
+      .startsWith(
+        """
+      > Task :evalTest FAILED
+      module test
+        facts
+          ✅ should pass
+          ❌ error
+             –– Pkl Error ––
+             exception
+
+             9 | throw("exception")
+                 ^^^^^^^^^^^^^^^^^^
+             at test#facts["error"][#1] (file:///file, line x)
+        examples
+          ✅ user 0
+          ✅ user 1 #0
+          ❌ user 1 #1
+             (file:///file, line x)
+             Expected: (file:///file, line x)
+             new {
+               name = "Parrot"
+               age = 35
+             }
+             Actual: (file:///file, line x)
+             new {
+               name = "Welma"
+               age = 35
+             }
+      ❌ 60.0% tests pass [2/5 failed], 66.7% asserts pass [2/6 failed]
+    """
+          .trimIndent()
+      )
+  }
+
   @Test
   fun `JUnit reports`() {
     val pklFile = writePklFile(contents = bigTest)
@@ -186,26 +229,16 @@ class TestsTest : AbstractTest() {
       .isEqualTo(
         """
       <?xml version="1.0" encoding="UTF-8"?>
-      <testsuite name="test" tests="5" failures="4">
-          <testcase classname="test" name="sum numbers"></testcase>
-          <testcase classname="test" name="divide numbers"></testcase>
-          <testcase classname="test" name="fail">
-              <failure message="Fact Failure">4 == 9 ❌ (file:///file, line x)</failure>
-              <failure message="Fact Failure">&quot;foo&quot; == &quot;bar&quot; ❌ (file:///file, line x)</failure>
+      <testsuite name="test" tests="6" failures="2">
+          <testcase classname="test.facts" name="sum numbers"></testcase>
+          <testcase classname="test.facts" name="divide numbers"></testcase>
+          <testcase classname="test.facts" name="fail">
+              <failure message="Fact Failure">4 == 9 (file:///file, line x)</failure>
+              <failure message="Fact Failure">&quot;foo&quot; == &quot;bar&quot; (file:///file, line x)</failure>
           </testcase>
-          <testcase classname="test" name="user 0"></testcase>
-          <testcase classname="test" name="user 1">
-              <failure message="Example Failure">(file:///file, line x)
-      Expected: (file:///file, line x)
-      new {
-        name = &quot;Pigeon&quot;
-        age = 40
-      }
-      Actual: (file:///file, line x)
-      new {
-        name = &quot;Pigeon&quot;
-        age = 41
-      }</failure>
+          <testcase classname="test.examples" name="user 0"></testcase>
+          <testcase classname="test.examples" name="user 1 #0"></testcase>
+          <testcase classname="test.examples" name="user 1 #1">
               <failure message="Example Failure">(file:///file, line x)
       Expected: (file:///file, line x)
       new {
@@ -227,6 +260,86 @@ class TestsTest : AbstractTest() {
       )
   }
 
+  @Test
+  fun `JUnit reports with error`() {
+    val pklFile =
+      writePklFile(
+        additionalFacts =
+          """
+      ["error"] {
+        throw("exception")
+      }
+    """
+            .trimIndent(),
+        additionalExamples = examples
+      )
+    writeFile("test.pkl-expected.pcf", bigTestExpected)
+
+    writeBuildFile("junitReportsDir = file('${pklFile.parent.toNormalizedPathString()}/build')")
+
+    runTask("evalTest", expectFailure = true)
+
+    val outputFile = testProjectDir.resolve("build/test.xml")
+    val report = outputFile.readText().stripFilesAndLines()
+
+    assertThat(report)
+      .isEqualTo(
+        """
+      <?xml version="1.0" encoding="UTF-8"?>
+      <testsuite name="test" tests="5" failures="2">
+          <testcase classname="test.facts" name="should pass"></testcase>
+          <testcase classname="test.facts" name="error">
+              <error message="exception">–– Pkl Error ––
+      exception
+
+      9 | throw(&quot;exception&quot;)
+          ^^^^^^^^^^^^^^^^^^
+      at test#facts[&quot;error&quot;][#1] (file:///file, line x)
+      </error>
+          </testcase>
+          <testcase classname="test.examples" name="user 0"></testcase>
+          <testcase classname="test.examples" name="user 1 #0"></testcase>
+          <testcase classname="test.examples" name="user 1 #1">
+              <failure message="Example Failure">(file:///file, line x)
+      Expected: (file:///file, line x)
+      new {
+        name = &quot;Parrot&quot;
+        age = 35
+      }
+      Actual: (file:///file, line x)
+      new {
+        name = &quot;Welma&quot;
+        age = 35
+      }</failure>
+          </testcase>
+      </testsuite>
+
+    """
+          .trimIndent()
+      )
+  }
+
+  private val examples =
+    """
+      ["user 0"] {
+        new {
+          name = "Cool"
+          age = 11
+        }
+      }
+      ["user 1"] {
+        new {
+          name = "Pigeon"
+          age = 40
+        }
+        new {
+          name = "Welma"
+          age = 35
+        }
+      }
+  """
+      .trimIndent()
+
   private val bigTest =
     """
     amends "pkl:test"
@@ -249,22 +362,7 @@ class TestsTest : AbstractTest() {
     }
     
     examples {
-      ["user 0"] {
-        new {
-          name = "Cool"
-          age = 11
-        }
-      }
-      ["user 1"] {
-        new {
-          name = "Pigeon"
-          age = 41
-        }
-        new {
-          name = "Welma"
-          age = 35
-        }
-      }
+      $examples
     }
   """
       .trimIndent()