Polish test result running and reporting (#738)

Changes: * Move class `TestResults` to package `org.pkl.core`, because it is a public class (it's the result of `Evaluator#evaluateTest`) * Change examples to treat individual examples as assertions in the same test. Previously, they were considered different tests with an incrementing number. This better aligns with how facts are treated. * Change `TestResults` to be a record, and introduce builders. * Remove "module" test result section (it is not really a section). * Add javadoc to `TestResults` * Formatting fix: prefix ✍️ emoji just like we do the ❌ and ✅ emojis * Consider writing examples as failures, not successes. `pkl test` will exit with code 10 if the only failing tests are due to writing examples.
2026-03-31 06:03:11 +02:00 · 2024-10-28 21:05:13 -07:00
parent 666f8c3939
commit acd2222534
15 changed files with 851 additions and 708 deletions
--- a/pkl-cli/src/test/kotlin/org/pkl/cli/CliProjectPackagerTest.kt
+++ b/pkl-cli/src/test/kotlin/org/pkl/cli/CliProjectPackagerTest.kt
@@ -212,6 +212,24 @@ class CliProjectPackagerTest {
      """
          .trimIndent()
      )
+    projectDir
+      .resolve("myTest.pkl-expected.pcf")
+      .writeString(
+        """
+        examples {
+          ["Bird"] {
+            new {
+              name = "Finch"
+              favoriteFruit {
+                name = "Tangerine"
+              }
+            }
+          }
+        }
+        
+      """
+          .trimIndent()
+      )
    projectDir
      .resolve("PklProject")
      .writeString(
--- a/pkl-cli/src/test/kotlin/org/pkl/cli/CliTestRunnerTest.kt
+++ b/pkl-cli/src/test/kotlin/org/pkl/cli/CliTestRunnerTest.kt
@@ -18,6 +18,7 @@ package org.pkl.cli
 import com.github.ajalt.clikt.core.MissingArgument
 import com.github.ajalt.clikt.core.subcommands
 import java.io.StringWriter
+import java.io.Writer
 import java.net.URI
 import java.nio.file.Path
 import org.assertj.core.api.Assertions.assertThat
@@ -98,13 +99,13 @@ class CliTestRunnerTest {
    assertThat(out.toString().stripFileAndLines(tempDir))
      .isEqualTo(
        """
-      module test
-        facts
-          ❌ fail
-             4 == 9
-      ❌ 0.0% tests pass [1/1 failed], 50.0% asserts pass [1/2 failed]
+        module test
+          facts
+            ❌ fail
+               4 == 9 (/tempDir/test.pkl, line xx)
+        ❌ 0.0% tests pass [1/1 failed], 50.0% asserts pass [1/2 failed]

-    """
+        """
          .trimIndent()
      )
    assertThat(err.toString()).isEqualTo("")
@@ -142,7 +143,7 @@ class CliTestRunnerTest {

             5 | throw("uh oh")
                 ^^^^^^^^^^^^^^
-             at test#facts["fail"][#1]
+             at test#facts["fail"][#1] (/tempDir/test.pkl, line xx)
      ❌ 0.0% tests pass [1/1 failed], 0.0% asserts pass [1/1 failed]

    """
@@ -183,7 +184,7 @@ class CliTestRunnerTest {

             5 | throw("uh oh")
                 ^^^^^^^^^^^^^^
-             at test#examples["fail"][#1]
+             at test#examples["fail"][#1] (/tempDir/test.pkl, line xx)
      ❌ 0.0% tests pass [1/1 failed], 0.0% asserts pass [1/1 failed]

    """
@@ -235,10 +236,10 @@ class CliTestRunnerTest {
          ❌ fail
             –– Pkl Error ––
             uh oh
-
+      
             5 | throw("uh oh")
                 ^^^^^^^^^^^^^^
-             at test#examples["fail"][#1]
+             at test#examples["fail"][#1] (/tempDir/test.pkl, line xx)
      ❌ 0.0% tests pass [1/1 failed], 0.0% asserts pass [1/1 failed]

    """
@@ -266,9 +267,10 @@ class CliTestRunnerTest {
    """
        .trimIndent()
    val input = tempDir.resolve("test.pkl").writeString(code).toString()
+    val noopWriter = noopWriter()
    val opts = CliBaseOptions(sourceModules = listOf(input.toUri()), settings = URI("pkl:settings"))
    val testOpts = CliTestOptions(junitDir = tempDir)
-    val runner = CliTestRunner(opts, testOpts)
+    val runner = CliTestRunner(opts, testOpts, noopWriter, noopWriter)
    assertThatCode { runner.run() }.hasMessageContaining("failed")

    val junitReport = tempDir.resolve("test.xml").readString().stripFileAndLines(tempDir)
@@ -279,7 +281,7 @@ class CliTestRunnerTest {
      <testsuite name="test" tests="2" failures="1">
          <testcase classname="test.facts" name="foo"></testcase>
          <testcase classname="test.facts" name="bar">
-              <failure message="Fact Failure">5 == 9</failure>
+              <failure message="Fact Failure">5 == 9 (/tempDir/test.pkl, line xx)</failure>
          </testcase>
          <system-err><![CDATA[9 = 9
      ]]></system-err>
@@ -308,9 +310,10 @@ class CliTestRunnerTest {
    """
        .trimIndent()
    val input = tempDir.resolve("test.pkl").writeString(code).toString()
+    val noopWriter = noopWriter()
    val opts = CliBaseOptions(sourceModules = listOf(input.toUri()), settings = URI("pkl:settings"))
    val testOpts = CliTestOptions(junitDir = tempDir)
-    val runner = CliTestRunner(opts, testOpts)
+    val runner = CliTestRunner(opts, testOpts, noopWriter, noopWriter)
    assertThatCode { runner.run() }.hasMessageContaining("failed")

    val junitReport = tempDir.resolve("test.xml").readString().stripFileAndLines(tempDir)
@@ -326,7 +329,7 @@ class CliTestRunnerTest {
      
      9 | throw(&quot;uh oh&quot;)
          ^^^^^^^^^^^^^^
-      at test#facts[&quot;fail&quot;][#1]
+      at test#facts[&quot;fail&quot;][#1] (/tempDir/test.pkl, line xx)
      </error>
          </testcase>
          <system-err><![CDATA[9 = 9
@@ -369,13 +372,14 @@ class CliTestRunnerTest {
        .trimIndent()
    val input = tempDir.resolve("test.pkl").writeString(foo).toString()
    val input2 = tempDir.resolve("test.pkl").writeString(bar).toString()
+    val noopWriter = noopWriter()
    val opts =
      CliBaseOptions(
        sourceModules = listOf(input.toUri(), input2.toUri()),
        settings = URI("pkl:settings")
      )
    val testOpts = CliTestOptions(junitDir = tempDir)
-    val runner = CliTestRunner(opts, testOpts)
+    val runner = CliTestRunner(opts, testOpts, noopWriter, noopWriter)
    assertThatCode { runner.run() }.hasMessageContaining("failed")
  }

@@ -392,6 +396,105 @@ class CliTestRunnerTest {
    assertThat(e1.message!!.replace("test", "eval")).isEqualTo(e2.helpMessage())
  }

-  private fun String.stripFileAndLines(tmpDir: Path) =
-    replace(tmpDir.toUri().toString(), "/tempDir/").replace(Regex(""" \(.*, line \d+\)"""), "")
+  @Test
+  fun `example length mismatch`(@TempDir tempDir: Path) {
+    val code =
+      """
+      amends "pkl:test"
+
+      examples {
+        ["nums"] {
+          1
+          2
+        }
+      }
+    """
+        .trimIndent()
+    val input = tempDir.resolve("test.pkl").writeString(code).toString()
+    tempDir
+      .resolve("test.pkl-expected.pcf")
+      .writeString(
+        """
+      examples {
+        ["nums"] {
+          1
+        }
+      }
+    """
+          .trimIndent()
+      )
+    val out = StringWriter()
+    val err = StringWriter()
+    val opts = CliBaseOptions(sourceModules = listOf(input.toUri()), settings = URI("pkl:settings"))
+    val testOpts = CliTestOptions()
+    val runner = CliTestRunner(opts, testOpts, consoleWriter = out, errWriter = err)
+    assertThatCode { runner.run() }.hasMessage("Tests failed.")
+
+    assertThat(out.toString().stripFileAndLines(tempDir))
+      .isEqualToNormalizingNewlines(
+        """
+        module test
+          examples
+            ❌ nums
+               (/tempDir/test.pkl, line xx)
+               Output mismatch: Expected "nums" to contain 1 examples, but found 2
+        ❌ 0.0% tests pass [1/1 failed], 0.0% asserts pass [1/1 failed]
+
+        """
+          .trimIndent()
+      )
+  }
+
+  @Test
+  fun `only written examples`(@TempDir tempDir: Path) {
+
+    val code =
+      """
+      amends "pkl:test"
+
+      examples {
+        ["nums"] {
+          1
+          2
+        }
+      }
+    """
+        .trimIndent()
+    val input = tempDir.resolve("test.pkl").writeString(code).toString()
+    val out = StringWriter()
+    val err = StringWriter()
+    val opts = CliBaseOptions(sourceModules = listOf(input.toUri()), settings = URI("pkl:settings"))
+    val testOpts = CliTestOptions()
+    val runner = CliTestRunner(opts, testOpts, consoleWriter = out, errWriter = err)
+    val exception = assertThrows<CliException> { runner.run() }
+    assertThat(exception.exitCode).isEqualTo(10)
+    assertThat(out.toString())
+      .isEqualTo(
+        """
+          module test
+            examples
+              ✍️ nums
+          1 examples written
+          
+          """
+          .trimIndent()
+      )
+  }
+
+  private fun String.stripFileAndLines(tmpDir: Path): String {
+    // handle platform differences in handling of file URIs
+    // (file:/// on *nix vs. file:/ on Windows)
+    return replace(tmpDir.toFile().toURI().toString(), "/tempDir/")
+      .replace(tmpDir.toUri().toString(), "/tempDir/")
+      .replace(Regex("line \\d+"), "line xx")
+  }
+
+  private fun noopWriter(): Writer =
+    object : Writer() {
+      override fun close() {}
+
+      override fun flush() {}
+
+      override fun write(cbuf: CharArray, off: Int, len: Int) {}
+    }
 }