Polish test result running and reporting (#738)

Changes:
* Move class `TestResults` to package `org.pkl.core`, because it is a public class (it's the result of `Evaluator#evaluateTest`)
* Change examples to treat individual examples as assertions in the same test. Previously, they were considered different tests with an incrementing number. This better aligns with how facts are treated.
* Change `TestResults` to be a record, and introduce builders.
* Remove "module" test result section (it is not really a section).
* Add javadoc to `TestResults`
* Formatting fix: prefix ✍️ emoji just like we do the ❌ and ✅ emojis 
* Consider writing examples as failures, not successes. `pkl test` will exit with code 10 if the only failing tests are due to writing examples.

docs/modules/pkl-cli/pages/index.adoc

@@ -409,6 +409,10 @@ Evaluate the given `<modules>` as _tests_, producing a test report and appropria
 
 Renderers defined in test files will be ignored by the `test` command.
 
+Tests that result in writing `pkl-expected.pcf` files are considered failing tests.
+If these are the only failures, the command exits with exit code 10.
+Otherwise, failures result in exit code 1.
+
 <modules>::
 The absolute or relative URIs of the modules to test. Relative URIs are resolved against the working directory.
 

pkl-cli/src/main/kotlin/org/pkl/cli/CliTestRunner.kt

@@ -60,12 +60,14 @@ constructor(
     val evaluator = builder.build()
     evaluator.use {
       var failed = false
+      var isExampleWrittenFailure = true
       val moduleNames = mutableSetOf<String>()
       for ((idx, moduleUri) in sources.withIndex()) {
         try {
           val results = evaluator.evaluateTest(uri(moduleUri), testOptions.overwrite)
           if (!failed) {
             failed = results.failed()
+            isExampleWrittenFailure = results.isExampleWrittenFailure.and(isExampleWrittenFailure)
           }
           SimpleReport().report(results, consoleWriter)
           if (sources.size > 1 && idx != sources.size - 1) {
@@ -101,7 +103,8 @@ constructor(
         }
       }
       if (failed) {
-        throw CliTestException(ErrorMessages.create("testsFailed"))
+        val exitCode = if (isExampleWrittenFailure) 10 else 1
+        throw CliTestException(ErrorMessages.create("testsFailed"), exitCode)
       }
     }
   }

pkl-cli/src/test/kotlin/org/pkl/cli/CliProjectPackagerTest.kt

@@ -212,6 +212,24 @@ class CliProjectPackagerTest {
       """
           .trimIndent()
       )
+    projectDir
+      .resolve("myTest.pkl-expected.pcf")
+      .writeString(
+        """
+        examples {
+          ["Bird"] {
+            new {
+              name = "Finch"
+              favoriteFruit {
+                name = "Tangerine"
+              }
+            }
+          }
+        }
+        
+      """
+          .trimIndent()
+      )
     projectDir
       .resolve("PklProject")
       .writeString(

pkl-cli/src/test/kotlin/org/pkl/cli/CliTestRunnerTest.kt

@@ -18,6 +18,7 @@ package org.pkl.cli
 import com.github.ajalt.clikt.core.MissingArgument
 import com.github.ajalt.clikt.core.subcommands
 import java.io.StringWriter
+import java.io.Writer
 import java.net.URI
 import java.nio.file.Path
 import org.assertj.core.api.Assertions.assertThat
@@ -98,13 +99,13 @@ class CliTestRunnerTest {
     assertThat(out.toString().stripFileAndLines(tempDir))
       .isEqualTo(
         """
-      module test
+        module test
-        facts
+          facts
-          ❌ fail
+            ❌ fail
-             4 == 9
+               4 == 9 (/tempDir/test.pkl, line xx)
-      ❌ 0.0% tests pass [1/1 failed], 50.0% asserts pass [1/2 failed]
+        ❌ 0.0% tests pass [1/1 failed], 50.0% asserts pass [1/2 failed]
 
-    """
+        """
           .trimIndent()
       )
     assertThat(err.toString()).isEqualTo("")
@@ -142,7 +143,7 @@ class CliTestRunnerTest {
 
              5 | throw("uh oh")
                  ^^^^^^^^^^^^^^
-             at test#facts["fail"][#1]
+             at test#facts["fail"][#1] (/tempDir/test.pkl, line xx)
       ❌ 0.0% tests pass [1/1 failed], 0.0% asserts pass [1/1 failed]
 
     """
@@ -183,7 +184,7 @@ class CliTestRunnerTest {
 
              5 | throw("uh oh")
                  ^^^^^^^^^^^^^^
-             at test#examples["fail"][#1]
+             at test#examples["fail"][#1] (/tempDir/test.pkl, line xx)
       ❌ 0.0% tests pass [1/1 failed], 0.0% asserts pass [1/1 failed]
 
     """
@@ -238,7 +239,7 @@ class CliTestRunnerTest {
       
              5 | throw("uh oh")
                  ^^^^^^^^^^^^^^
-             at test#examples["fail"][#1]
+             at test#examples["fail"][#1] (/tempDir/test.pkl, line xx)
       ❌ 0.0% tests pass [1/1 failed], 0.0% asserts pass [1/1 failed]
 
     """
@@ -266,9 +267,10 @@ class CliTestRunnerTest {
     """
         .trimIndent()
     val input = tempDir.resolve("test.pkl").writeString(code).toString()
+    val noopWriter = noopWriter()
     val opts = CliBaseOptions(sourceModules = listOf(input.toUri()), settings = URI("pkl:settings"))
     val testOpts = CliTestOptions(junitDir = tempDir)
-    val runner = CliTestRunner(opts, testOpts)
+    val runner = CliTestRunner(opts, testOpts, noopWriter, noopWriter)
     assertThatCode { runner.run() }.hasMessageContaining("failed")
 
     val junitReport = tempDir.resolve("test.xml").readString().stripFileAndLines(tempDir)
@@ -279,7 +281,7 @@ class CliTestRunnerTest {
       <testsuite name="test" tests="2" failures="1">
           <testcase classname="test.facts" name="foo"></testcase>
           <testcase classname="test.facts" name="bar">
-              <failure message="Fact Failure">5 == 9</failure>
+              <failure message="Fact Failure">5 == 9 (/tempDir/test.pkl, line xx)</failure>
           </testcase>
           <system-err><![CDATA[9 = 9
       ]]></system-err>
@@ -308,9 +310,10 @@ class CliTestRunnerTest {
     """
         .trimIndent()
     val input = tempDir.resolve("test.pkl").writeString(code).toString()
+    val noopWriter = noopWriter()
     val opts = CliBaseOptions(sourceModules = listOf(input.toUri()), settings = URI("pkl:settings"))
     val testOpts = CliTestOptions(junitDir = tempDir)
-    val runner = CliTestRunner(opts, testOpts)
+    val runner = CliTestRunner(opts, testOpts, noopWriter, noopWriter)
     assertThatCode { runner.run() }.hasMessageContaining("failed")
 
     val junitReport = tempDir.resolve("test.xml").readString().stripFileAndLines(tempDir)
@@ -326,7 +329,7 @@ class CliTestRunnerTest {
       
       9 | throw(&quot;uh oh&quot;)
           ^^^^^^^^^^^^^^
-      at test#facts[&quot;fail&quot;][#1]
+      at test#facts[&quot;fail&quot;][#1] (/tempDir/test.pkl, line xx)
       </error>
           </testcase>
           <system-err><![CDATA[9 = 9
@@ -369,13 +372,14 @@ class CliTestRunnerTest {
         .trimIndent()
     val input = tempDir.resolve("test.pkl").writeString(foo).toString()
     val input2 = tempDir.resolve("test.pkl").writeString(bar).toString()
+    val noopWriter = noopWriter()
     val opts =
       CliBaseOptions(
         sourceModules = listOf(input.toUri(), input2.toUri()),
         settings = URI("pkl:settings")
       )
     val testOpts = CliTestOptions(junitDir = tempDir)
-    val runner = CliTestRunner(opts, testOpts)
+    val runner = CliTestRunner(opts, testOpts, noopWriter, noopWriter)
     assertThatCode { runner.run() }.hasMessageContaining("failed")
   }
 
@@ -392,6 +396,105 @@ class CliTestRunnerTest {
     assertThat(e1.message!!.replace("test", "eval")).isEqualTo(e2.helpMessage())
   }
 
-  private fun String.stripFileAndLines(tmpDir: Path) =
+  @Test
-    replace(tmpDir.toUri().toString(), "/tempDir/").replace(Regex(""" \(.*, line \d+\)"""), "")
+  fun `example length mismatch`(@TempDir tempDir: Path) {
+    val code =
+      """
+      amends "pkl:test"
+
+      examples {
+        ["nums"] {
+          1
+          2
+        }
+      }
+    """
+        .trimIndent()
+    val input = tempDir.resolve("test.pkl").writeString(code).toString()
+    tempDir
+      .resolve("test.pkl-expected.pcf")
+      .writeString(
+        """
+      examples {
+        ["nums"] {
+          1
+        }
+      }
+    """
+          .trimIndent()
+      )
+    val out = StringWriter()
+    val err = StringWriter()
+    val opts = CliBaseOptions(sourceModules = listOf(input.toUri()), settings = URI("pkl:settings"))
+    val testOpts = CliTestOptions()
+    val runner = CliTestRunner(opts, testOpts, consoleWriter = out, errWriter = err)
+    assertThatCode { runner.run() }.hasMessage("Tests failed.")
+
+    assertThat(out.toString().stripFileAndLines(tempDir))
+      .isEqualToNormalizingNewlines(
+        """
+        module test
+          examples
+            ❌ nums
+               (/tempDir/test.pkl, line xx)
+               Output mismatch: Expected "nums" to contain 1 examples, but found 2
+        ❌ 0.0% tests pass [1/1 failed], 0.0% asserts pass [1/1 failed]
+
+        """
+          .trimIndent()
+      )
+  }
+
+  @Test
+  fun `only written examples`(@TempDir tempDir: Path) {
+
+    val code =
+      """
+      amends "pkl:test"
+
+      examples {
+        ["nums"] {
+          1
+          2
+        }
+      }
+    """
+        .trimIndent()
+    val input = tempDir.resolve("test.pkl").writeString(code).toString()
+    val out = StringWriter()
+    val err = StringWriter()
+    val opts = CliBaseOptions(sourceModules = listOf(input.toUri()), settings = URI("pkl:settings"))
+    val testOpts = CliTestOptions()
+    val runner = CliTestRunner(opts, testOpts, consoleWriter = out, errWriter = err)
+    val exception = assertThrows<CliException> { runner.run() }
+    assertThat(exception.exitCode).isEqualTo(10)
+    assertThat(out.toString())
+      .isEqualTo(
+        """
+          module test
+            examples
+              ✍️ nums
+          1 examples written
+          
+          """
+          .trimIndent()
+      )
+  }
+
+  private fun String.stripFileAndLines(tmpDir: Path): String {
+    // handle platform differences in handling of file URIs
+    // (file:/// on *nix vs. file:/ on Windows)
+    return replace(tmpDir.toFile().toURI().toString(), "/tempDir/")
+      .replace(tmpDir.toUri().toString(), "/tempDir/")
+      .replace(Regex("line \\d+"), "line xx")
+  }
+
+  private fun noopWriter(): Writer =
+    object : Writer() {
+      override fun close() {}
+
+      override fun flush() {}
+
+      override fun write(cbuf: CharArray, off: Int, len: Int) {}
+    }
 }

pkl-commons-cli/src/main/kotlin/org/pkl/commons/cli/CliTestException.kt

@@ -15,4 +15,4 @@
  */
 package org.pkl.commons.cli
 
-class CliTestException(msg: String) : CliException(msg, 1)
+class CliTestException(msg: String, exitCode: Int) : CliException(msg, exitCode)

pkl-core/src/main/java/org/pkl/core/Evaluator.java

@@ -16,7 +16,6 @@
 package org.pkl.core;
 
 import java.util.Map;
-import org.pkl.core.runtime.TestResults;
 import org.pkl.core.runtime.VmEvalException;
 
 /**
@@ -194,9 +193,9 @@ public interface Evaluator extends AutoCloseable {
    * <p>This requires that the target module be a test module; it must either amend or extend module
    * {@code "pkl:test"}. Otherwise, a type mismatch error is thrown.
    *
-   * <p>This method will write possibly {@code pcf-expected.pkl} and {@code pcf-actual.pcf} files as
+   * <p>This method will write possibly {@code pkl-expected.pcf} and {@code pkl-actual.pcf} files as
    * a sibling of the test module. The {@code overwrite} parameter causes the evaluator to overwrite
-   * {@code pcf-expected.pkl} files if they currently exist.
+   * {@code pkl-expected.pkl} files if they currently exist.
    *
    * @throws PklException if an error occurs during evaluation
    * @throws IllegalStateException if this evaluator has already been closed

pkl-core/src/main/java/org/pkl/core/EvaluatorImpl.java

@@ -40,7 +40,6 @@ import org.pkl.core.runtime.BaseModule;
 import org.pkl.core.runtime.Identifier;
 import org.pkl.core.runtime.ModuleResolver;
 import org.pkl.core.runtime.ResourceManager;
-import org.pkl.core.runtime.TestResults;
 import org.pkl.core.runtime.TestRunner;
 import org.pkl.core.runtime.VmContext;
 import org.pkl.core.runtime.VmException;

pkl-core/src/main/java/org/pkl/core/TestResults.java

@@ -0,0 +1,301 @@
+/*
+ * Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.pkl.core;
+
+import java.util.ArrayList;
+import java.util.List;
+import org.pkl.core.util.Nullable;
+
+/**
+ * The results of testing a Pkl test module.
+ *
+ * <p>A test module is a module that amends {@code pkl:test}, and is evaluated with {@link
+ * Evaluator#evaluateTest(ModuleSource, boolean)}.
+ *
+ * <p>Test results have two sections; facts, and examples. Each section has multiple individual
+ * results, which themselves contain individual assertions.
+ *
+ * @since 0.27.0
+ * @param moduleName The name of the module that was tested.
+ * @param displayUri A location URI formatted to be displayed.
+ * @param facts The result of testing facts.
+ * @param examples The result of testing examples.
+ * @param logs The log output resulting from running the test.
+ * @param error An error that arose from evaluating the test module itself.
+ *     <p>If non-null, {@code facts} and {@code examples} are guaranteed to have 0 results.
+ */
+@SuppressWarnings("UnusedReturnValue")
+public record TestResults(
+    String moduleName,
+    String displayUri,
+    TestSectionResults facts,
+    TestSectionResults examples,
+    String logs,
+    @Nullable Error error) {
+
+  /** The total number of tests between facts and examples. */
+  public int totalTests() {
+    return facts.totalTests() + examples.totalTests();
+  }
+
+  /**
+   * The total number of failed {@linkplain TestResult test results}.
+   *
+   * <p>A {@link TestResult} has failed if any of its assertions have a {@link Failure} or an {@link
+   * Error}.
+   */
+  public int totalFailures() {
+    return facts.totalFailures() + examples.totalFailures();
+  }
+
+  /** The total number of assertions between facts and examples. */
+  public int totalAsserts() {
+    return facts.totalAsserts() + examples.totalAsserts();
+  }
+
+  /**
+   * The total number of individual assertions that have failed.
+   *
+   * <p>A single test can have multiple failed assertions.
+   */
+  public int totalAssertsFailed() {
+    return facts.totalAssertsFailedOrErrored() + examples.totalAssertsFailedOrErrored();
+  }
+
+  /**
+   * Whether the test in aggregate has failed or not.
+   *
+   * <p>An individual {@link TestResult} has failed if any of its assertions have a {@link Failure}
+   * or an {@link Error}.
+   */
+  public boolean failed() {
+    return error != null || facts.failed() || examples.failed();
+  }
+
+  /**
+   * Whether the test result has failed due to examples being written.
+   *
+   * <p>Returns {@code true} if and only if there are failures, and all failures are due to examples
+   * being written.
+   */
+  public boolean isExampleWrittenFailure() {
+    if (!failed() || !examples.failed()) return false;
+    for (var testResult : examples.results) {
+      if (!testResult.isExampleWritten) {
+        return false;
+      }
+    }
+    return true;
+  }
+
+  public static class Builder {
+
+    private final String moduleName;
+    private final String displayUri;
+    private TestSectionResults factsSection =
+        new TestSectionResults(TestSectionName.FACTS, List.of());
+    private TestSectionResults examplesSection =
+        new TestSectionResults(TestSectionName.EXAMPLES, List.of());
+    private String stdErr = "";
+    private @Nullable Error error = null;
+
+    public Builder(String moduleName, String displayUri) {
+      this.moduleName = moduleName;
+      this.displayUri = displayUri;
+    }
+
+    public Builder setError(Error error) {
+      this.error = error;
+      return this;
+    }
+
+    public Builder setStdErr(String stdErr) {
+      this.stdErr = stdErr;
+      return this;
+    }
+
+    public Builder setFactsSection(TestSectionResults factsSection) {
+      assert factsSection.name() == TestSectionName.FACTS;
+      this.factsSection = factsSection;
+      return this;
+    }
+
+    public Builder setExamplesSection(TestSectionResults examplesSection) {
+      assert examplesSection.name() == TestSectionName.EXAMPLES;
+      this.examplesSection = examplesSection;
+      return this;
+    }
+
+    public TestResults build() {
+      return new TestResults(moduleName, displayUri, factsSection, examplesSection, stdErr, error);
+    }
+  }
+
+  public enum TestSectionName {
+    FACTS("facts"),
+    EXAMPLES("examples");
+
+    private final String name;
+
+    TestSectionName(final String name) {
+      this.name = name;
+    }
+
+    @Override
+    public String toString() {
+      return name;
+    }
+  }
+
+  /**
+   * The results of either facts or examples.
+   *
+   * @param name The name of this section
+   * @param results The results of the individual tests in this section.
+   */
+  public record TestSectionResults(TestSectionName name, List<TestResult> results) {
+    public int totalTests() {
+      return results.size();
+    }
+
+    public int totalAsserts() {
+      var total = 0;
+      for (var res : results) {
+        total += res.totalAsserts();
+      }
+      return total;
+    }
+
+    public int totalAssertsFailedOrErrored() {
+      var total = 0;
+      for (var res : results) {
+        total += res.totalAssertsFailedOrErrored();
+      }
+      return total;
+    }
+
+    public int totalFailures() {
+      var total = 0;
+      for (var res : results) {
+        if (res.isFailure()) {
+          total++;
+        }
+      }
+      return total;
+    }
+
+    public boolean failed() {
+      for (var res : results) {
+        if (res.isFailure()) {
+          return true;
+        }
+      }
+      return false;
+    }
+
+    public boolean hasError() {
+      for (var res : results) {
+        if (res.hasErrors()) {
+          return true;
+        }
+      }
+      return false;
+    }
+  }
+
+  /**
+   * The result of a single test.
+   *
+   * <p>A test can have multiple assertions, where the assertion itself can have failed, or have
+   * thrown an exception.
+   *
+   * @param name The name of the test.
+   * @param totalAsserts The total number of assertions in the test.
+   * @param failures The number of assertion failures in the test.
+   * @param errors The number of errors that were thrown when the test was run.
+   * @param isExampleWritten Whether the test is considered as having its example written or not.
+   */
+  public record TestResult(
+      String name,
+      int totalAsserts,
+      List<Failure> failures,
+      List<Error> errors,
+      boolean isExampleWritten) {
+
+    public int totalAssertsFailedOrErrored() {
+      return failures().size() + errors.size();
+    }
+
+    public boolean isFailure() {
+      return totalAssertsFailedOrErrored() > 0;
+    }
+
+    public boolean hasErrors() {
+      return !errors().isEmpty();
+    }
+
+    public boolean hasFailures() {
+      return !failures.isEmpty();
+    }
+
+    public static class Builder {
+
+      private final String name;
+      private final List<Failure> failures = new ArrayList<>();
+      private final List<Error> errors = new ArrayList<>();
+      private boolean isExampleWritten;
+      private int count = 0;
+
+      public Builder(String name) {
+        this.name = name;
+      }
+
+      public Builder addSuccess() {
+        count++;
+        return this;
+      }
+
+      public int getCount() {
+        return count;
+      }
+
+      public Builder addFailure(Failure failure) {
+        this.failures.add(failure);
+        count++;
+        return this;
+      }
+
+      public Builder addError(Error error) {
+        this.errors.add(error);
+        count++;
+        return this;
+      }
+
+      public Builder setExampleWritten(boolean exampleWritten) {
+        this.isExampleWritten = exampleWritten;
+        return this;
+      }
+
+      public TestResult build() {
+        return new TestResult(name, count, failures, errors, isExampleWritten);
+      }
+    }
+  }
+
+  public record Error(String message, PklException exception) {}
+
+  public record Failure(String kind, String message) {}
+}

pkl-core/src/main/java/org/pkl/core/runtime/TestResults.java

@@ -1,358 +0,0 @@
-/*
- * Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     https://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.pkl.core.runtime;
-
-import com.oracle.truffle.api.source.SourceSection;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-import org.pkl.core.PklException;
-import org.pkl.core.runtime.TestResults.TestSectionResults.TestSection;
-
-/** Aggregate test results for a module. Used to verify test failures and generate reports. */
-public final class TestResults {
-  public final String moduleName;
-  public final String displayUri;
-  public final TestSectionResults module = new TestSectionResults(TestSection.MODULE);
-  public final TestSectionResults facts = new TestSectionResults(TestSection.FACTS);
-  public final TestSectionResults examples = new TestSectionResults(TestSection.EXAMPLES);
-  private String err = "";
-
-  public TestResults(String moduleName, String displayUri) {
-    this.moduleName = moduleName;
-    this.displayUri = displayUri;
-  }
-
-  public int totalTests() {
-    return module.totalTests() + facts.totalTests() + examples.totalTests();
-  }
-
-  public int totalFailures() {
-    return module.totalFailures() + facts.totalFailures() + examples.totalFailures();
-  }
-
-  public int totalAsserts() {
-    return module.totalAsserts() + facts.totalAsserts() + examples.totalAsserts();
-  }
-
-  public int totalAssertsFailed() {
-    return module.totalAssertsFailed() + facts.totalAssertsFailed() + examples.totalAssertsFailed();
-  }
-
-  public boolean failed() {
-    return module.failed() || facts.failed() || examples.failed();
-  }
-
-  public String getErr() {
-    return err;
-  }
-
-  public void setErr(String err) {
-    this.err = err;
-  }
-
-  public static class TestSectionResults {
-    public final TestSection name;
-    private final List<TestResult> results = new ArrayList<>();
-    private Error error;
-
-    public TestSectionResults(TestSection name) {
-      this.name = name;
-    }
-
-    public void setError(Error error) {
-      this.error = error;
-    }
-
-    public Error getError() {
-      return error;
-    }
-
-    public boolean hasError() {
-      return error != null;
-    }
-
-    public List<TestResult> getResults() {
-      return Collections.unmodifiableList(results);
-    }
-
-    public TestResult newResult(String name) {
-      var result = new TestResult(name, this.name == TestSection.EXAMPLES);
-      results.add(result);
-      return result;
-    }
-
-    public TestResult newResult(String name, Failure failure) {
-      var result = new TestResult(name, this.name == TestSection.EXAMPLES);
-      result.addFailure(failure);
-      results.add(result);
-      return result;
-    }
-
-    public TestResult newResult(String name, Error error) {
-      var result = new TestResult(name, this.name == TestSection.EXAMPLES);
-      result.addError(error);
-      results.add(result);
-      return result;
-    }
-
-    public int totalTests() {
-      var total = results.size();
-      return (hasError() ? ++total : total);
-    }
-
-    public int totalAsserts() {
-      int total = 0;
-      for (var res : results) {
-        total += res.totalAsserts();
-      }
-      return (hasError() ? ++total : total);
-    }
-
-    public int totalAssertsFailed() {
-      int total = 0;
-      for (var res : results) {
-        total += res.totalAssertsFailed();
-      }
-      return (hasError() ? ++total : total);
-    }
-
-    public int totalFailures() {
-      int total = 0;
-      for (var res : results) {
-        if (res.isFailure()) total++;
-      }
-      return (hasError() ? ++total : total);
-    }
-
-    public boolean failed() {
-      if (hasError()) return true;
-
-      for (var res : results) {
-        if (res.isFailure()) return true;
-      }
-      return false;
-    }
-
-    public static class TestResult {
-      public final String name;
-      private int totalAsserts = 0;
-      private int totalAssertsFailed = 0;
-      private final List<Failure> failures = new ArrayList<>();
-      private final List<Error> errors = new ArrayList<>();
-      public final boolean isExample;
-      private boolean isExampleWritten = false;
-
-      public TestResult(String name, boolean isExample) {
-        this.name = name;
-        this.isExample = isExample;
-      }
-
-      public boolean isSuccess() {
-        return failures.isEmpty() && errors.isEmpty();
-      }
-
-      public boolean isFailure() {
-        return !isSuccess();
-      }
-
-      public boolean isExampleWritten() {
-        return isExampleWritten;
-      }
-
-      public void setExampleWritten(boolean exampleWritten) {
-        isExampleWritten = exampleWritten;
-      }
-
-      public List<Failure> getFailures() {
-        return Collections.unmodifiableList(failures);
-      }
-
-      public void addFailure(Failure description) {
-        failures.add(description);
-        totalAssertsFailed++;
-      }
-
-      public List<Error> getErrors() {
-        return Collections.unmodifiableList(errors);
-      }
-
-      public void addError(Error err) {
-        errors.add(err);
-        totalAssertsFailed++;
-      }
-
-      public int totalAsserts() {
-        return totalAsserts;
-      }
-
-      public void countAssert() {
-        totalAsserts++;
-      }
-
-      public int totalAssertsFailed() {
-        return totalAssertsFailed;
-      }
-    }
-
-    public static class Failure {
-
-      private final String kind;
-      private final String failure;
-      private final String location;
-
-      private Failure(String kind, String failure, String location) {
-        this.kind = kind;
-        this.failure = failure;
-        this.location = location;
-      }
-
-      public String getKind() {
-        return kind;
-      }
-
-      public String getFailure() {
-        return failure;
-      }
-
-      public String getLocation() {
-        return location;
-      }
-
-      public static String renderLocation(String location) {
-        return "(" + location + ")";
-      }
-
-      public String getRendered() {
-        String rendered;
-
-        if (kind == "Fact Failure") {
-          rendered = failure + " " + renderLocation(getLocation());
-        } else {
-          rendered = renderLocation(getLocation()) + "\n" + failure;
-        }
-
-        return rendered;
-      }
-
-      public static Failure buildFactFailure(SourceSection sourceSection, String location) {
-        return new Failure("Fact Failure", sourceSection.getCharacters().toString(), location);
-      }
-
-      public static Failure buildExampleLengthMismatchFailure(
-          String location, String property, int expectedLength, int actualLength) {
-        var builder = new StringBuilder();
-        builder
-            .append("Output mismatch: Expected \"")
-            .append(property)
-            .append("\" to contain ")
-            .append(expectedLength)
-            .append(" examples, but found ")
-            .append(actualLength);
-
-        return new Failure("Output Mismatch (Length)", builder.toString(), location);
-      }
-
-      public static Failure buildExamplePropertyMismatchFailure(
-          String location, String property, boolean isMissingInExpected) {
-
-        String exists_in;
-        String missing_in;
-
-        if (isMissingInExpected) {
-          exists_in = "actual";
-          missing_in = "expected";
-        } else {
-          exists_in = "expected";
-          missing_in = "actual";
-        }
-
-        var builder = new StringBuilder();
-        builder
-            .append("Output mismatch: \"")
-            .append(property)
-            .append("\" exists in ")
-            .append(exists_in)
-            .append(" but not in ")
-            .append(missing_in)
-            .append(" output");
-
-        return new Failure("Output Mismatch", builder.toString(), location);
-      }
-
-      public static Failure buildExampleFailure(
-          String location,
-          String expectedLocation,
-          String expectedValue,
-          String actualLocation,
-          String actualValue) {
-        var builder = new StringBuilder();
-        builder
-            .append("Expected: ")
-            .append(renderLocation(expectedLocation))
-            .append("\n")
-            .append(expectedValue)
-            .append("\n")
-            .append("Actual: ")
-            .append(renderLocation(actualLocation))
-            .append("\n")
-            .append(actualValue);
-
-        return new Failure("Example Failure", builder.toString(), location);
-      }
-    }
-
-    public static class Error {
-
-      private final String message;
-      private final PklException exception;
-
-      public Error(String message, PklException exception) {
-        this.message = message;
-        this.exception = exception;
-      }
-
-      public String getMessage() {
-        return message;
-      }
-
-      public Exception getException() {
-        return exception;
-      }
-
-      public String getRendered() {
-        return exception.getMessage();
-      }
-    }
-
-    public enum TestSection {
-      MODULE("module"),
-      FACTS("facts"),
-      EXAMPLES("examples");
-
-      private final String name;
-
-      TestSection(final String name) {
-        this.name = name;
-      }
-
-      @Override
-      public String toString() {
-        return name;
-      }
-    }
-  }
-}

pkl-core/src/main/java/org/pkl/core/runtime/TestRunner.java

@@ -15,17 +15,23 @@
  */
 package org.pkl.core.runtime;
 
+import com.oracle.truffle.api.source.SourceSection;
 import java.io.IOException;
 import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
 import org.pkl.core.BufferedLogger;
 import org.pkl.core.StackFrameTransformer;
+import org.pkl.core.TestResults;
+import org.pkl.core.TestResults.Failure;
+import org.pkl.core.TestResults.TestResult;
+import org.pkl.core.TestResults.TestSectionName;
+import org.pkl.core.TestResults.TestSectionResults;
 import org.pkl.core.ast.member.ObjectMember;
 import org.pkl.core.module.ModuleKeys;
-import org.pkl.core.runtime.TestResults.TestSectionResults;
-import org.pkl.core.runtime.TestResults.TestSectionResults.Error;
-import org.pkl.core.runtime.TestResults.TestSectionResults.Failure;
 import org.pkl.core.stdlib.PklConverter;
 import org.pkl.core.stdlib.base.PcfRenderer;
 import org.pkl.core.util.EconomicMaps;
@@ -48,31 +54,20 @@ public final class TestRunner {
 
   public TestResults run(VmTyped testModule) {
     var info = VmUtils.getModuleInfo(testModule);
-    var results = new TestResults(info.getModuleName(), getDisplayUri(info));
+    var resultsBuilder = new TestResults.Builder(info.getModuleName(), getDisplayUri(info));
 
     try {
       checkAmendsPklTest(testModule);
     } catch (VmException v) {
-      var error = new Error(v.getMessage(), v.toPklException(stackFrameTransformer));
+      var error = new TestResults.Error(v.getMessage(), v.toPklException(stackFrameTransformer));
-      results.module.setError(error);
+      return resultsBuilder.setError(error).build();
     }
 
-    try {
+    resultsBuilder.setFactsSection(runFacts(testModule));
-      runFacts(testModule, results.facts);
+    resultsBuilder.setExamplesSection(runExamples(testModule, info));
-    } catch (VmException v) {
-      var error = new Error(v.getMessage(), v.toPklException(stackFrameTransformer));
-      results.facts.setError(error);
-    }
 
-    try {
+    resultsBuilder.setStdErr(logger.getLogs());
-      runExamples(testModule, info, results.examples);
+    return resultsBuilder.build();
-    } catch (VmException v) {
-      var error = new Error(v.getMessage(), v.toPklException(stackFrameTransformer));
-      results.examples.setError(error);
-    }
-
-    results.setErr(logger.getLogs());
-    return results;
   }
 
   private void checkAmendsPklTest(VmTyped value) {
@@ -86,41 +81,48 @@ public final class TestRunner {
     }
   }
 
-  private void runFacts(VmTyped testModule, TestSectionResults results) {
+  private TestSectionResults runFacts(VmTyped testModule) {
     var facts = VmUtils.readMember(testModule, Identifier.FACTS);
-    if (facts instanceof VmNull) return;
+    if (facts instanceof VmNull) return new TestSectionResults(TestSectionName.FACTS, List.of());
 
+    var testResults = new ArrayList<TestResult>();
     var factsMapping = (VmMapping) facts;
     factsMapping.forceAndIterateMemberValues(
         (groupKey, groupMember, groupValue) -> {
           var listing = (VmListing) groupValue;
-          var result = results.newResult(String.valueOf(groupKey));
+          var name = String.valueOf(groupKey);
-          return listing.iterateMembers(
+          var resultBuilder = new TestResult.Builder(name);
+          listing.iterateMembers(
               (idx, member) -> {
                 if (member.isLocalOrExternalOrHidden()) {
                   return true;
                 }
-
-                result.countAssert();
-
                 try {
                   var factValue = VmUtils.readMember(listing, idx);
                   if (factValue == Boolean.FALSE) {
-                    result.addFailure(
+                    var failure = factFailure(member.getSourceSection(), getDisplayUri(member));
-                        Failure.buildFactFailure(member.getSourceSection(), getDisplayUri(member)));
+                    resultBuilder.addFailure(failure);
+                  } else {
+                    resultBuilder.addSuccess();
                   }
                 } catch (VmException err) {
-                  result.addError(
+                  var error =
-                      new Error(err.getMessage(), err.toPklException(stackFrameTransformer)));
+                      new TestResults.Error(
+                          err.getMessage(), err.toPklException(stackFrameTransformer));
+                  resultBuilder.addError(error);
                 }
                 return true;
               });
+          testResults.add(resultBuilder.build());
+          return true;
         });
+    return new TestSectionResults(TestSectionName.FACTS, Collections.unmodifiableList(testResults));
   }
 
-  private void runExamples(VmTyped testModule, ModuleInfo info, TestSectionResults results) {
+  private TestSectionResults runExamples(VmTyped testModule, ModuleInfo info) {
     var examples = VmUtils.readMember(testModule, Identifier.EXAMPLES);
-    if (examples instanceof VmNull) return;
+    if (examples instanceof VmNull)
+      return new TestSectionResults(TestSectionName.EXAMPLES, List.of());
 
     var moduleUri = info.getModuleKey().getUri();
     if (!moduleUri.getScheme().equalsIgnoreCase("file")) {
@@ -154,72 +156,59 @@ public final class TestRunner {
     }
 
     if (Files.exists(expectedOutputFile)) {
-      doRunAndValidateExamples(examplesMapping, expectedOutputFile, actualOutputFile, results);
+      return doRunAndValidateExamples(examplesMapping, expectedOutputFile, actualOutputFile);
     } else {
-      doRunAndWriteExamples(examplesMapping, expectedOutputFile, results);
+      return doRunAndWriteExamples(examplesMapping, expectedOutputFile);
     }
   }
 
-  private void doRunAndValidateExamples(
+  private TestSectionResults doRunAndValidateExamples(
-      VmMapping examples,
+      VmMapping examples, Path expectedOutputFile, Path actualOutputFile) {
-      Path expectedOutputFile,
-      Path actualOutputFile,
-      TestSectionResults results) {
     var expectedExampleOutputs = loadExampleOutputs(expectedOutputFile);
     var actualExampleOutputs = new MutableReference<VmDynamic>(null);
     var allGroupsSucceeded = new MutableBoolean(true);
     var errored = new MutableBoolean(false);
+    var testResults = new ArrayList<TestResult>();
     examples.forceAndIterateMemberValues(
         (groupKey, groupMember, groupValue) -> {
           var testName = String.valueOf(groupKey);
           var group = (VmListing) groupValue;
           var expectedGroup =
               (VmDynamic) VmUtils.readMemberOrNull(expectedExampleOutputs, groupKey);
+          var testResultBuilder = new TestResult.Builder(testName);
 
           if (expectedGroup == null) {
-            results
+            testResultBuilder.addFailure(
-                .newResult(
+                examplePropertyMismatchFailure(getDisplayUri(groupMember), testName, true));
-                    testName,
+            testResults.add(testResultBuilder.build());
-                    Failure.buildExamplePropertyMismatchFailure(
-                        getDisplayUri(groupMember), testName, true))
-                .countAssert();
             return true;
           }
 
           if (group.getLength() != expectedGroup.getLength()) {
-            results
+            testResultBuilder.addFailure(
-                .newResult(
+                exampleLengthMismatchFailure(
+                    getDisplayUri(groupMember),
                     testName,
-                    Failure.buildExampleLengthMismatchFailure(
+                    expectedGroup.getLength(),
-                        getDisplayUri(groupMember),
+                    group.getLength()));
-                        testName,
+            testResults.add(testResultBuilder.build());
-                        expectedGroup.getLength(),
-                        group.getLength()))
-                .countAssert();
             return true;
           }
 
-          var groupSucceeded = new MutableBoolean(true);
           group.iterateMembers(
               ((exampleIndex, exampleMember) -> {
                 if (exampleMember.isLocalOrExternalOrHidden()) {
                   return true;
                 }
 
-                var exampleName =
-                    group.getLength() == 1 ? testName : testName + " #" + exampleIndex;
-
                 Object exampleValue;
                 try {
                   exampleValue = VmUtils.readMember(group, exampleIndex);
                 } catch (VmException err) {
                   errored.set(true);
-                  results
+                  testResultBuilder.addError(
-                      .newResult(
+                      new TestResults.Error(
-                          exampleName,
+                          err.getMessage(), err.toPklException(stackFrameTransformer)));
-                          new Error(err.getMessage(), err.toPklException(stackFrameTransformer)))
-                      .countAssert();
-                  groupSucceeded.set(false);
                   return true;
                 }
                 var expectedValue = VmUtils.readMember(expectedGroup, exampleIndex);
@@ -236,8 +225,6 @@ public final class TestRunner {
                     actualExampleOutputs.set(loadExampleOutputs(actualOutputFile));
                   }
 
-                  groupSucceeded.set(false);
-
                   var expectedMember = VmUtils.findMember(expectedGroup, exampleIndex);
                   assert expectedMember != null;
 
@@ -253,26 +240,22 @@ public final class TestRunner {
                         .build();
                   }
 
-                  results
+                  testResultBuilder.addFailure(
-                      .newResult(
+                      exampleFailure(
-                          exampleName,
+                          getDisplayUri(exampleMember),
-                          Failure.buildExampleFailure(
+                          getDisplayUri(expectedMember),
-                              getDisplayUri(exampleMember),
+                          expectedValuePcf,
-                              getDisplayUri(expectedMember),
+                          getDisplayUri(actualMember),
-                              expectedValuePcf,
+                          exampleValuePcf,
-                              getDisplayUri(actualMember),
+                          testResultBuilder.getCount()));
-                              exampleValuePcf))
-                      .countAssert();
                 } else {
-                  results.newResult(exampleName).countAssert();
+                  testResultBuilder.addSuccess();
                 }
 
                 return true;
               }));
 
-          if (!groupSucceeded.get()) {
+          testResults.add(testResultBuilder.build());
-            allGroupsSucceeded.set(false);
-          }
 
           return true;
         });
@@ -285,12 +268,12 @@ public final class TestRunner {
           if (examples.getCachedValue(groupKey) == null) {
             var testName = String.valueOf(groupKey);
             allGroupsSucceeded.set(false);
-            results
+            var result =
-                .newResult(
+                new TestResult.Builder(testName)
-                    testName,
+                    .addFailure(
-                    Failure.buildExamplePropertyMismatchFailure(
+                        examplePropertyMismatchFailure(getDisplayUri(groupMember), testName, false))
-                        getDisplayUri(groupMember), testName, false))
+                    .build();
-                .countAssert();
+            testResults.add(result);
           }
           return true;
         });
@@ -298,49 +281,50 @@ public final class TestRunner {
     if (!allGroupsSucceeded.get() && actualExampleOutputs.isNull() && !errored.get()) {
       writeExampleOutputs(actualOutputFile, examples);
     }
+    return new TestSectionResults(
+        TestSectionName.EXAMPLES, Collections.unmodifiableList(testResults));
   }
 
-  private void doRunAndWriteExamples(
+  private TestSectionResults doRunAndWriteExamples(VmMapping examples, Path outputFile) {
-      VmMapping examples, Path outputFile, TestSectionResults results) {
+    var testResults = new ArrayList<TestResult>();
-    var allSucceeded =
+    var allSucceeded = new MutableBoolean(true);
-        examples.forceAndIterateMemberValues(
+    examples.forceAndIterateMemberValues(
-            (groupKey, groupMember, groupValue) -> {
+        (groupKey, groupMember, groupValue) -> {
-              var testName = String.valueOf(groupKey);
+          var testName = String.valueOf(groupKey);
-              var listing = (VmListing) groupValue;
+          var listing = (VmListing) groupValue;
-              var success =
+          var testResultBuilder = new TestResult.Builder(testName);
-                  listing.iterateMembers(
+          var success = new MutableBoolean(true);
-                      (idx, member) -> {
+          listing.iterateMembers(
-                        if (member.isLocalOrExternalOrHidden()) {
+              (idx, member) -> {
-                          return true;
+                if (member.isLocalOrExternalOrHidden()) {
-                        }
+                  return true;
-
+                }
-                        var exampleName =
+                try {
-                            listing.getLength() == 1 ? testName : testName + " #" + idx;
+                  VmUtils.readMember(listing, idx);
-
+                  return true;
-                        try {
+                } catch (VmException err) {
-                          VmUtils.readMember(listing, idx);
+                  testResultBuilder.addError(
-                          return true;
+                      new TestResults.Error(
-                        } catch (VmException err) {
+                          err.getMessage(), err.toPklException(stackFrameTransformer)));
-                          results
+                  allSucceeded.set(false);
-                              .newResult(
+                  success.set(false);
-                                  exampleName,
+                  return true;
-                                  new Error(
+                }
-                                      err.getMessage(), err.toPklException(stackFrameTransformer)))
+              });
-                              .countAssert();
+          if (success.get()) {
-                          return false;
+            // treat writing an example as a message
-                        }
+            testResultBuilder.setExampleWritten(true);
-                      });
+            testResultBuilder.addFailure(
-              if (!success) {
+                writtenExampleOutputFailure(testName, getDisplayUri(groupMember)));
-                return false;
+          }
-              }
+          testResults.add(testResultBuilder.build());
-              var result = results.newResult(testName);
+          return true;
-              result.countAssert();
+        });
-              result.setExampleWritten(true);
+    if (allSucceeded.get()) {
-              return true;
-            });
-    if (allSucceeded) {
       writeExampleOutputs(outputFile, examples);
     }
+    return new TestSectionResults(
+        TestSectionName.EXAMPLES, Collections.unmodifiableList(testResults));
   }
 
   private void writeExampleOutputs(Path outputFile, VmMapping examples) {
@@ -402,4 +386,87 @@ public final class TestRunner {
     return VmUtils.getDisplayUri(
         moduleInfo.getModuleKey().getUri(), VmContext.get(null).getFrameTransformer());
   }
+
+  private static Failure factFailure(SourceSection sourceSection, String location) {
+    String message = sourceSection.getCharacters().toString() + " " + renderLocation(location);
+    return new Failure("Fact Failure", message);
+  }
+
+  private static Failure exampleLengthMismatchFailure(
+      String location, String property, int expectedLength, int actualLength) {
+    String msg =
+        renderLocation(location)
+            + "\n"
+            + "Output mismatch: Expected \""
+            + property
+            + "\" to contain "
+            + expectedLength
+            + " examples, but found "
+            + actualLength;
+
+    return new Failure("Output Mismatch (Length)", msg);
+  }
+
+  private static Failure examplePropertyMismatchFailure(
+      String location, String property, boolean isMissingInExpected) {
+
+    String existsIn;
+    String missingIn;
+
+    if (isMissingInExpected) {
+      existsIn = "actual";
+      missingIn = "expected";
+    } else {
+      existsIn = "expected";
+      missingIn = "actual";
+    }
+
+    String message =
+        renderLocation(location)
+            + "\n"
+            + "Output mismatch: \""
+            + property
+            + "\" exists in "
+            + existsIn
+            + " but not in "
+            + missingIn
+            + " output";
+
+    return new Failure("Output Mismatch", message);
+  }
+
+  private static Failure exampleFailure(
+      String location,
+      String expectedLocation,
+      String expectedValue,
+      String actualLocation,
+      String actualValue,
+      int exampleNumber) {
+    String err =
+        "#"
+            + exampleNumber
+            + " "
+            + renderLocation(location)
+            + ":\n  "
+            + "Expected: "
+            + renderLocation(expectedLocation)
+            + "\n  "
+            + expectedValue.replaceAll("\n", "\n  ")
+            + "\n  "
+            + "Actual: "
+            + renderLocation(actualLocation)
+            + "\n  "
+            + actualValue.replaceAll("\n", "\n  ");
+
+    return new Failure("Example Failure", err);
+  }
+
+  private static String renderLocation(String location) {
+    return "(" + location + ")";
+  }
+
+  private static Failure writtenExampleOutputFailure(String testName, String location) {
+    var message = renderLocation(location) + "\n" + "Wrote expected output for test " + testName;
+    return new Failure("Example Output Written", message);
+  }
 }

pkl-core/src/main/java/org/pkl/core/stdlib/test/report/JUnitReport.java

@@ -20,13 +20,13 @@ import java.io.Writer;
 import java.util.ArrayList;
 import java.util.function.Consumer;
 import org.graalvm.collections.EconomicMap;
+import org.pkl.core.TestResults;
+import org.pkl.core.TestResults.Error;
+import org.pkl.core.TestResults.TestResult;
+import org.pkl.core.TestResults.TestSectionResults;
 import org.pkl.core.ast.member.ObjectMember;
 import org.pkl.core.runtime.BaseModule;
 import org.pkl.core.runtime.Identifier;
-import org.pkl.core.runtime.TestResults;
-import org.pkl.core.runtime.TestResults.TestSectionResults;
-import org.pkl.core.runtime.TestResults.TestSectionResults.Error;
-import org.pkl.core.runtime.TestResults.TestSectionResults.TestResult;
 import org.pkl.core.runtime.VmDynamic;
 import org.pkl.core.runtime.VmMapping;
 import org.pkl.core.runtime.VmTyped;
@@ -44,44 +44,51 @@ public final class JUnitReport implements TestReport {
   }
 
   private VmDynamic buildSuite(TestResults results) {
-    var testCases = testCases(results.moduleName, results.facts);
+    if (results.error() != null) {
-    testCases.addAll(testCases(results.moduleName, results.examples));
+      var testCase = rootTestCase(results, results.error());
+      var attrs =
+          buildAttributes(
+              "name", results.moduleName(),
+              "tests", 1,
+              "failures", 1);
+      return buildXmlElement("testsuite", attrs, testCase);
+    }
 
-    if (!results.getErr().isBlank()) {
+    var testCases = testCases(results.moduleName(), results.facts());
+    testCases.addAll(testCases(results.moduleName(), results.examples()));
+
+    if (!results.logs().isBlank()) {
       var err =
           buildXmlElement(
               "system-err",
               VmMapping.empty(),
-              members -> members.put("body", syntheticElement(makeCdata(results.getErr()))));
+              members -> members.put("body", syntheticElement(makeCdata(results.logs()))));
       testCases.add(err);
     }
 
     var attrs =
         buildAttributes(
-            "name", results.moduleName,
+            "name", results.moduleName(),
             "tests", (long) results.totalTests(),
             "failures", (long) results.totalFailures());
 
     return buildXmlElement("testsuite", attrs, testCases.toArray(new VmDynamic[0]));
   }
 
+  private VmDynamic rootTestCase(TestResults results, TestResults.Error error) {
+    var testCaseAttrs =
+        buildAttributes("classname", results.moduleName(), "name", results.moduleName());
+    var err = error(error);
+    return buildXmlElement("testcase", testCaseAttrs, err.toArray(new VmDynamic[0]));
+  }
+
   private ArrayList<VmDynamic> testCases(String moduleName, TestSectionResults testSectionResults) {
     var elements = new ArrayList<VmDynamic>(testSectionResults.totalTests());
 
-    if (testSectionResults.hasError()) {
+    for (var res : testSectionResults.results()) {
-      var error = error(testSectionResults.getError());
-
-      var attrs =
-          buildAttributes("classname", moduleName + "." + testSectionResults.name, "name", "error");
-      var element = buildXmlElement("testcase", attrs, error.toArray(new VmDynamic[0]));
-
-      elements.add(element);
-    }
-
-    for (var res : testSectionResults.getResults()) {
       var attrs =
           buildAttributes(
-              "classname", moduleName + "." + testSectionResults.name, "name", res.name);
+              "classname", moduleName + "." + testSectionResults.name(), "name", res.name());
       var failures = failures(res);
       failures.addAll(errors(res));
       var element = buildXmlElement("testcase", attrs, failures.toArray(new VmDynamic[0]));
@@ -93,14 +100,12 @@ public final class JUnitReport implements TestReport {
   private ArrayList<VmDynamic> failures(TestResult res) {
     var list = new ArrayList<VmDynamic>();
     long i = 0;
-    for (var fail : res.getFailures()) {
+    for (var fail : res.failures()) {
-      var attrs = buildAttributes("message", fail.getKind());
+      var attrs = buildAttributes("message", fail.kind());
       long element = i++;
       list.add(
           buildXmlElement(
-              "failure",
+              "failure", attrs, members -> members.put(element, syntheticElement(fail.message()))));
-              attrs,
-              members -> members.put(element, syntheticElement(fail.getRendered()))));
     }
     return list;
   }
@@ -108,27 +113,26 @@ public final class JUnitReport implements TestReport {
   private ArrayList<VmDynamic> errors(TestResult res) {
     var list = new ArrayList<VmDynamic>();
     long i = 0;
-    for (var error : res.getErrors()) {
+    for (var error : res.errors()) {
-      var attrs = buildAttributes("message", error.getMessage());
+      var attrs = buildAttributes("message", error.message());
       long element = i++;
       list.add(
           buildXmlElement(
               "error",
               attrs,
-              members ->
+              members -> members.put(element, syntheticElement(error.exception().getMessage()))));
-                  members.put(element, syntheticElement(error.getException().getMessage()))));
     }
     return list;
   }
 
   private ArrayList<VmDynamic> error(Error error) {
     var list = new ArrayList<VmDynamic>();
-    var attrs = buildAttributes("message", error.getMessage());
+    var attrs = buildAttributes("message", error.message());
     list.add(
         buildXmlElement(
             "error",
             attrs,
-            members -> members.put(1, syntheticElement("\n" + error.getRendered()))));
+            members -> members.put(1, syntheticElement("\n" + error.exception().getMessage()))));
     return list;
   }
 

pkl-core/src/main/java/org/pkl/core/stdlib/test/report/SimpleReport.java

@@ -18,9 +18,9 @@ package org.pkl.core.stdlib.test.report;
 import java.io.IOException;
 import java.io.Writer;
 import java.util.stream.Collectors;
-import org.pkl.core.runtime.TestResults;
+import org.pkl.core.TestResults;
-import org.pkl.core.runtime.TestResults.TestSectionResults;
+import org.pkl.core.TestResults.TestResult;
-import org.pkl.core.runtime.TestResults.TestSectionResults.TestResult;
+import org.pkl.core.TestResults.TestSectionResults;
 import org.pkl.core.util.StringUtils;
 
 public final class SimpleReport implements TestReport {
@@ -29,10 +29,22 @@ public final class SimpleReport implements TestReport {
   public void report(TestResults results, Writer writer) throws IOException {
     var builder = new StringBuilder();
 
-    builder.append("module ").append(results.moduleName).append("\n");
+    builder.append("module ").append(results.moduleName()).append("\n");
 
-    reportResults(results.facts, builder);
+    if (results.error() != null) {
-    reportResults(results.examples, builder);
+      var rendered = results.error().exception().getMessage();
+      appendPadded(builder, rendered, "  ");
+      builder.append('\n');
+    } else {
+      reportResults(results.facts(), builder);
+      reportResults(results.examples(), builder);
+    }
+
+    if (results.isExampleWrittenFailure()) {
+      builder.append(results.examples().totalFailures()).append(" examples written\n");
+      writer.append(builder);
+      return;
+    }
 
     builder.append(results.failed() ? "❌ " : "✅ ");
 
@@ -51,16 +63,11 @@ public final class SimpleReport implements TestReport {
   }
 
   private void reportResults(TestSectionResults section, StringBuilder builder) {
-    if (!section.getResults().isEmpty()) {
+    if (!section.results().isEmpty()) {
-      builder.append("  ").append(section.name).append("\n");
+      builder.append("  ").append(section.name()).append("\n");
 
       StringUtils.joinToStringBuilder(
-          builder, section.getResults(), "\n", res -> reportResult(res, builder));
+          builder, section.results(), "\n", res -> reportResult(res, builder));
-      builder.append("\n");
-    } else if (section.hasError()) {
-      builder.append("  ").append(section.name).append("\n");
-      var error = section.getError().getRendered();
-      appendPadded(builder, error, "    ");
       builder.append("\n");
     }
   }
@@ -69,23 +76,22 @@ public final class SimpleReport implements TestReport {
     builder.append("    ");
 
     if (result.isExampleWritten()) {
-      builder.append(result.name).append(" ✍️");
+      builder.append("✍️ ").append(result.name());
     } else {
-      builder.append(result.isFailure() ? "❌ " : "✅ ").append(result.name);
+      builder.append(result.isFailure() ? "❌ " : "✅ ").append(result.name());
-
       if (result.isFailure()) {
         var failurePadding = "       ";
         builder.append("\n");
         StringUtils.joinToStringBuilder(
             builder,
-            result.getFailures(),
+            result.failures(),
             "\n",
-            failure -> appendPadded(builder, failure.getRendered(), failurePadding));
+            failure -> appendPadded(builder, failure.message(), failurePadding));
         StringUtils.joinToStringBuilder(
             builder,
-            result.getErrors(),
+            result.errors(),
             "\n",
-            error -> appendPadded(builder, error.getException().getMessage(), failurePadding));
+            error -> appendPadded(builder, error.exception().getMessage(), failurePadding));
       }
     }
   }
@@ -96,15 +102,15 @@ public final class SimpleReport implements TestReport {
         lines.lines().collect(Collectors.toList()),
         "\n",
         str -> {
-          if (str.length() > 0) builder.append(padding).append(str);
+          if (!str.isEmpty()) builder.append(padding).append(str);
         });
   }
 
   private String makeStatsLine(String kind, int total, int failed, boolean isFailed) {
     var passed = total - failed;
-    var pct_passed = total > 0 ? 100.0 * passed / total : 0.0;
+    var passRate = total > 0 ? 100.0 * passed / total : 0.0;
 
-    String line = String.format("%.1f%% %s pass", pct_passed, kind);
+    String line = String.format("%.1f%% %s pass", passRate, kind);
 
     if (isFailed) {
       line += String.format(" [%d/%d failed]", failed, total);

pkl-core/src/main/java/org/pkl/core/stdlib/test/report/TestReport.java

@@ -21,7 +21,7 @@ import java.io.Writer;
 import java.nio.charset.StandardCharsets;
 import java.nio.file.Path;
 import org.pkl.core.PklBugException;
-import org.pkl.core.runtime.TestResults;
+import org.pkl.core.TestResults;
 import org.pkl.core.util.StringBuilderWriter;
 
 public interface TestReport {

pkl-core/src/test/kotlin/org/pkl/core/EvaluateTestsTest.kt

@@ -54,8 +54,8 @@ class EvaluateTestsTest {
     assertThat(results.displayUri).isEqualTo("repl:text")
     assertThat(results.totalTests()).isEqualTo(1)
     assertThat(results.failed()).isFalse
-    assertThat(results.facts.results[0].name).isEqualTo("should pass")
+    assertThat(results.facts().results[0].name).isEqualTo("should pass")
-    assertThat(results.err.isBlank()).isTrue
+    assertThat(results.logs().isBlank()).isTrue
   }
 
   @Test
@@ -82,16 +82,16 @@ class EvaluateTestsTest {
     assertThat(results.totalFailures()).isEqualTo(1)
     assertThat(results.failed()).isTrue
 
-    val res = results.facts.results[0]
+    val res = results.facts().results[0]
     assertThat(res.name).isEqualTo("should fail")
-    assertThat(results.facts.hasError()).isFalse
+    assertThat(results.facts().hasError()).isFalse
     assertThat(res.failures.size).isEqualTo(2)
 
     val fail1 = res.failures[0]
-    assertThat(fail1.rendered).isEqualTo("1 == 2 (repl:text)")
+    assertThat(fail1.message).isEqualTo("1 == 2 (repl:text)")
 
     val fail2 = res.failures[1]
-    assertThat(fail2.rendered).isEqualTo(""""foo" == "bar" (repl:text)""")
+    assertThat(fail2.message).isEqualTo(""""foo" == "bar" (repl:text)""")
   }
 
   @Test
@@ -118,7 +118,7 @@ class EvaluateTestsTest {
     assertThat(results.totalFailures()).isEqualTo(1)
     assertThat(results.failed()).isTrue
 
-    val res = results.facts.results[0]
+    val res = results.facts().results[0]
     assertThat(res.name).isEqualTo("should fail")
     assertThat(res.failures).hasSize(1)
     assertThat(res.errors).hasSize(1)
@@ -180,7 +180,7 @@ class EvaluateTestsTest {
     assertThat(results.displayUri).startsWith("file:///").endsWith(".pkl")
     assertThat(results.totalTests()).isEqualTo(1)
     assertThat(results.failed()).isFalse
-    assertThat(results.examples.results[0].name).isEqualTo("user")
+    assertThat(results.examples().results[0].name).isEqualTo("user")
   }
 
   @Test
@@ -232,9 +232,9 @@ class EvaluateTestsTest {
     assertThat(results.totalFailures()).isEqualTo(1)
     assertThat(results.failed()).isTrue
 
-    assertThat(results.facts.results[0].name).isEqualTo("should fail")
+    assertThat(results.facts().results[0].name).isEqualTo("should fail")
-    assertThat(results.facts.results[0].failures.size).isEqualTo(2)
+    assertThat(results.facts().results[0].failures.size).isEqualTo(2)
-    assertThat(results.examples.results[0].name).isEqualTo("user")
+    assertThat(results.examples().results[0].name).isEqualTo("user")
   }
 
   @Test
@@ -286,7 +286,7 @@ class EvaluateTestsTest {
     assertThat(results.totalFailures()).isEqualTo(1)
     assertThat(results.failed()).isTrue
 
-    val res = results.facts.results[0]
+    val res = results.facts().results[0]
     assertThat(res.name).isEqualTo("should fail")
     assertThat(res.failures).hasSize(0)
     assertThat(res.errors).hasSize(1)
@@ -294,7 +294,7 @@ class EvaluateTestsTest {
     val error = res.errors[0]
     assertThat(error.message).isEqualTo("exception")
 
-    assertThat(results.examples.results[0].name).isEqualTo("user")
+    assertThat(results.examples().results[0].name).isEqualTo("user")
   }
 
   @Test
@@ -339,26 +339,26 @@ class EvaluateTestsTest {
     assertThat(results.failed()).isTrue
     assertThat(results.totalFailures()).isEqualTo(1)
 
-    val res = results.examples.results[0]
+    val res = results.examples().results[0]
     assertThat(res.name).isEqualTo("user")
-    assertFalse(results.examples.hasError())
+    assertFalse(results.examples().hasError())
 
     val fail1 = res.failures[0]
-    assertThat(fail1.rendered.stripFileAndLines(tempDir))
+    assertThat(fail1.message.stripFileAndLines(tempDir))
       .isEqualTo(
         """
-      (/tempDir/example.pkl)
+        #0 (/tempDir/example.pkl):
-      Expected: (/tempDir/example.pkl-expected.pcf)
+          Expected: (/tempDir/example.pkl-expected.pcf)
-      new {
+          new {
-        name = "Alice"
+            name = "Alice"
-        age = 45
+            age = 45
-      }
+          }
-      Actual: (/tempDir/example.pkl-actual.pcf)
+          Actual: (/tempDir/example.pkl-actual.pcf)
-      new {
+          new {
-        name = "Bob"
+            name = "Bob"
-        age = 33
+            age = 33
-      }
+          }
-    """
+        """
           .trimIndent()
       )
   }

pkl-gradle/src/test/kotlin/org/pkl/gradle/TestsTest.kt

@@ -112,34 +112,33 @@ class TestsTest : AbstractTest() {
     assertThat(output.trimStart())
       .startsWith(
         """
-      > Task :evalTestGatherImports
+        > Task :evalTestGatherImports
 
-      > Task :evalTest FAILED
+        > Task :evalTest FAILED
-      pkl: TRACE: 8 = 8 (file:///file, line x)
+        pkl: TRACE: 8 = 8 (file:///file, line x)
-      module test
+        module test
-        facts
+          facts
-          ✅ sum numbers
+            ✅ sum numbers
-          ✅ divide numbers
+            ✅ divide numbers
-          ❌ fail
+            ❌ fail
-             4 == 9 (file:///file, line x)
+               4 == 9 (file:///file, line x)
-             "foo" == "bar" (file:///file, line x)
+               "foo" == "bar" (file:///file, line x)
-        examples
+          examples
-          ✅ user 0
+            ✅ user 0
-          ✅ user 1 #0
+            ❌ user 1
-          ❌ user 1 #1
+               #1 (file:///file, line x):
-             (file:///file, line x)
+                 Expected: (file:///file, line x)
-             Expected: (file:///file, line x)
+                 new {
-             new {
+                   name = "Parrot"
-               name = "Parrot"
+                   age = 35
-               age = 35
+                 }
-             }
+                 Actual: (file:///file, line x)
-             Actual: (file:///file, line x)
+                 new {
-             new {
+                   name = "Welma"
-               name = "Welma"
+                   age = 35
-               age = 35
+                 }
-             }
+        ❌ 60.0% tests pass [2/5 failed], 66.7% asserts pass [3/9 failed]
-      ❌ 66.7% tests pass [2/6 failed], 66.7% asserts pass [3/9 failed]
+        """
-    """
           .trimIndent()
       )
   }
@@ -151,10 +150,10 @@ class TestsTest : AbstractTest() {
 
     writeBuildFile("overwrite = true")
 
-    val output = runTask("evalTest").output
+    val output = runTask("evalTest", expectFailure = true).output
 
-    assertThat(output).contains("user 0 ✍️")
+    assertThat(output).contains("✍️ user 0")
-    assertThat(output).contains("user 1 ✍️")
+    assertThat(output).contains("✍️ user 1")
   }
 
   @Test
@@ -183,36 +182,36 @@ class TestsTest : AbstractTest() {
     assertThat(output.trimStart())
       .startsWith(
         """
-      > Task :evalTestGatherImports
+        > Task :evalTestGatherImports
         
-      > Task :evalTest FAILED
+        > Task :evalTest FAILED
-      module test
+        module test
-        facts
+          facts
-          ✅ should pass
+            ✅ should pass
-          ❌ error
+            ❌ error
-             –– Pkl Error ––
+               –– Pkl Error ––
-             exception
+               exception
         
-             9 | throw("exception")
+               9 | throw("exception")
-                 ^^^^^^^^^^^^^^^^^^
+                   ^^^^^^^^^^^^^^^^^^
-             at test#facts["error"][#1] (file:///file, line x)
+               at test#facts["error"][#1] (file:///file, line x)
-        examples
+          examples
-          ✅ user 0
+            ✅ user 0
-          ✅ user 1 #0
+            ❌ user 1
-          ❌ user 1 #1
+               #1 (file:///file, line x):
-             (file:///file, line x)
+                 Expected: (file:///file, line x)
-             Expected: (file:///file, line x)
+                 new {
-             new {
+                   name = "Parrot"
-               name = "Parrot"
+                   age = 35
-               age = 35
+                 }
-             }
+                 Actual: (file:///file, line x)
-             Actual: (file:///file, line x)
+                 new {
-             new {
+                   name = "Welma"
-               name = "Welma"
+                   age = 35
-               age = 35
+                 }
-             }
+        ❌ 50.0% tests pass [2/4 failed], 66.7% asserts pass [2/6 failed]
-      ❌ 60.0% tests pass [2/5 failed], 66.7% asserts pass [2/6 failed]
+
-    """
+        """
           .trimIndent()
       )
   }
@@ -232,34 +231,33 @@ class TestsTest : AbstractTest() {
     assertThat(report)
       .isEqualTo(
         """
-      <?xml version="1.0" encoding="UTF-8"?>
+        <?xml version="1.0" encoding="UTF-8"?>
-      <testsuite name="test" tests="6" failures="2">
+        <testsuite name="test" tests="5" failures="2">
-          <testcase classname="test.facts" name="sum numbers"></testcase>
+            <testcase classname="test.facts" name="sum numbers"></testcase>
-          <testcase classname="test.facts" name="divide numbers"></testcase>
+            <testcase classname="test.facts" name="divide numbers"></testcase>
-          <testcase classname="test.facts" name="fail">
+            <testcase classname="test.facts" name="fail">
-              <failure message="Fact Failure">4 == 9 (file:///file, line x)</failure>
+                <failure message="Fact Failure">4 == 9 (file:///file, line x)</failure>
-              <failure message="Fact Failure">&quot;foo&quot; == &quot;bar&quot; (file:///file, line x)</failure>
+                <failure message="Fact Failure">&quot;foo&quot; == &quot;bar&quot; (file:///file, line x)</failure>
-          </testcase>
+            </testcase>
-          <testcase classname="test.examples" name="user 0"></testcase>
+            <testcase classname="test.examples" name="user 0"></testcase>
-          <testcase classname="test.examples" name="user 1 #0"></testcase>
+            <testcase classname="test.examples" name="user 1">
-          <testcase classname="test.examples" name="user 1 #1">
+                <failure message="Example Failure">#1 (file:///file, line x):
-              <failure message="Example Failure">(file:///file, line x)
+          Expected: (file:///file, line x)
-      Expected: (file:///file, line x)
+          new {
-      new {
+            name = &quot;Parrot&quot;
-        name = &quot;Parrot&quot;
+            age = 35
-        age = 35
+          }
-      }
+          Actual: (file:///file, line x)
-      Actual: (file:///file, line x)
+          new {
-      new {
+            name = &quot;Welma&quot;
-        name = &quot;Welma&quot;
+            age = 35
-        age = 35
+          }</failure>
-      }</failure>
+            </testcase>
-          </testcase>
+            <system-err><![CDATA[8 = 8
-          <system-err><![CDATA[8 = 8
+        ]]></system-err>
-      ]]></system-err>
+        </testsuite>
-      </testsuite>
         
-    """
+        """
           .trimIndent()
       )
   }
@@ -289,36 +287,35 @@ class TestsTest : AbstractTest() {
     assertThat(report)
       .isEqualTo(
         """
-      <?xml version="1.0" encoding="UTF-8"?>
+        <?xml version="1.0" encoding="UTF-8"?>
-      <testsuite name="test" tests="5" failures="2">
+        <testsuite name="test" tests="4" failures="2">
-          <testcase classname="test.facts" name="should pass"></testcase>
+            <testcase classname="test.facts" name="should pass"></testcase>
-          <testcase classname="test.facts" name="error">
+            <testcase classname="test.facts" name="error">
-              <error message="exception">–– Pkl Error ––
+                <error message="exception">–– Pkl Error ––
-      exception
+        exception
         
-      9 | throw(&quot;exception&quot;)
+        9 | throw(&quot;exception&quot;)
-          ^^^^^^^^^^^^^^^^^^
+            ^^^^^^^^^^^^^^^^^^
-      at test#facts[&quot;error&quot;][#1] (file:///file, line x)
+        at test#facts[&quot;error&quot;][#1] (file:///file, line x)
-      </error>
+        </error>
-          </testcase>
+            </testcase>
-          <testcase classname="test.examples" name="user 0"></testcase>
+            <testcase classname="test.examples" name="user 0"></testcase>
-          <testcase classname="test.examples" name="user 1 #0"></testcase>
+            <testcase classname="test.examples" name="user 1">
-          <testcase classname="test.examples" name="user 1 #1">
+                <failure message="Example Failure">#1 (file:///file, line x):
-              <failure message="Example Failure">(file:///file, line x)
+          Expected: (file:///file, line x)
-      Expected: (file:///file, line x)
+          new {
-      new {
+            name = &quot;Parrot&quot;
-        name = &quot;Parrot&quot;
+            age = 35
-        age = 35
+          }
-      }
+          Actual: (file:///file, line x)
-      Actual: (file:///file, line x)
+          new {
-      new {
+            name = &quot;Welma&quot;
-        name = &quot;Welma&quot;
+            age = 35
-        age = 35
+          }</failure>
-      }</failure>
+            </testcase>
-          </testcase>
+        </testsuite>
-      </testsuite>
 
-    """
+        """
           .trimIndent()
       )
   }