Improve testing with stats and errors per test section (#498)

* Emojis are moves to the left to be aligned
* A summary line is added with test counts
* Facts and Examples are grouped under their own section

pkl-core/src/main/java/org/pkl/core/runtime/TestResults.java

@@ -20,61 +20,40 @@ import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
 import org.pkl.core.PklException;
+import org.pkl.core.runtime.TestResults.TestSectionResults.TestSection;
 
 /** Aggregate test results for a module. Used to verify test failures and generate reports. */
 public final class TestResults {
-
-  private final String module;
-  private final String displayUri;
-  private final List<TestResult> results = new ArrayList<>();
+  public final String moduleName;
+  public final String displayUri;
+  public final TestSectionResults module = new TestSectionResults(TestSection.MODULE);
+  public final TestSectionResults facts = new TestSectionResults(TestSection.FACTS);
+  public final TestSectionResults examples = new TestSectionResults(TestSection.EXAMPLES);
   private String err = "";
 
-  public TestResults(String module, String displayUri) {
-    this.module = module;
+  public TestResults(String moduleName, String displayUri) {
+    this.moduleName = moduleName;
     this.displayUri = displayUri;
   }
 
-  public String getModuleName() {
-    return module;
-  }
-
-  public String getDisplayUri() {
-    return displayUri;
-  }
-
-  public List<TestResult> getResults() {
-    return Collections.unmodifiableList(results);
-  }
-
-  public TestResult newResult(String name) {
-    var result = new TestResult(name);
-    results.add(result);
-    return result;
-  }
-
-  public void newResult(String name, Failure failure) {
-    var result = new TestResult(name);
-    result.addFailure(failure);
-    results.add(result);
-  }
-
   public int totalTests() {
-    return results.size();
+    return module.totalTests() + facts.totalTests() + examples.totalTests();
   }
 
   public int totalFailures() {
-    int total = 0;
-    for (var res : results) {
-      total += res.getFailures().size();
-    }
-    return total;
+    return module.totalFailures() + facts.totalFailures() + examples.totalFailures();
+  }
+
+  public int totalAsserts() {
+    return module.totalAsserts() + facts.totalAsserts() + examples.totalAsserts();
+  }
+
+  public int totalAssertsFailed() {
+    return module.totalAssertsFailed() + facts.totalAssertsFailed() + examples.totalAssertsFailed();
   }
 
   public boolean failed() {
-    for (var res : results) {
-      if (res.isFailure()) return true;
-    }
-    return false;
+    return module.failed() || facts.failed() || examples.failed();
   }
 
   public String getErr() {
@@ -85,147 +64,295 @@ public final class TestResults {
     this.err = err;
   }
 
-  public static class TestResult {
+  public static class TestSectionResults {
+    public final TestSection name;
+    private final List<TestResult> results = new ArrayList<>();
+    private Error error;
 
-    private final String name;
-    private final List<Failure> failures = new ArrayList<>();
-    private final List<Error> errors = new ArrayList<>();
-    private boolean isExampleWritten = false;
-
-    public TestResult(String name) {
+    public TestSectionResults(TestSection name) {
       this.name = name;
     }
 
-    public boolean isSuccess() {
-      return failures.isEmpty() && errors.isEmpty();
+    public void setError(Error error) {
+      this.error = error;
     }
 
-    boolean isFailure() {
-      return !isSuccess();
+    public Error getError() {
+      return error;
     }
 
-    public String getName() {
-      return name;
+    public boolean hasError() {
+      return error != null;
     }
 
-    public boolean isExampleWritten() {
-      return isExampleWritten;
+    public List<TestResult> getResults() {
+      return Collections.unmodifiableList(results);
     }
 
-    public void setExampleWritten(boolean exampleWritten) {
-      isExampleWritten = exampleWritten;
+    public TestResult newResult(String name) {
+      var result = new TestResult(name, this.name == TestSection.EXAMPLES);
+      results.add(result);
+      return result;
     }
 
-    public List<Failure> getFailures() {
-      return Collections.unmodifiableList(failures);
+    public TestResult newResult(String name, Failure failure) {
+      var result = new TestResult(name, this.name == TestSection.EXAMPLES);
+      result.addFailure(failure);
+      results.add(result);
+      return result;
     }
 
-    public void addFailure(Failure description) {
-      failures.add(description);
+    public TestResult newResult(String name, Error error) {
+      var result = new TestResult(name, this.name == TestSection.EXAMPLES);
+      result.addError(error);
+      results.add(result);
+      return result;
     }
 
-    public List<Error> getErrors() {
-      return Collections.unmodifiableList(errors);
+    public int totalTests() {
+      var total = results.size();
+      return (hasError() ? ++total : total);
     }
 
-    public void addError(Error err) {
-      errors.add(err);
-    }
-  }
-
-  public static class Failure {
-
-    private final String kind;
-    private final String rendered;
-
-    private Failure(String kind, String rendered) {
-      this.kind = kind;
-      this.rendered = rendered;
-    }
-
-    public String getKind() {
-      return kind;
-    }
-
-    public String getRendered() {
-      return rendered;
-    }
-
-    public static Failure buildFactFailure(SourceSection sourceSection, String description) {
-      return new Failure(
-          "Fact Failure", sourceSection.getCharacters() + " ❌ (" + description + ")");
-    }
-
-    public static Failure buildExampleLengthMismatchFailure(
-        String location, String property, int expectedLength, int actualLength) {
-      String builder =
-          "("
-              + location
-              + ")\n"
-              + "Output mismatch: Expected \""
-              + property
-              + "\" to contain "
-              + expectedLength
-              + " examples, but found "
-              + actualLength;
-      return new Failure("Output Mismatch (Length)", builder);
-    }
-
-    public static Failure buildExamplePropertyMismatchFailure(
-        String location, String property, boolean isMissingInExpected) {
-      var builder = new StringBuilder();
-      builder
-          .append("(")
-          .append(location)
-          .append(")\n")
-          .append("Output mismatch: \"")
-          .append(property);
-      if (isMissingInExpected) {
-        builder.append("\" exists in actual but not in expected output");
-      } else {
-        builder.append("\" exists in expected but not in actual output");
+    public int totalAsserts() {
+      int total = 0;
+      for (var res : results) {
+        total += res.totalAsserts();
       }
-      return new Failure("Output Mismatch", builder.toString());
+      return (hasError() ? ++total : total);
     }
 
-    public static Failure buildExampleFailure(
-        String location,
-        String expectedLocation,
-        String expectedValue,
-        String actualLocation,
-        String actualValue) {
-      String builder =
-          "("
-              + location
-              + ")\n"
-              + "Expected: ("
-              + expectedLocation
-              + ")\n"
-              + expectedValue
-              + "\nActual: ("
-              + actualLocation
-              + ")\n"
-              + actualValue;
-      return new Failure("Example Failure", builder);
-    }
-  }
-
-  public static class Error {
-
-    private final String message;
-    private final PklException exception;
-
-    public Error(String message, PklException exception) {
-      this.message = message;
-      this.exception = exception;
+    public int totalAssertsFailed() {
+      int total = 0;
+      for (var res : results) {
+        total += res.totalAssertsFailed();
+      }
+      return (hasError() ? ++total : total);
     }
 
-    public String getMessage() {
-      return message;
+    public int totalFailures() {
+      int total = 0;
+      for (var res : results) {
+        if (res.isFailure()) total++;
+      }
+      return (hasError() ? ++total : total);
     }
 
-    public Exception getException() {
-      return exception;
+    public boolean failed() {
+      if (hasError()) return true;
+
+      for (var res : results) {
+        if (res.isFailure()) return true;
+      }
+      return false;
+    }
+
+    public static class TestResult {
+      public final String name;
+      private int totalAsserts = 0;
+      private int totalAssertsFailed = 0;
+      private final List<Failure> failures = new ArrayList<>();
+      private final List<Error> errors = new ArrayList<>();
+      public final boolean isExample;
+      private boolean isExampleWritten = false;
+
+      public TestResult(String name, boolean isExample) {
+        this.name = name;
+        this.isExample = isExample;
+      }
+
+      public boolean isSuccess() {
+        return failures.isEmpty() && errors.isEmpty();
+      }
+
+      public boolean isFailure() {
+        return !isSuccess();
+      }
+
+      public boolean isExampleWritten() {
+        return isExampleWritten;
+      }
+
+      public void setExampleWritten(boolean exampleWritten) {
+        isExampleWritten = exampleWritten;
+      }
+
+      public List<Failure> getFailures() {
+        return Collections.unmodifiableList(failures);
+      }
+
+      public void addFailure(Failure description) {
+        failures.add(description);
+        totalAssertsFailed++;
+      }
+
+      public List<Error> getErrors() {
+        return Collections.unmodifiableList(errors);
+      }
+
+      public void addError(Error err) {
+        errors.add(err);
+        totalAssertsFailed++;
+      }
+
+      public int totalAsserts() {
+        return totalAsserts;
+      }
+
+      public void countAssert() {
+        totalAsserts++;
+      }
+
+      public int totalAssertsFailed() {
+        return totalAssertsFailed;
+      }
+    }
+
+    public static class Failure {
+
+      private final String kind;
+      private final String failure;
+      private final String location;
+
+      private Failure(String kind, String failure, String location) {
+        this.kind = kind;
+        this.failure = failure;
+        this.location = location;
+      }
+
+      public String getKind() {
+        return kind;
+      }
+
+      public String getFailure() {
+        return failure;
+      }
+
+      public String getLocation() {
+        return location;
+      }
+
+      public static String renderLocation(String location) {
+        return "(" + location + ")";
+      }
+
+      public String getRendered() {
+        String rendered;
+
+        if (kind == "Fact Failure") {
+          rendered = failure + " " + renderLocation(getLocation());
+        } else {
+          rendered = renderLocation(getLocation()) + "\n" + failure;
+        }
+
+        return rendered;
+      }
+
+      public static Failure buildFactFailure(SourceSection sourceSection, String location) {
+        return new Failure("Fact Failure", sourceSection.getCharacters().toString(), location);
+      }
+
+      public static Failure buildExampleLengthMismatchFailure(
+          String location, String property, int expectedLength, int actualLength) {
+        var builder = new StringBuilder();
+        builder
+            .append("Output mismatch: Expected \"")
+            .append(property)
+            .append("\" to contain ")
+            .append(expectedLength)
+            .append(" examples, but found ")
+            .append(actualLength);
+
+        return new Failure("Output Mismatch (Length)", builder.toString(), location);
+      }
+
+      public static Failure buildExamplePropertyMismatchFailure(
+          String location, String property, boolean isMissingInExpected) {
+
+        String exists_in;
+        String missing_in;
+
+        if (isMissingInExpected) {
+          exists_in = "actual";
+          missing_in = "expected";
+        } else {
+          exists_in = "expected";
+          missing_in = "actual";
+        }
+
+        var builder = new StringBuilder();
+        builder
+            .append("Output mismatch: \"")
+            .append(property)
+            .append("\" exists in ")
+            .append(exists_in)
+            .append(" but not in ")
+            .append(missing_in)
+            .append(" output");
+
+        return new Failure("Output Mismatch", builder.toString(), location);
+      }
+
+      public static Failure buildExampleFailure(
+          String location,
+          String expectedLocation,
+          String expectedValue,
+          String actualLocation,
+          String actualValue) {
+        var builder = new StringBuilder();
+        builder
+            .append("Expected: ")
+            .append(renderLocation(expectedLocation))
+            .append("\n")
+            .append(expectedValue)
+            .append("\n")
+            .append("Actual: ")
+            .append(renderLocation(actualLocation))
+            .append("\n")
+            .append(actualValue);
+
+        return new Failure("Example Failure", builder.toString(), location);
+      }
+    }
+
+    public static class Error {
+
+      private final String message;
+      private final PklException exception;
+
+      public Error(String message, PklException exception) {
+        this.message = message;
+        this.exception = exception;
+      }
+
+      public String getMessage() {
+        return message;
+      }
+
+      public Exception getException() {
+        return exception;
+      }
+
+      public String getRendered() {
+        return exception.getMessage();
+      }
+    }
+
+    public enum TestSection {
+      MODULE("module"),
+      FACTS("facts"),
+      EXAMPLES("examples");
+
+      private final String name;
+
+      TestSection(final String name) {
+        this.name = name;
+      }
+
+      @Override
+      public String toString() {
+        return name;
+      }
     }
   }
 }

pkl-core/src/main/java/org/pkl/core/runtime/TestRunner.java

@@ -23,8 +23,9 @@ import org.pkl.core.BufferedLogger;
 import org.pkl.core.StackFrameTransformer;
 import org.pkl.core.ast.member.ObjectMember;
 import org.pkl.core.module.ModuleKeys;
-import org.pkl.core.runtime.TestResults.Error;
-import org.pkl.core.runtime.TestResults.Failure;
+import org.pkl.core.runtime.TestResults.TestSectionResults;
+import org.pkl.core.runtime.TestResults.TestSectionResults.Error;
+import org.pkl.core.runtime.TestResults.TestSectionResults.Failure;
 import org.pkl.core.stdlib.PklConverter;
 import org.pkl.core.stdlib.base.PcfRenderer;
 import org.pkl.core.util.EconomicMaps;
@@ -51,12 +52,25 @@ public final class TestRunner {
 
     try {
       checkAmendsPklTest(testModule);
-      runFacts(testModule, results);
-      runExamples(testModule, info, results);
     } catch (VmException v) {
-      var meta = results.newResult(info.getModuleName());
-      meta.addError(new Error(v.getMessage(), v.toPklException(stackFrameTransformer)));
+      var error = new Error(v.getMessage(), v.toPklException(stackFrameTransformer));
+      results.module.setError(error);
     }
+
+    try {
+      runFacts(testModule, results.facts);
+    } catch (VmException v) {
+      var error = new Error(v.getMessage(), v.toPklException(stackFrameTransformer));
+      results.facts.setError(error);
+    }
+
+    try {
+      runExamples(testModule, info, results.examples);
+    } catch (VmException v) {
+      var error = new Error(v.getMessage(), v.toPklException(stackFrameTransformer));
+      results.examples.setError(error);
+    }
+
     results.setErr(logger.getLogs());
     return results;
   }
@@ -72,7 +86,7 @@ public final class TestRunner {
     }
   }
 
-  private void runFacts(VmTyped testModule, TestResults results) {
+  private void runFacts(VmTyped testModule, TestSectionResults results) {
     var facts = VmUtils.readMember(testModule, Identifier.FACTS);
     if (facts instanceof VmNull) return;
 
@@ -86,6 +100,9 @@ public final class TestRunner {
                 if (member.isLocalOrExternalOrHidden()) {
                   return true;
                 }
+
+                result.countAssert();
+
                 try {
                   var factValue = VmUtils.readMember(listing, idx);
                   if (factValue == Boolean.FALSE) {
@@ -101,7 +118,7 @@ public final class TestRunner {
         });
   }
 
-  private void runExamples(VmTyped testModule, ModuleInfo info, TestResults results) {
+  private void runExamples(VmTyped testModule, ModuleInfo info, TestSectionResults results) {
     var examples = VmUtils.readMember(testModule, Identifier.EXAMPLES);
     if (examples instanceof VmNull) return;
 
@@ -144,7 +161,10 @@ public final class TestRunner {
   }
 
   private void doRunAndValidateExamples(
-      VmMapping examples, Path expectedOutputFile, Path actualOutputFile, TestResults results) {
+      VmMapping examples,
+      Path expectedOutputFile,
+      Path actualOutputFile,
+      TestSectionResults results) {
     var expectedExampleOutputs = loadExampleOutputs(expectedOutputFile);
     var actualExampleOutputs = new MutableReference<VmDynamic>(null);
     var allGroupsSucceeded = new MutableBoolean(true);
@@ -155,23 +175,27 @@ public final class TestRunner {
           var group = (VmListing) groupValue;
           var expectedGroup =
               (VmDynamic) VmUtils.readMemberOrNull(expectedExampleOutputs, groupKey);
-          var result = results.newResult(testName);
 
           if (expectedGroup == null) {
-            results.newResult(
-                testName,
-                Failure.buildExamplePropertyMismatchFailure(
-                    getDisplayUri(groupMember), String.valueOf(groupKey), true));
+            results
+                .newResult(
+                    testName,
+                    Failure.buildExamplePropertyMismatchFailure(
+                        getDisplayUri(groupMember), testName, true))
+                .countAssert();
             return true;
           }
 
           if (group.getLength() != expectedGroup.getLength()) {
-            result.addFailure(
-                Failure.buildExampleLengthMismatchFailure(
-                    getDisplayUri(groupMember),
-                    String.valueOf(groupKey),
-                    expectedGroup.getLength(),
-                    group.getLength()));
+            results
+                .newResult(
+                    testName,
+                    Failure.buildExampleLengthMismatchFailure(
+                        getDisplayUri(groupMember),
+                        testName,
+                        expectedGroup.getLength(),
+                        group.getLength()))
+                .countAssert();
             return true;
           }
 
@@ -181,13 +205,20 @@ public final class TestRunner {
                 if (exampleMember.isLocalOrExternalOrHidden()) {
                   return true;
                 }
+
+                var exampleName =
+                    group.getLength() == 1 ? testName : testName + " #" + exampleIndex;
+
                 Object exampleValue;
                 try {
                   exampleValue = VmUtils.readMember(group, exampleIndex);
                 } catch (VmException err) {
                   errored.set(true);
-                  result.addError(
-                      new Error(err.getMessage(), err.toPklException(stackFrameTransformer)));
+                  results
+                      .newResult(
+                          exampleName,
+                          new Error(err.getMessage(), err.toPklException(stackFrameTransformer)))
+                      .countAssert();
                   groupSucceeded.set(false);
                   return true;
                 }
@@ -222,13 +253,18 @@ public final class TestRunner {
                         .build();
                   }
 
-                  result.addFailure(
-                      Failure.buildExampleFailure(
-                          getDisplayUri(exampleMember),
-                          getDisplayUri(expectedMember),
-                          expectedValuePcf,
-                          getDisplayUri(actualMember),
-                          exampleValuePcf));
+                  results
+                      .newResult(
+                          exampleName,
+                          Failure.buildExampleFailure(
+                              getDisplayUri(exampleMember),
+                              getDisplayUri(expectedMember),
+                              expectedValuePcf,
+                              getDisplayUri(actualMember),
+                              exampleValuePcf))
+                      .countAssert();
+                } else {
+                  results.newResult(exampleName).countAssert();
                 }
 
                 return true;
@@ -247,12 +283,14 @@ public final class TestRunner {
             return true;
           }
           if (examples.getCachedValue(groupKey) == null) {
+            var testName = String.valueOf(groupKey);
             allGroupsSucceeded.set(false);
             results
-                .newResult(String.valueOf(groupKey))
-                .addFailure(
+                .newResult(
+                    testName,
                     Failure.buildExamplePropertyMismatchFailure(
-                        getDisplayUri(groupMember), String.valueOf(groupKey), false));
+                        getDisplayUri(groupMember), testName, false))
+                .countAssert();
           }
           return true;
         });
@@ -262,10 +300,12 @@ public final class TestRunner {
     }
   }
 
-  private void doRunAndWriteExamples(VmMapping examples, Path outputFile, TestResults results) {
+  private void doRunAndWriteExamples(
+      VmMapping examples, Path outputFile, TestSectionResults results) {
     var allSucceeded =
         examples.forceAndIterateMemberValues(
             (groupKey, groupMember, groupValue) -> {
+              var testName = String.valueOf(groupKey);
               var listing = (VmListing) groupValue;
               var success =
                   listing.iterateMembers(
@@ -273,22 +313,29 @@ public final class TestRunner {
                         if (member.isLocalOrExternalOrHidden()) {
                           return true;
                         }
+
+                        var exampleName =
+                            listing.getLength() == 1 ? testName : testName + " #" + idx;
+
                         try {
                           VmUtils.readMember(listing, idx);
                           return true;
                         } catch (VmException err) {
                           results
-                              .newResult(String.valueOf(groupKey))
-                              .addError(
+                              .newResult(
+                                  exampleName,
                                   new Error(
-                                      err.getMessage(), err.toPklException(stackFrameTransformer)));
+                                      err.getMessage(), err.toPklException(stackFrameTransformer)))
+                              .countAssert();
                           return false;
                         }
                       });
               if (!success) {
                 return false;
               }
-              results.newResult(String.valueOf(groupKey)).setExampleWritten(true);
+              var result = results.newResult(testName);
+              result.countAssert();
+              result.setExampleWritten(true);
               return true;
             });
     if (allSucceeded) {

pkl-core/src/main/java/org/pkl/core/stdlib/test/report/JUnitReport.java

@@ -24,7 +24,9 @@ import org.pkl.core.ast.member.ObjectMember;
 import org.pkl.core.runtime.BaseModule;
 import org.pkl.core.runtime.Identifier;
 import org.pkl.core.runtime.TestResults;
-import org.pkl.core.runtime.TestResults.TestResult;
+import org.pkl.core.runtime.TestResults.TestSectionResults;
+import org.pkl.core.runtime.TestResults.TestSectionResults.Error;
+import org.pkl.core.runtime.TestResults.TestSectionResults.TestResult;
 import org.pkl.core.runtime.VmDynamic;
 import org.pkl.core.runtime.VmMapping;
 import org.pkl.core.runtime.VmTyped;
@@ -41,25 +43,45 @@ public final class JUnitReport implements TestReport {
     writer.append(renderXML("    ", "1.0", buildSuite(results)));
   }
 
-  private VmDynamic buildSuite(TestResults res) {
-    var testCases = testCases(res);
-    if (!res.getErr().isBlank()) {
+  private VmDynamic buildSuite(TestResults results) {
+    var testCases = testCases(results.moduleName, results.facts);
+    testCases.addAll(testCases(results.moduleName, results.examples));
+
+    if (!results.getErr().isBlank()) {
       var err =
           buildXmlElement(
               "system-err",
               VmMapping.empty(),
-              members -> members.put("body", syntheticElement(makeCdata(res.getErr()))));
+              members -> members.put("body", syntheticElement(makeCdata(results.getErr()))));
       testCases.add(err);
     }
-    return buildXmlElement(
-        "testsuite", buildRootAttributes(res), testCases.toArray(new VmDynamic[0]));
+
+    var attrs =
+        buildAttributes(
+            "name", results.moduleName,
+            "tests", (long) results.totalTests(),
+            "failures", (long) results.totalFailures());
+
+    return buildXmlElement("testsuite", attrs, testCases.toArray(new VmDynamic[0]));
   }
 
-  private ArrayList<VmDynamic> testCases(TestResults results) {
-    var className = results.getModuleName();
-    var elements = new ArrayList<VmDynamic>(results.totalTests());
-    for (var res : results.getResults()) {
-      var attrs = buildAttributes("classname", className, "name", res.getName());
+  private ArrayList<VmDynamic> testCases(String moduleName, TestSectionResults testSectionResults) {
+    var elements = new ArrayList<VmDynamic>(testSectionResults.totalTests());
+
+    if (testSectionResults.hasError()) {
+      var error = error(testSectionResults.getError());
+
+      var attrs =
+          buildAttributes("classname", moduleName + "." + testSectionResults.name, "name", "error");
+      var element = buildXmlElement("testcase", attrs, error.toArray(new VmDynamic[0]));
+
+      elements.add(element);
+    }
+
+    for (var res : testSectionResults.getResults()) {
+      var attrs =
+          buildAttributes(
+              "classname", moduleName + "." + testSectionResults.name, "name", res.name);
       var failures = failures(res);
       failures.addAll(errors(res));
       var element = buildXmlElement("testcase", attrs, failures.toArray(new VmDynamic[0]));
@@ -99,6 +121,17 @@ public final class JUnitReport implements TestReport {
     return list;
   }
 
+  private ArrayList<VmDynamic> error(Error error) {
+    var list = new ArrayList<VmDynamic>();
+    var attrs = buildAttributes("message", error.getMessage());
+    list.add(
+        buildXmlElement(
+            "error",
+            attrs,
+            members -> members.put(1, syntheticElement("\n" + error.getRendered()))));
+    return list;
+  }
+
   private VmDynamic buildXmlElement(String name, VmMapping attributes, VmDynamic... elements) {
     return buildXmlElement(
         name,
@@ -130,16 +163,6 @@ public final class JUnitReport implements TestReport {
         members.size() - 4);
   }
 
-  private VmMapping buildRootAttributes(TestResults results) {
-    return buildAttributes(
-        "name",
-        results.getModuleName(),
-        "tests",
-        (long) results.totalTests(),
-        "failures",
-        (long) results.totalFailures());
-  }
-
   private VmMapping buildAttributes(Object... attributes) {
     EconomicMap<Object, ObjectMember> attrs = EconomicMaps.create(attributes.length);
     for (int i = 0; i < attributes.length; i += 2) {

pkl-core/src/main/java/org/pkl/core/stdlib/test/report/SimpleReport.java

@@ -19,8 +19,8 @@ import java.io.IOException;
 import java.io.Writer;
 import java.util.stream.Collectors;
 import org.pkl.core.runtime.TestResults;
-import org.pkl.core.runtime.TestResults.Failure;
-import org.pkl.core.runtime.TestResults.TestResult;
+import org.pkl.core.runtime.TestResults.TestSectionResults;
+import org.pkl.core.runtime.TestResults.TestSectionResults.TestResult;
 import org.pkl.core.util.StringUtils;
 
 public final class SimpleReport implements TestReport {
@@ -28,38 +28,66 @@ public final class SimpleReport implements TestReport {
   @Override
   public void report(TestResults results, Writer writer) throws IOException {
     var builder = new StringBuilder();
-    builder.append("module ");
-    builder.append(results.getModuleName());
-    builder.append(" (").append(results.getDisplayUri()).append(")\n");
-    StringUtils.joinToStringBuilder(
-        builder, results.getResults(), "\n", res -> reportResult(res, builder));
+
+    builder.append("module ").append(results.moduleName).append("\n");
+
+    reportResults(results.facts, builder);
+    reportResults(results.examples, builder);
+
+    builder.append(results.failed() ? "❌ " : "✅ ");
+
+    var totalStatsLine =
+        makeStatsLine("tests", results.totalTests(), results.totalFailures(), results.failed());
+    builder.append(totalStatsLine);
+
+    var totalAssertsStatsLine =
+        makeStatsLine(
+            "asserts", results.totalAsserts(), results.totalAssertsFailed(), results.failed());
+    builder.append(", ").append(totalAssertsStatsLine);
+
     builder.append("\n");
+
     writer.append(builder);
   }
 
-  private void reportResult(TestResult result, StringBuilder builder) {
-    builder.append("  ").append(result.getName());
-    if (result.isExampleWritten()) {
-      builder.append(" ✍️");
-    } else if (result.isSuccess()) {
-      builder.append(" ✅");
-    } else {
-      builder.append(" ❌\n");
+  private void reportResults(TestSectionResults section, StringBuilder builder) {
+    if (!section.getResults().isEmpty()) {
+      builder.append("  ").append(section.name).append("\n");
+
       StringUtils.joinToStringBuilder(
-          builder, result.getFailures(), "\n", failure -> reportFailure(failure, builder));
-      StringUtils.joinToStringBuilder(
-          builder,
-          result.getErrors(),
-          "\n",
-          error -> {
-            builder.append("    Error:\n");
-            appendPadded(builder, error.getException().getMessage(), "        ");
-          });
+          builder, section.getResults(), "\n", res -> reportResult(res, builder));
+      builder.append("\n");
+    } else if (section.hasError()) {
+      builder.append("  ").append(section.name).append("\n");
+      var error = section.getError().getRendered();
+      appendPadded(builder, error, "    ");
+      builder.append("\n");
     }
   }
 
-  public static void reportFailure(Failure failure, StringBuilder builder) {
-    appendPadded(builder, failure.getRendered(), "    ");
+  private void reportResult(TestResult result, StringBuilder builder) {
+    builder.append("    ");
+
+    if (result.isExampleWritten()) {
+      builder.append(result.name).append(" ✍️");
+    } else {
+      builder.append(result.isFailure() ? "❌ " : "✅ ").append(result.name);
+
+      if (result.isFailure()) {
+        var failurePadding = "       ";
+        builder.append("\n");
+        StringUtils.joinToStringBuilder(
+            builder,
+            result.getFailures(),
+            "\n",
+            failure -> appendPadded(builder, failure.getRendered(), failurePadding));
+        StringUtils.joinToStringBuilder(
+            builder,
+            result.getErrors(),
+            "\n",
+            error -> appendPadded(builder, error.getException().getMessage(), failurePadding));
+      }
+    }
   }
 
   private static void appendPadded(StringBuilder builder, String lines, String padding) {
@@ -67,6 +95,23 @@ public final class SimpleReport implements TestReport {
         builder,
         lines.lines().collect(Collectors.toList()),
         "\n",
-        str -> builder.append(padding).append(str));
+        str -> {
+          if (str.length() > 0) builder.append(padding).append(str);
+        });
+  }
+
+  private String makeStatsLine(String kind, int total, int failed, boolean isFailed) {
+    var passed = total - failed;
+    var pct_passed = total > 0 ? 100.0 * passed / total : 0.0;
+
+    String line = String.format("%.1f%% %s pass", pct_passed, kind);
+
+    if (isFailed) {
+      line += String.format(" [%d/%d failed]", failed, total);
+    } else {
+      line += String.format(" [%d passed]", passed);
+    }
+
+    return line;
   }
 }

pkl-core/src/test/kotlin/org/pkl/core/EvaluateTestsTest.kt

@@ -54,7 +54,7 @@ class EvaluateTestsTest {
     assertThat(results.displayUri).isEqualTo("repl:text")
     assertThat(results.totalTests()).isEqualTo(1)
     assertThat(results.failed()).isFalse
-    assertThat(results.results[0].name).isEqualTo("should pass")
+    assertThat(results.facts.results[0].name).isEqualTo("should pass")
     assertThat(results.err.isBlank()).isTrue
   }
 
@@ -79,18 +79,19 @@ class EvaluateTestsTest {
       )
 
     assertThat(results.totalTests()).isEqualTo(1)
-    assertThat(results.totalFailures()).isEqualTo(2)
+    assertThat(results.totalFailures()).isEqualTo(1)
     assertThat(results.failed()).isTrue
 
-    val res = results.results[0]
+    val res = results.facts.results[0]
     assertThat(res.name).isEqualTo("should fail")
-    assertThat(res.errors).isEmpty()
+    assertThat(results.facts.hasError()).isFalse
+    assertThat(res.failures.size).isEqualTo(2)
 
     val fail1 = res.failures[0]
-    assertThat(fail1.rendered).isEqualTo("1 == 2 ❌ (repl:text)")
+    assertThat(fail1.rendered).isEqualTo("1 == 2 (repl:text)")
 
     val fail2 = res.failures[1]
-    assertThat(fail2.rendered).isEqualTo(""""foo" == "bar" ❌ (repl:text)""")
+    assertThat(fail2.rendered).isEqualTo(""""foo" == "bar" (repl:text)""")
   }
 
   @Test
@@ -117,7 +118,7 @@ class EvaluateTestsTest {
     assertThat(results.totalFailures()).isEqualTo(1)
     assertThat(results.failed()).isTrue
 
-    val res = results.results[0]
+    val res = results.facts.results[0]
     assertThat(res.name).isEqualTo("should fail")
     assertThat(res.failures).hasSize(1)
     assertThat(res.errors).hasSize(1)
@@ -179,7 +180,121 @@ class EvaluateTestsTest {
     assertThat(results.displayUri).startsWith("file:///").endsWith(".pkl")
     assertThat(results.totalTests()).isEqualTo(1)
     assertThat(results.failed()).isFalse
-    assertThat(results.results[0].name).isEqualTo("user")
+    assertThat(results.examples.results[0].name).isEqualTo("user")
+  }
+
+  @Test
+  fun `test fact failures with successful example`(@TempDir tempDir: Path) {
+    val file = tempDir.createTempFile(prefix = "example", suffix = ".pkl")
+    Files.writeString(
+      file,
+      """
+      amends "pkl:test"
+
+      facts {
+        ["should fail"] {
+          1 == 2
+          "foo" == "bar"
+        }
+      }
+
+      examples {
+        ["user"] {
+          new {
+            name = "Bob"
+            age = 33
+          }
+        }
+      }
+    """
+        .trimIndent()
+    )
+
+    Files.writeString(
+      createExpected(file),
+      """
+      examples {
+        ["user"] {
+          new {
+            name = "Bob"
+            age = 33
+          }
+        }
+      }
+    """
+        .trimIndent()
+    )
+
+    val results = evaluator.evaluateTest(path(file), false)
+    assertThat(results.moduleName).startsWith("example")
+    assertThat(results.displayUri).startsWith("file:///").endsWith(".pkl")
+    assertThat(results.totalTests()).isEqualTo(2)
+    assertThat(results.totalFailures()).isEqualTo(1)
+    assertThat(results.failed()).isTrue
+
+    assertThat(results.facts.results[0].name).isEqualTo("should fail")
+    assertThat(results.facts.results[0].failures.size).isEqualTo(2)
+    assertThat(results.examples.results[0].name).isEqualTo("user")
+  }
+
+  @Test
+  fun `test fact error with successful example`(@TempDir tempDir: Path) {
+    val file = tempDir.createTempFile(prefix = "example", suffix = ".pkl")
+    Files.writeString(
+      file,
+      """
+      amends "pkl:test"
+
+      facts {
+        ["should fail"] {
+          throw("exception")
+        }
+      }
+
+      examples {
+        ["user"] {
+          new {
+            name = "Bob"
+            age = 33
+          }
+        }
+      }
+    """
+        .trimIndent()
+    )
+
+    Files.writeString(
+      createExpected(file),
+      """
+      examples {
+        ["user"] {
+          new {
+            name = "Bob"
+            age = 33
+          }
+        }
+      }
+    """
+        .trimIndent()
+    )
+
+    val results = evaluator.evaluateTest(path(file), false)
+    assertThat(results.moduleName).startsWith("example")
+    assertThat(results.displayUri).startsWith("file:///").endsWith(".pkl")
+
+    assertThat(results.totalTests()).isEqualTo(2)
+    assertThat(results.totalFailures()).isEqualTo(1)
+    assertThat(results.failed()).isTrue
+
+    val res = results.facts.results[0]
+    assertThat(res.name).isEqualTo("should fail")
+    assertThat(res.failures).hasSize(0)
+    assertThat(res.errors).hasSize(1)
+
+    val error = res.errors[0]
+    assertThat(error.message).isEqualTo("exception")
+
+    assertThat(results.examples.results[0].name).isEqualTo("user")
   }
 
   @Test
@@ -224,9 +339,9 @@ class EvaluateTestsTest {
     assertThat(results.failed()).isTrue
     assertThat(results.totalFailures()).isEqualTo(1)
 
-    val res = results.results[0]
+    val res = results.examples.results[0]
     assertThat(res.name).isEqualTo("user")
-    assertThat(res.errors.isEmpty()).isTrue
+    assertFalse(results.examples.hasError())
 
     val fail1 = res.failures[0]
     assertThat(fail1.rendered.stripFileAndLines(tempDir))