Introduce "minimal" test reporter (#1563)

pkl-core/src/main/java/org/pkl/core/stdlib/test/report/BaseReporter.java

@@ -1,5 +1,5 @@
 /*
- * Copyright © 2024-2025 Apple Inc. and the Pkl project authors. All rights reserved.
+ * Copyright © 2026 Apple Inc. and the Pkl project authors. All rights reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -24,41 +24,21 @@ import java.util.Locale;
 import java.util.stream.Collectors;
 import org.pkl.core.TestResults;
 import org.pkl.core.TestResults.TestResult;
-import org.pkl.core.TestResults.TestSectionResults;
 import org.pkl.core.util.AnsiStringBuilder;
 import org.pkl.core.util.AnsiStringBuilder.AnsiCode;
 import org.pkl.core.util.AnsiTheme;
 import org.pkl.core.util.StringUtils;
 
-public final class SimpleReport implements TestReport {
+public abstract class BaseReporter implements TestReporter {
+  protected static final String passingMark = "✔ ";
+  protected static final String failingMark = "✘ ";
 
-  private static final String passingMark = "✔ ";
-  private static final String failingMark = "✘ ";
+  protected final boolean useColor;
 
-  private final boolean useColor;
-
-  public SimpleReport(boolean useColor) {
+  public BaseReporter(boolean useColor) {
     this.useColor = useColor;
   }
 
-  @Override
-  public void report(TestResults results, Writer writer) throws IOException {
-    var builder = new AnsiStringBuilder(useColor);
-
-    builder.append("module ").append(results.moduleName()).append('\n');
-
-    if (results.error() != null) {
-      var rendered = results.error().exception().getMessage();
-      appendPadded(builder, rendered, "  ");
-      builder.append('\n');
-    } else {
-      reportResults(results.facts(), builder);
-      reportResults(results.examples(), builder);
-    }
-
-    writer.append(builder.toString());
-  }
-
   @Override
   public void summarize(List<TestResults> allTestResults, Writer writer) throws IOException {
     var totalTests = 0;
@@ -91,16 +71,7 @@ public final class SimpleReport implements TestReport {
     writer.append(builder.toString());
   }
 
-  private void reportResults(TestSectionResults section, AnsiStringBuilder builder) {
-    if (!section.results().isEmpty()) {
-      builder.append("  ").append(section.name()).append('\n');
-      StringUtils.joinToStringBuilder(
-          builder, section.results(), "\n", res -> reportResult(res, builder));
-      builder.append('\n');
-    }
-  }
-
-  private void reportResult(TestResult result, AnsiStringBuilder builder) {
+  protected void reportResult(TestResult result, AnsiStringBuilder builder) {
     builder.append("    ");
 
     if (result.isExampleWritten()) {
@@ -129,7 +100,7 @@ public final class SimpleReport implements TestReport {
     }
   }
 
-  private static void appendPadded(AnsiStringBuilder builder, String lines, String padding) {
+  protected static void appendPadded(AnsiStringBuilder builder, String lines, String padding) {
     StringUtils.joinToStringBuilder(
         builder,
         lines.lines().collect(Collectors.toList()),

pkl-core/src/main/java/org/pkl/core/stdlib/test/report/JUnitReporter.java

@@ -37,15 +37,15 @@ import org.pkl.core.stdlib.PklConverter;
 import org.pkl.core.stdlib.xml.RendererNodes.Renderer;
 import org.pkl.core.util.EconomicMaps;
 
-public final class JUnitReport implements TestReport {
+public final class JUnitReporter implements TestReporter {
 
   private final String aggregateSuiteName;
 
-  public JUnitReport(String aggregateSuiteName) {
+  public JUnitReporter(String aggregateSuiteName) {
     this.aggregateSuiteName = aggregateSuiteName;
   }
 
-  public JUnitReport() {
+  public JUnitReporter() {
     this("");
   }
 

pkl-core/src/main/java/org/pkl/core/stdlib/test/report/MinimalReporter.java

@@ -0,0 +1,67 @@
+/*
+ * Copyright © 2026 Apple Inc. and the Pkl project authors. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.pkl.core.stdlib.test.report;
+
+import java.io.IOException;
+import java.io.Writer;
+import java.util.List;
+import org.pkl.core.TestResults;
+import org.pkl.core.TestResults.TestResult;
+import org.pkl.core.TestResults.TestSectionResults;
+import org.pkl.core.util.AnsiStringBuilder;
+import org.pkl.core.util.StringUtils;
+
+/** Minimal reporter. Only reports failures and errors. */
+public final class MinimalReporter extends BaseReporter {
+
+  public MinimalReporter(boolean useColor) {
+    super(useColor);
+  }
+
+  @Override
+  public void report(TestResults results, Writer writer) throws IOException {
+    var builder = new AnsiStringBuilder(useColor);
+
+    if (results.error() != null) {
+      builder.append("module ").append(results.moduleName()).append('\n');
+
+      var rendered = results.error().exception().getMessage();
+      appendPadded(builder, rendered, "  ");
+      builder.append('\n');
+    } else {
+      var factFailures = results.facts().results().stream().filter(TestResult::isFailure).toList();
+      var exampleFailures =
+          results.examples().results().stream().filter(TestResult::isFailure).toList();
+      if (!factFailures.isEmpty() || !exampleFailures.isEmpty()) {
+        builder.append("module ").append(results.moduleName()).append('\n');
+
+        reportResults(results.facts(), factFailures, builder);
+        reportResults(results.examples(), exampleFailures, builder);
+      }
+    }
+
+    writer.append(builder.toString());
+  }
+
+  private void reportResults(
+      TestSectionResults section, List<TestResults.TestResult> results, AnsiStringBuilder builder) {
+    if (!results.isEmpty()) {
+      builder.append("  ").append(section.name()).append('\n');
+      StringUtils.joinToStringBuilder(builder, results, "\n", res -> reportResult(res, builder));
+      builder.append('\n');
+    }
+  }
+}

pkl-core/src/main/java/org/pkl/core/stdlib/test/report/SpecReporter.java

@@ -0,0 +1,57 @@
+/*
+ * Copyright © 2024-2026 Apple Inc. and the Pkl project authors. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.pkl.core.stdlib.test.report;
+
+import java.io.IOException;
+import java.io.Writer;
+import org.pkl.core.TestResults;
+import org.pkl.core.TestResults.TestSectionResults;
+import org.pkl.core.util.AnsiStringBuilder;
+import org.pkl.core.util.StringUtils;
+
+public final class SpecReporter extends BaseReporter {
+
+  public SpecReporter(boolean useColor) {
+    super(useColor);
+  }
+
+  @Override
+  public void report(TestResults results, Writer writer) throws IOException {
+    var builder = new AnsiStringBuilder(useColor);
+
+    builder.append("module ").append(results.moduleName()).append('\n');
+
+    if (results.error() != null) {
+      var rendered = results.error().exception().getMessage();
+      appendPadded(builder, rendered, "  ");
+      builder.append('\n');
+    } else {
+      reportResults(results.facts(), builder);
+      reportResults(results.examples(), builder);
+    }
+
+    writer.append(builder.toString());
+  }
+
+  private void reportResults(TestSectionResults section, AnsiStringBuilder builder) {
+    if (!section.results().isEmpty()) {
+      builder.append("  ").append(section.name()).append('\n');
+      StringUtils.joinToStringBuilder(
+          builder, section.results(), "\n", res -> reportResult(res, builder));
+      builder.append('\n');
+    }
+  }
+}

pkl-core/src/main/java/org/pkl/core/stdlib/test/report/TestReporter.java

@@ -1,5 +1,5 @@
 /*
- * Copyright © 2024-2025 Apple Inc. and the Pkl project authors. All rights reserved.
+ * Copyright © 2024-2026 Apple Inc. and the Pkl project authors. All rights reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -25,7 +25,7 @@ import org.pkl.core.PklBugException;
 import org.pkl.core.TestResults;
 import org.pkl.core.util.StringBuilderWriter;
 
-public interface TestReport {
+public interface TestReporter {
 
   void report(TestResults results, Writer writer) throws IOException;
 

pkl-core/src/test/kotlin/org/pkl/core/stdlib/MinimalReportTest.kt

@@ -0,0 +1,158 @@
+/*
+ * Copyright © 2026 Apple Inc. and the Pkl project authors. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.pkl.core.stdlib
+
+import java.io.StringWriter
+import org.assertj.core.api.Assertions.assertThat
+import org.junit.jupiter.api.Test
+import org.pkl.core.TestResults
+import org.pkl.core.TestResults.TestResult
+import org.pkl.core.TestResults.TestSectionResults
+import org.pkl.core.stdlib.test.report.MinimalReporter
+
+class MinimalReportTest {
+
+  @Test
+  fun `report with only passing tests does not show module or test names`() {
+    val resultsBuilder = TestResults.Builder("module1", "module1")
+    resultsBuilder.setFactsSection(
+      TestSectionResults(
+        TestResults.TestSectionName.FACTS,
+        listOf(TestResult("passing fact", 1, emptyList(), emptyList(), false)),
+      )
+    )
+    resultsBuilder.setExamplesSection(
+      TestSectionResults(
+        TestResults.TestSectionName.EXAMPLES,
+        listOf(TestResult("passing example", 1, emptyList(), emptyList(), false)),
+      )
+    )
+    val testResults = resultsBuilder.build()
+
+    val writer = StringWriter()
+    val minimalReport = MinimalReporter(false)
+    minimalReport.report(testResults, writer)
+
+    assertThat(writer.toString()).isEmpty()
+  }
+
+  @Test
+  fun `report with failures shows module name and only failed tests`() {
+    val resultsBuilder = TestResults.Builder("module1", "module1")
+    resultsBuilder.setFactsSection(
+      TestSectionResults(
+        TestResults.TestSectionName.FACTS,
+        listOf(
+          TestResult("passing fact", 1, emptyList(), emptyList(), false),
+          TestResult(
+            "failing fact",
+            1,
+            listOf(TestResults.Failure("Fact Failure", "failed")),
+            emptyList(),
+            false,
+          ),
+        ),
+      )
+    )
+    resultsBuilder.setExamplesSection(
+      TestSectionResults(
+        TestResults.TestSectionName.EXAMPLES,
+        listOf(TestResult("passing example", 1, emptyList(), emptyList(), false)),
+      )
+    )
+    val testResults = resultsBuilder.build()
+
+    val writer = StringWriter()
+    val minimalReport = MinimalReporter(false)
+    minimalReport.report(testResults, writer)
+
+    val output = writer.toString()
+    assertThat(output).contains("module module1")
+    assertThat(output).contains("failing fact")
+    assertThat(output).doesNotContain("passing fact")
+    assertThat(output).doesNotContain("passing example")
+    assertThat(output).doesNotContain("examples")
+  }
+
+  @Test
+  fun `summarize includes stats even when all tests pass`() {
+    val resultsBuilder = TestResults.Builder("module1", "module1")
+    resultsBuilder.setFactsSection(
+      TestSectionResults(
+        TestResults.TestSectionName.FACTS,
+        listOf(TestResult("passing fact", 1, emptyList(), emptyList(), false)),
+      )
+    )
+    resultsBuilder.setExamplesSection(
+      TestSectionResults(TestResults.TestSectionName.EXAMPLES, emptyList())
+    )
+    val testResults = listOf(resultsBuilder.build())
+
+    val writer = StringWriter()
+    val minimalReport = MinimalReporter(false)
+    minimalReport.summarize(testResults, writer)
+
+    val output = writer.toString()
+    assertThat(output).contains("100.0% tests pass")
+    assertThat(output).contains("1 passed")
+  }
+
+  @Test
+  fun `summarize method should generate correct output for failures`() {
+    val resultsBuilder = TestResults.Builder("module1", "module1")
+    resultsBuilder.setFactsSection(
+      TestSectionResults(
+        TestResults.TestSectionName.FACTS,
+        listOf(
+          TestResult(
+            "example1",
+            321919,
+            listOf(TestResults.Failure("Fact Failure", "failed")),
+            emptyList(),
+            false,
+          )
+        ),
+      )
+    )
+    resultsBuilder.setExamplesSection(
+      TestSectionResults(
+        TestResults.TestSectionName.EXAMPLES,
+        listOf(
+          TestResult(
+            "example1",
+            432525,
+            listOf(TestResults.Failure("Output Mismatch", "does not match")),
+            emptyList(),
+            false,
+          )
+        ),
+      )
+    )
+    val testResults = listOf(resultsBuilder.build())
+
+    val writer = StringWriter()
+    val minimalReport = MinimalReporter(false)
+    minimalReport.summarize(testResults, writer)
+
+    val expectedOutput =
+      """
+      0.0% tests pass [2/2 failed], 99.9% asserts pass [2/754444 failed]
+      """
+        .trimIndent()
+
+    assertThat(writer.toString().trimIndent()).isEqualTo(expectedOutput)
+  }
+}

pkl-core/src/test/kotlin/org/pkl/core/stdlib/SimpleReportTest.kt

@@ -16,19 +16,18 @@
 package org.pkl.core.stdlib
 
 import java.io.StringWriter
-import java.util.*
 import org.assertj.core.api.Assertions.assertThat
 import org.junit.jupiter.api.Test
 import org.pkl.core.TestResults
 import org.pkl.core.TestResults.TestResult
 import org.pkl.core.TestResults.TestSectionResults
-import org.pkl.core.stdlib.test.report.SimpleReport
+import org.pkl.core.stdlib.test.report.SpecReporter
 
 class SimpleReportTest {
 
   @Test
   fun `summarize method should generate correct output`() {
-    var resultsBuilder = TestResults.Builder("module1", "module1")
+    val resultsBuilder = TestResults.Builder("module1", "module1")
     resultsBuilder.setFactsSection(
       TestSectionResults(
         TestResults.TestSectionName.FACTS,
@@ -60,7 +59,7 @@ class SimpleReportTest {
     val testResults = listOf(resultsBuilder.build())
 
     val writer = StringWriter()
-    val simpleReport = SimpleReport(false)
+    val simpleReport = SpecReporter(false)
     simpleReport.summarize(testResults, writer)
 
     val expectedOutput =