Aggregate junit report into one file (#1056)

Some systems require junit report to be in a single file. For example `bazel` https://bazel.build/reference/test-encyclopedia needs single file to be available in `XML_OUTPUT_FILE` path.

To avoid implementing junit aggregation in pkl wrappers in different places this PR instead adds a `--junit-aggregate-reports` flag to return all junit reports as a single file.

Additional flag `--junit-aggregate-suite-name` is added to allow overriding global test suite name from default `pkl-tests`

docs/modules/pkl-cli/pages/index.adoc

@@ -465,9 +465,31 @@ Default: (none) +
 Example: `./build/test-results` +
 Directory where to store JUnit reports.
 
+By default, one file will be created for each test module. This behavior can be changed with `--junit-aggregate-reports`, which will instead create a single JUnit report file with all test results.
+
 No JUnit reports will be generated if this option is not present.
 ====
 
+[[junit-aggregate-reports]]
+.--junit-aggregate-reports
+[%collapsible]
+====
+Aggregate JUnit reports into a single file.
+
+By default it will be `pkl-tests.xml` but you can override it using `--junit-aggregate-suite-name` flag.
+====
+
+[[junit-aggregate-suite-name]]
+.--junit-aggregate-suite-name
+[%collapsible]
+====
+Default: (none) +
+Example: `my-tests` +
+The name of the root JUnit test suite.
+
+Used in combination with `--junit-aggregate-reports` flag.
+====
+
 [[overwrite]]
 .--overwrite
 [%collapsible]
@@ -555,6 +577,26 @@ Directory where to store JUnit reports.
 No JUnit reports will be generated if this option is not present.
 ====
 
+[[junit-aggregate-reports]]
+.--junit-aggregate-reports
+[%collapsible]
+====
+Aggregate JUnit reports into a single file.
+
+By default it will be `pkl-tests.xml` but you can override it using `--junit-aggregate-suite-name` flag.
+====
+
+[[junit-aggregate-suite-name]]
+.--junit-aggregate-suite-name
+[%collapsible]
+====
+Default: (none) +
+Example: `my-tests` +
+The name of the root JUnit test suite.
+
+Used in combination with `--junit-aggregate-reports` flag.
+====
+
 .--overwrite
 [%collapsible]
 ====

docs/modules/pkl-gradle/pages/index.adoc

@@ -195,7 +195,7 @@ Example 1: `multipleFileOutputDir = layout.projectDirectory.dir("output")` +
 Example 2: `+multipleFileOutputDir = layout.projectDirectory.file("%{moduleDir}/output")+`
 The directory where a module's output files are placed.
 
-Setting this option causes Pkl to evaluate a module's `output.files` property 
+Setting this option causes Pkl to evaluate a module's `output.files` property
 and write the files specified therein.
 Within `output.files`, a key determines a file's path relative to `multipleFileOutputDir`,
 and a value determines the file's contents.
@@ -207,7 +207,7 @@ This option cannot be used together with any of the following:
 
 This option supports the same placeholders as xref:output-file[outputFile].
 
-For additional details, see xref:language-reference:index.adoc#multiple-file-output[Multiple File Output] 
+For additional details, see xref:language-reference:index.adoc#multiple-file-output[Multiple File Output]
 in the language reference.
 ====
 
@@ -298,6 +298,22 @@ Example: `junitReportsDir = layout.buildDirectory.dir("reports")` +
 Whether and where to generate JUnit XML reports.
 ====
 
+[[junit-aggregate-reports]]
+.junitAggregateReports: Property<Boolean>
+[%collapsible]
+====
+Default: `false` +
+Aggregate JUnit reports into a single file.
+====
+
+[[junit-aggregate-suite-name]]
+.junitAggregateSuiteName: Property<String>
+[%collapsible]
+====
+Default: `null` +
+The name of the root JUnit test suite.
+====
+
 [[overwrite]]
 .overwrite: Property<Boolean>
 [%collapsible]
@@ -391,7 +407,7 @@ For Spring Boot applications, and for users of `pkl-config-java` compiling the g
 Default: `"org.pkl.config.java.mapper.NonNull"` +
 Example: `nonNullAnnotation = "org.project.MyAnnotation"` +
 Fully qualified name of the annotation type to use for annotating non-null types. +
-The specified annotation type must be annotated with `@java.lang.annotation.Target(ElementType.TYPE_USE)` 
+The specified annotation type must be annotated with `@java.lang.annotation.Target(ElementType.TYPE_USE)`
 or the generated code may not compile.
 ====
 
@@ -431,7 +447,7 @@ build.gradle.kts::
 +
 [source,kotlin]
 ----
-pkl { 
+pkl {
   kotlinCodeGenerators {
     register("genKotlin") {
       sourceModules.addAll(files("Template1.pkl", "Template2.pkl"))

pkl-cli/src/main/kotlin/org/pkl/cli/CliTestRunner.kt

@@ -65,6 +65,12 @@ constructor(
       val moduleNames = mutableSetOf<String>()
       val reporter = SimpleReport(useColor)
       val allTestResults = mutableListOf<TestResults>()
+
+      val junitDir = testOptions.junitDir
+      if (junitDir != null) {
+        junitDir.toFile().mkdirs()
+      }
+
       for ((idx, moduleUri) in sources.withIndex()) {
         try {
           val results = evaluator.evaluateTest(uri(moduleUri), testOptions.overwrite)
@@ -78,9 +84,7 @@ constructor(
             consoleWriter.append('\n')
           }
           consoleWriter.flush()
-          val junitDir = testOptions.junitDir
           if (junitDir != null) {
-            junitDir.toFile().mkdirs()
             val moduleName = "${results.moduleName}.xml"
             if (moduleName in moduleNames) {
               throw RuntimeException(
@@ -94,7 +98,10 @@ constructor(
               )
             }
             moduleNames += moduleName
-            JUnitReport().reportToPath(results, junitDir.resolve(moduleName))
+
+            if (!testOptions.junitAggregateReports) {
+              JUnitReport().reportToPath(results, junitDir.resolve(moduleName))
+            }
           }
         } catch (ex: Exception) {
           errWriter.appendLine("Error evaluating module ${moduleUri.path}:")
@@ -106,6 +113,11 @@ constructor(
           failed = true
         }
       }
+      if (testOptions.junitAggregateReports && junitDir != null) {
+        val fileName = "${testOptions.junitAggregateSuiteName}.xml"
+        JUnitReport(testOptions.junitAggregateSuiteName)
+          .summarizeToPath(allTestResults, junitDir.resolve(fileName))
+      }
       consoleWriter.append('\n')
       reporter.summarize(allTestResults, consoleWriter)
       consoleWriter.flush()

pkl-cli/src/test/kotlin/org/pkl/cli/CliTestRunnerTest.kt

@@ -385,6 +385,174 @@ class CliTestRunnerTest {
     assertThatCode { runner.run() }.hasMessageContaining("failed")
   }
 
+  @Test
+  fun `CliTestRunner test per module`(@TempDir tempDir: Path) {
+    val code1 =
+      """
+      amends "pkl:test"
+
+      facts {
+        ["foo"] {
+          true
+        }
+      }
+    """
+        .trimIndent()
+
+    val code2 =
+      """
+      amends "pkl:test"
+
+      facts {
+        ["bar"] {
+          true
+        }
+      }
+    """
+        .trimIndent()
+    val input1 = tempDir.resolve("test1.pkl").writeString(code1).toString()
+    val input2 = tempDir.resolve("test2.pkl").writeString(code2).toString()
+    val noopWriter = noopWriter()
+    val opts =
+      CliBaseOptions(
+        sourceModules = listOf(input1.toUri(), input2.toUri()),
+        settings = URI("pkl:settings"),
+      )
+    val testOpts = CliTestOptions(junitDir = tempDir)
+    val runner = CliTestRunner(opts, testOpts, noopWriter, noopWriter)
+    runner.run()
+
+    assertThat(tempDir.resolve("test1.xml")).isNotEmptyFile()
+    assertThat(tempDir.resolve("test2.xml")).isNotEmptyFile()
+  }
+
+  @Test
+  fun `CliTestRunner test aggregate`(@TempDir tempDir: Path) {
+    val code1 =
+      """
+      amends "pkl:test"
+
+      facts {
+        ["foo"] {
+          true
+        }
+        ["bar"] {
+          5 == 9
+        }
+      }
+    """
+        .trimIndent()
+
+    val code2 =
+      """
+      amends "pkl:test"
+
+      facts {
+        ["xxx"] {
+          false
+        }
+        ["yyy"] {
+          false
+        }
+        ["zzz"] {
+          true
+        }
+      }
+    """
+        .trimIndent()
+    val input1 = tempDir.resolve("test1.pkl").writeString(code1).toString()
+    val input2 = tempDir.resolve("test2.pkl").writeString(code2).toString()
+    val noopWriter = noopWriter()
+    val opts =
+      CliBaseOptions(
+        sourceModules = listOf(input1.toUri(), input2.toUri()),
+        settings = URI("pkl:settings"),
+      )
+    val testOpts = CliTestOptions(junitDir = tempDir, junitAggregateReports = true)
+    val runner = CliTestRunner(opts, testOpts, noopWriter, noopWriter)
+    assertThatCode { runner.run() }.hasMessageContaining("failed")
+
+    assertThat(tempDir.resolve("pkl-tests.xml")).isNotEmptyFile()
+    assertThat(tempDir.resolve("test1.xml")).doesNotExist()
+    assertThat(tempDir.resolve("test2.xml")).doesNotExist()
+
+    val junitReport = tempDir.resolve("pkl-tests.xml").readString().stripFileAndLines(tempDir)
+
+    assertThat(junitReport)
+      .isEqualTo(
+        """
+      <?xml version="1.0" encoding="UTF-8"?>
+      <testsuites name="pkl-tests" tests="5" failures="3">
+          <testsuite name="test1" tests="2" failures="1">
+              <testcase classname="test1.facts" name="foo"></testcase>
+              <testcase classname="test1.facts" name="bar">
+                  <failure message="Fact Failure">5 == 9 (/tempDir/test1.pkl, line xx)</failure>
+              </testcase>
+          </testsuite>
+          <testsuite name="test2" tests="3" failures="2">
+              <testcase classname="test2.facts" name="xxx">
+                  <failure message="Fact Failure">false (/tempDir/test2.pkl, line xx)</failure>
+              </testcase>
+              <testcase classname="test2.facts" name="yyy">
+                  <failure message="Fact Failure">false (/tempDir/test2.pkl, line xx)</failure>
+              </testcase>
+              <testcase classname="test2.facts" name="zzz"></testcase>
+          </testsuite>
+      </testsuites>
+      
+    """
+          .trimIndent()
+      )
+  }
+
+  @Test
+  fun `CliTestRunner test aggregate suite name`(@TempDir tempDir: Path) {
+    val code1 =
+      """
+      amends "pkl:test"
+
+      facts {
+        ["foo"] {
+          true
+        }
+      }
+    """
+        .trimIndent()
+
+    val code2 =
+      """
+      amends "pkl:test"
+
+      facts {
+        ["bar"] {
+          true
+        }
+      }
+    """
+        .trimIndent()
+    val input1 = tempDir.resolve("test1.pkl").writeString(code1).toString()
+    val input2 = tempDir.resolve("test2.pkl").writeString(code2).toString()
+    val noopWriter = noopWriter()
+    val opts =
+      CliBaseOptions(
+        sourceModules = listOf(input1.toUri(), input2.toUri()),
+        settings = URI("pkl:settings"),
+      )
+    val testOpts =
+      CliTestOptions(
+        junitDir = tempDir,
+        junitAggregateReports = true,
+        junitAggregateSuiteName = "custom",
+      )
+    val runner = CliTestRunner(opts, testOpts, noopWriter, noopWriter)
+    runner.run()
+
+    assertThat(tempDir.resolve("custom.xml")).isNotEmptyFile()
+    assertThat(tempDir.resolve("pkl-tests.xml")).doesNotExist()
+    assertThat(tempDir.resolve("test1.xml")).doesNotExist()
+    assertThat(tempDir.resolve("test2.xml")).doesNotExist()
+  }
+
   @Test
   fun `no source modules specified has same message as pkl eval`() {
     val e1 = assertThrows<CliException> { CliTestRunner(CliBaseOptions(), CliTestOptions()).run() }

pkl-commons-cli/src/main/kotlin/org/pkl/commons/cli/CliTestOptions.kt

@@ -1,5 +1,5 @@
 /*
- * Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved.
+ * Copyright © 2024-2025 Apple Inc. and the Pkl project authors. All rights reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -17,4 +17,9 @@ package org.pkl.commons.cli
 
 import java.nio.file.Path
 
-class CliTestOptions(val junitDir: Path? = null, val overwrite: Boolean = false)
+class CliTestOptions(
+  val junitDir: Path? = null,
+  val overwrite: Boolean = false,
+  val junitAggregateReports: Boolean = false,
+  val junitAggregateSuiteName: String = "pkl-tests",
+)

pkl-commons-cli/src/main/kotlin/org/pkl/commons/cli/commands/TestOptions.kt

@@ -16,6 +16,7 @@
 package org.pkl.commons.cli.commands
 
 import com.github.ajalt.clikt.parameters.groups.OptionGroup
+import com.github.ajalt.clikt.parameters.options.default
 import com.github.ajalt.clikt.parameters.options.flag
 import com.github.ajalt.clikt.parameters.options.option
 import com.github.ajalt.clikt.parameters.types.path
@@ -31,8 +32,26 @@ class TestOptions : OptionGroup() {
       )
       .path()
 
+  private val junitAggregateReports: Boolean by
+    option(
+        names = arrayOf("--junit-aggregate-reports"),
+        help = "Aggregate JUnit reports into a single file.",
+      )
+      .flag()
+
+  private val junitAggregateSuiteName: String by
+    option(
+        names = arrayOf("--junit-aggregate-suite-name"),
+        metavar = "name",
+        help = "The name of the root JUnit test suite.",
+      )
+      .single()
+      .default("pkl-tests")
+
   private val overwrite: Boolean by
     option(names = arrayOf("--overwrite"), help = "Force generation of expected examples.").flag()
 
-  val cliTestOptions: CliTestOptions by lazy { CliTestOptions(junitReportDir, overwrite) }
+  val cliTestOptions: CliTestOptions by lazy {
+    CliTestOptions(junitReportDir, overwrite, junitAggregateReports, junitAggregateSuiteName)
+  }
 }

pkl-core/src/main/java/org/pkl/core/stdlib/test/report/JUnitReport.java

@@ -1,5 +1,5 @@
 /*
- * Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved.
+ * Copyright © 2024-2025 Apple Inc. and the Pkl project authors. All rights reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -18,7 +18,9 @@ package org.pkl.core.stdlib.test.report;
 import java.io.IOException;
 import java.io.Writer;
 import java.util.ArrayList;
+import java.util.List;
 import java.util.function.Consumer;
+import java.util.stream.Collectors;
 import org.graalvm.collections.EconomicMap;
 import org.pkl.core.TestResults;
 import org.pkl.core.TestResults.Error;
@@ -38,11 +40,45 @@ import org.pkl.core.util.EconomicMaps;
 
 public final class JUnitReport implements TestReport {
 
+  private final String aggregateSuiteName;
+
+  public JUnitReport(String aggregateSuiteName) {
+    this.aggregateSuiteName = aggregateSuiteName;
+  }
+
+  public JUnitReport() {
+    this("");
+  }
+
   @Override
   public void report(TestResults results, Writer writer) throws IOException {
     writer.append(renderXML("    ", "1.0", buildSuite(results)));
   }
 
+  @Override
+  public void summarize(List<TestResults> allTestResults, Writer writer) throws IOException {
+    var totalTests = allTestResults.stream().collect(Collectors.summingLong(r -> r.totalTests()));
+    var totalFailures =
+        allTestResults.stream().collect(Collectors.summingLong(r -> r.totalFailures()));
+
+    assert aggregateSuiteName != null;
+
+    var attrs =
+        buildAttributes(
+            "name", aggregateSuiteName,
+            "tests", totalTests,
+            "failures", totalFailures);
+
+    var tests =
+        allTestResults.stream()
+            .map(r -> buildSuite(r))
+            .collect(Collectors.toCollection(ArrayList::new));
+
+    var suite = buildXmlElement("testsuites", attrs, tests.toArray(new VmDynamic[0]));
+
+    writer.append(renderXML("    ", "1.0", suite));
+  }
+
   private VmDynamic buildSuite(TestResults results) {
     if (results.error() != null) {
       var testCase = rootTestCase(results, results.error());

pkl-core/src/main/java/org/pkl/core/stdlib/test/report/SimpleReport.java

@@ -1,5 +1,5 @@
 /*
- * Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved.
+ * Copyright © 2024-2025 Apple Inc. and the Pkl project authors. All rights reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -57,6 +57,7 @@ public final class SimpleReport implements TestReport {
     writer.append(builder.toString());
   }
 
+  @Override
   public void summarize(List<TestResults> allTestResults, Writer writer) throws IOException {
     var totalTests = 0;
     var totalFailedTests = 0;

pkl-core/src/main/java/org/pkl/core/stdlib/test/report/TestReport.java

@@ -1,5 +1,5 @@
 /*
- * Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved.
+ * Copyright © 2024-2025 Apple Inc. and the Pkl project authors. All rights reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -20,6 +20,7 @@ import java.io.IOException;
 import java.io.Writer;
 import java.nio.charset.StandardCharsets;
 import java.nio.file.Path;
+import java.util.List;
 import org.pkl.core.PklBugException;
 import org.pkl.core.TestResults;
 import org.pkl.core.util.StringBuilderWriter;
@@ -28,7 +29,9 @@ public interface TestReport {
 
   void report(TestResults results, Writer writer) throws IOException;
 
-  default String report(TestResults results) {
+  void summarize(List<TestResults> allTestResults, Writer writer) throws IOException;
+
+  default String report(TestResults results) throws IOException {
     try {
       var builder = new StringBuilder();
       var writer = new StringBuilderWriter(builder);
@@ -44,4 +47,10 @@ public interface TestReport {
       report(results, writer);
     }
   }
+
+  default void summarizeToPath(List<TestResults> allTestResults, Path path) throws IOException {
+    try (var writer = new FileWriter(path.toFile(), StandardCharsets.UTF_8)) {
+      summarize(allTestResults, writer);
+    }
+  }
 }

pkl-gradle/src/main/java/org/pkl/gradle/task/ProjectPackageTask.java

@@ -1,5 +1,5 @@
 /*
- * Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved.
+ * Copyright © 2024-2025 Apple Inc. and the Pkl project authors. All rights reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -43,6 +43,13 @@ public abstract class ProjectPackageTask extends BasePklTask {
   @OutputDirectory
   public abstract DirectoryProperty getJunitReportsDir();
 
+  @Input
+  @Optional
+  public abstract Property<Boolean> getJunitAggregateReports();
+
+  @Input
+  public abstract Property<String> getJunitAggregateSuiteName();
+
   @Input
   public abstract Property<Boolean> getOverwrite();
 
@@ -50,6 +57,10 @@ public abstract class ProjectPackageTask extends BasePklTask {
   @Optional
   public abstract Property<Boolean> getSkipPublishCheck();
 
+  public ProjectPackageTask() {
+    this.getJunitAggregateSuiteName().convention("pkl-tests");
+  }
+
   @Override
   protected void doRunTask() {
     var projectDirectories =
@@ -59,12 +70,15 @@ public abstract class ProjectPackageTask extends BasePklTask {
     if (projectDirectories.isEmpty()) {
       throw new InvalidUserDataException("No project directories specified.");
     }
+
     new CliProjectPackager(
             getCliBaseOptions(),
             projectDirectories,
             new CliTestOptions(
                 mapAndGetOrNull(getJunitReportsDir(), it -> it.getAsFile().toPath()),
-                getOverwrite().get()),
+                getOverwrite().get(),
+                getJunitAggregateReports().getOrElse(false),
+                getJunitAggregateSuiteName().get()),
             getOutputPath().get().getAsFile().getAbsolutePath(),
             getSkipPublishCheck().getOrElse(false),
             new PrintWriter(System.out),

pkl-gradle/src/main/java/org/pkl/gradle/task/TestTask.java

@@ -1,5 +1,5 @@
 /*
- * Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved.
+ * Copyright © 2024-2025 Apple Inc. and the Pkl project authors. All rights reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -29,16 +29,29 @@ public abstract class TestTask extends ModulesTask {
   @OutputDirectory
   public abstract DirectoryProperty getJunitReportsDir();
 
+  @Input
+  @Optional
+  public abstract Property<Boolean> getJunitAggregateReports();
+
+  @Input
+  public abstract Property<String> getJunitAggregateSuiteName();
+
   @Input
   public abstract Property<Boolean> getOverwrite();
 
+  public TestTask() {
+    this.getJunitAggregateSuiteName().convention("pkl-tests");
+  }
+
   @Override
   protected void doRunTask() {
     new CliTestRunner(
             getCliBaseOptions(),
             new CliTestOptions(
                 mapAndGetOrNull(getJunitReportsDir(), it -> it.getAsFile().toPath()),
-                getOverwrite().get()),
+                getOverwrite().get(),
+                getJunitAggregateReports().getOrElse(false),
+                getJunitAggregateSuiteName().get()),
             new PrintWriter(System.out),
             new PrintWriter(System.err))
         .run();