Fix missing resources in native pkldoc, and disable test mode (#1175)

This fixes two issues:

1. Test mode is enabled in pkldoc without the ability to turn it off
2. Native pkldoc is missing required resources

This also adds tests for both `jpkldoc` and `pkldoc`.

pkl-commons-test/src/main/kotlin/org/pkl/commons/test/Executables.kt

@@ -0,0 +1,85 @@
+/*
+ * Copyright © 2024-2025 Apple Inc. and the Pkl project authors. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.pkl.commons.test
+
+import java.nio.file.Files
+import java.nio.file.Path
+import kotlin.io.path.exists
+import org.pkl.commons.test.FileTestUtils.rootProjectDir
+
+sealed class ExecutablePaths(protected val gradleProject: String) {
+  abstract val allNative: List<Path>
+
+  val existingNative: List<Path>
+    get() = allNative.filter(Files::exists)
+
+  val firstExistingNative: Path
+    get() =
+      existingNative.firstOrNull()
+        ?: throw AssertionError(
+          "Native executable not found on system. " +
+            "To fix this problem, run `./gradlew $gradleProject:assembleNative`."
+        )
+
+  protected fun executable(name: String): Path =
+    rootProjectDir.resolve("$gradleProject/build/executable").resolve(name)
+
+  protected fun javaExecutable(name: String): Path {
+    val isWindows = System.getProperty("os.name").startsWith("Windows")
+    val effectiveName = if (isWindows) "$name.bat" else name
+    return rootProjectDir.resolve("$gradleProject/build/executable").resolve(effectiveName).also {
+      path ->
+      if (!path.exists()) {
+        throw AssertionError(
+          "Java executable not found on system. " +
+            "To fix this problem, run `./gradlew $gradleProject:javaExecutable`."
+        )
+      }
+    }
+  }
+}
+
+@Suppress("ClassName")
+object Executables {
+
+  object pkl : ExecutablePaths("pkl-cli") {
+    val macAarch64: Path = executable("pkl-macos-aarch64")
+    val macAmd64: Path = executable("pkl-macos-amd64")
+    val linuxAarch64: Path = executable("pkl-linux-aarch64")
+    val linuxAmd64: Path = executable("pkl-linux-amd64")
+    val alpineAmd64: Path = executable("pkl-alpine-linux-amd64")
+    val windowsAmd64: Path = executable("pkl-windows-amd64.exe")
+
+    // order (aarch64 before amd64, linux before alpine) affects [firstExisting]
+    override val allNative: List<Path> =
+      listOf(macAarch64, macAmd64, linuxAarch64, linuxAmd64, alpineAmd64, windowsAmd64)
+  }
+
+  object pkldoc : ExecutablePaths("pkl-doc") {
+    val macAarch64: Path = executable("pkldoc-macos-aarch64")
+    val macAmd64: Path = executable("pkldoc-macos-amd64")
+    val linuxAarch64: Path = executable("pkldoc-linux-aarch64")
+    val linuxAmd64: Path = executable("pkldoc-linux-amd64")
+    val alpineAmd64: Path = executable("pkldoc-alpine-linux-amd64")
+    val windowsAmd64: Path = executable("pkldoc-windows-amd64.exe")
+
+    val javaExecutable: Path by lazy { javaExecutable("jpkldoc") }
+
+    // order (aarch64 before amd64, linux before alpine) affects [firstExisting]
+    override val allNative: List<Path> =
+      listOf(macAarch64, macAmd64, linuxAarch64, linuxAmd64, alpineAmd64, windowsAmd64)
+  }
+}

pkl-commons-test/src/main/kotlin/org/pkl/commons/test/PklExecutablePaths.kt

@@ -1,47 +0,0 @@
-/*
- * Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     https://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.pkl.commons.test
-
-import java.nio.file.Files
-import java.nio.file.Path
-import org.pkl.commons.test.FileTestUtils.rootProjectDir
-
-object PklExecutablePaths {
-  val macAarch64: Path = executablePath("pkl-macos-aarch64")
-  val macAmd64: Path = executablePath("pkl-macos-amd64")
-  val linuxAarch64: Path = executablePath("pkl-linux-aarch64")
-  val linuxAmd64: Path = executablePath("pkl-linux-amd64")
-  val alpineAmd64: Path = executablePath("pkl-alpine-linux-amd64")
-  val windowsAmd64: Path = executablePath("pkl-windows-amd64.exe")
-
-  // order (aarch64 before amd64, linux before alpine) affects [firstExisting]
-  val all: List<Path> =
-    listOf(macAarch64, macAmd64, linuxAarch64, linuxAmd64, alpineAmd64, windowsAmd64)
-
-  val existing: List<Path>
-    get() = all.filter(Files::exists)
-
-  val firstExisting: Path
-    get() =
-      existing.firstOrNull()
-        ?: throw AssertionError(
-          "Native executable not found on system. " +
-            "To fix this problem, run `./gradlew assembleNative`."
-        )
-
-  private fun executablePath(name: String): Path =
-    rootProjectDir.resolve("pkl-cli/build/executable").resolve(name)
-}

pkl-core/src/test/kotlin/org/pkl/core/LanguageSnippetTestsEngine.kt

@@ -27,10 +27,10 @@ import org.junit.platform.engine.EngineDiscoveryRequest
 import org.junit.platform.engine.TestDescriptor
 import org.junit.platform.engine.UniqueId
 import org.junit.platform.engine.support.descriptor.EngineDescriptor
+import org.pkl.commons.test.Executables
 import org.pkl.commons.test.FileTestUtils
 import org.pkl.commons.test.InputOutputTestEngine
 import org.pkl.commons.test.PackageServer
-import org.pkl.commons.test.PklExecutablePaths
 import org.pkl.core.http.HttpClient
 import org.pkl.core.project.Project
 import org.pkl.core.util.IoUtils
@@ -298,27 +298,27 @@ abstract class AbstractNativeLanguageSnippetTestsEngine : AbstractLanguageSnippe
 }
 
 class MacAmd64LanguageSnippetTestsEngine : AbstractNativeLanguageSnippetTestsEngine() {
-  override val pklExecutablePath: Path = PklExecutablePaths.macAmd64
+  override val pklExecutablePath: Path = Executables.pkl.macAmd64
   override val testClass: KClass<*> = MacLanguageSnippetTests::class
 }
 
 class MacAarch64LanguageSnippetTestsEngine : AbstractNativeLanguageSnippetTestsEngine() {
-  override val pklExecutablePath: Path = PklExecutablePaths.macAarch64
+  override val pklExecutablePath: Path = Executables.pkl.macAarch64
   override val testClass: KClass<*> = MacLanguageSnippetTests::class
 }
 
 class LinuxAmd64LanguageSnippetTestsEngine : AbstractNativeLanguageSnippetTestsEngine() {
-  override val pklExecutablePath: Path = PklExecutablePaths.linuxAmd64
+  override val pklExecutablePath: Path = Executables.pkl.linuxAmd64
   override val testClass: KClass<*> = LinuxLanguageSnippetTests::class
 }
 
 class LinuxAarch64LanguageSnippetTestsEngine : AbstractNativeLanguageSnippetTestsEngine() {
-  override val pklExecutablePath: Path = PklExecutablePaths.linuxAarch64
+  override val pklExecutablePath: Path = Executables.pkl.linuxAarch64
   override val testClass: KClass<*> = LinuxLanguageSnippetTests::class
 }
 
 class AlpineLanguageSnippetTestsEngine : AbstractNativeLanguageSnippetTestsEngine() {
-  override val pklExecutablePath: Path = PklExecutablePaths.alpineAmd64
+  override val pklExecutablePath: Path = Executables.pkl.alpineAmd64
   override val testClass: KClass<*> = AlpineLanguageSnippetTests::class
 }
 
@@ -340,7 +340,7 @@ private val windowsNativeExcludedTests
     )
 
 class WindowsLanguageSnippetTestsEngine : AbstractNativeLanguageSnippetTestsEngine() {
-  override val pklExecutablePath: Path = PklExecutablePaths.windowsAmd64
+  override val pklExecutablePath: Path = Executables.pkl.windowsAmd64
   override val testClass: KClass<*> = WindowsLanguageSnippetTests::class
   override val excludedTests: List<Regex>
     get() = super.excludedTests + windowsNativeExcludedTests + windowsExcludedTests

pkl-doc/pkl-doc.gradle.kts

@@ -66,6 +66,31 @@ publishing {
   }
 }
 
+val testNativeExecutable by
+  tasks.registering(Test::class) {
+    inputs.dir("src/test/files/DocGeneratorTest/input")
+    outputs.dir("src/test/files/DocGeneratorTest/output")
+    systemProperty("org.pkl.doc.NativeExecutableTest", "true")
+    include(listOf("**/NativeExecutableTest.class"))
+  }
+
+val testJavaExecutable by
+  tasks.registering(Test::class) {
+    dependsOn(tasks.javaExecutable)
+    inputs.dir("src/test/files/DocGeneratorTest/input")
+    outputs.dir("src/test/files/DocGeneratorTest/output")
+    systemProperty("org.pkl.doc.JavaExecutableTest", "true")
+    include(listOf("**/JavaExecutableTest.class"))
+  }
+
+tasks.check { dependsOn(testJavaExecutable) }
+
+tasks.testNative { dependsOn(testNativeExecutable) }
+
+tasks.withType<NativeImageBuild> { extraNativeImageArgs.add("-H:IncludeResources=org/pkl/doc/.*") }
+
 tasks.jar { manifest { attributes += mapOf("Main-Class" to "org.pkl.doc.Main") } }
 
 htmlValidator { sources = files("src/test/files/DocGeneratorTest/output") }
+
+tasks.validateHtml { mustRunAfter(testJavaExecutable) }

pkl-doc/src/main/kotlin/org/pkl/doc/Main.kt

@@ -69,6 +69,9 @@ class DocCommand : BaseCommand(name = "pkldoc", helpLink = helpLink) {
       .single()
       .flag(default = false)
 
+  private val isTestMode by
+    option(names = arrayOf("--test-mode"), help = "Internal test mode", hidden = true).flag()
+
   private val projectOptions by ProjectOptions()
 
   override val helpString: String = "Generate HTML documentation from Pkl modules and packages."
@@ -78,7 +81,7 @@ class DocCommand : BaseCommand(name = "pkldoc", helpLink = helpLink) {
       CliDocGeneratorOptions(
         baseOptions.baseOptions(modules, projectOptions),
         outputDir,
-        true,
+        isTestMode,
         noSymlinks,
       )
     CliDocGenerator(options).run()

pkl-doc/src/test/kotlin/org/pkl/doc/CliDocGeneratorTest.kt

@@ -18,11 +18,9 @@ package org.pkl.doc
 import com.google.common.jimfs.Configuration
 import com.google.common.jimfs.Jimfs
 import java.net.URI
-import java.nio.file.FileSystem
 import java.nio.file.Files
 import java.nio.file.Path
 import kotlin.io.path.*
-import org.assertj.core.api.Assertions
 import org.assertj.core.api.Assertions.assertThat
 import org.junit.jupiter.api.Test
 import org.junit.jupiter.api.assertThrows
@@ -31,83 +29,20 @@ import org.junit.jupiter.params.ParameterizedTest
 import org.junit.jupiter.params.provider.MethodSource
 import org.pkl.commons.cli.CliBaseOptions
 import org.pkl.commons.cli.CliException
-import org.pkl.commons.readString
-import org.pkl.commons.test.FileTestUtils
 import org.pkl.commons.test.PackageServer
-import org.pkl.commons.test.listFilesRecursively
-import org.pkl.commons.toPath
 import org.pkl.commons.walk
 import org.pkl.core.Version
-import org.pkl.core.util.IoUtils
 import org.pkl.doc.DocGenerator.Companion.current
 
 class CliDocGeneratorTest {
   companion object {
-    private val tempFileSystem: FileSystem by lazy { Jimfs.newFileSystem(Configuration.unix()) }
+    private val tempFileSystem by lazy { Jimfs.newFileSystem(Configuration.unix()) }
 
-    private val tmpOutputDir by lazy {
+    private val tmpOutputDir: Path by lazy {
       tempFileSystem.getPath("/work/output").apply { createDirectories() }
     }
 
-    private val projectDir = FileTestUtils.rootProjectDir.resolve("pkl-doc")
-
-    private val inputDir: Path by lazy {
-      projectDir.resolve("src/test/files/DocGeneratorTest/input").apply { assert(exists()) }
-    }
-
-    private val docsiteModule: URI by lazy {
-      inputDir.resolve("docsite-info.pkl").apply { assert(exists()) }.toUri()
-    }
-
-    internal val package1PackageModule: URI by lazy {
-      inputDir.resolve("com.package1/doc-package-info.pkl").apply { assert(exists()) }.toUri()
-    }
-
-    private val package2PackageModule: URI by lazy {
-      inputDir.resolve("com.package2/doc-package-info.pkl").apply { assert(exists()) }.toUri()
-    }
-
-    internal val package1InputModules: List<URI> by lazy {
-      inputDir
-        .resolve("com.package1")
-        .listFilesRecursively()
-        .filter { it.fileName.toString() != "doc-package-info.pkl" }
-        .map { it.toUri() }
-    }
-
-    private val package2InputModules: List<URI> by lazy {
-      inputDir
-        .resolve("com.package2")
-        .listFilesRecursively()
-        .filter { it.fileName.toString() != "doc-package-info.pkl" }
-        .map { it.toUri() }
-    }
-
-    private val expectedOutputDir: Path by lazy {
-      projectDir.resolve("src/test/files/DocGeneratorTest/output").createDirectories()
-    }
-
-    private val expectedOutputFiles: List<Path> by lazy { expectedOutputDir.listFilesRecursively() }
-
-    private val actualOutputDir: Path by lazy { tempFileSystem.getPath("/work/DocGeneratorTest") }
-
-    private val actualOutputFiles: List<Path> by lazy { actualOutputDir.listFilesRecursively() }
-
-    private val expectedRelativeOutputFiles: List<String> by lazy {
-      expectedOutputFiles.map { path ->
-        IoUtils.toNormalizedPathString(expectedOutputDir.relativize(path)).let { str ->
-          // Git will by default clone symlinks as shortcuts on Windows, and shortcuts have a
-          // `.lnk` extension.
-          if (IoUtils.isWindows() && str.endsWith(".lnk")) str.dropLast(4) else str
-        }
-      }
-    }
-
-    private val actualRelativeOutputFiles: List<String> by lazy {
-      actualOutputFiles.map { IoUtils.toNormalizedPathString(actualOutputDir.relativize(it)) }
-    }
-
-    private val binaryFileExtensions = setOf("woff2", "png", "svg")
+    private val helper = DocGeneratorTestHelper()
 
     private fun runDocGenerator(outputDir: Path, cacheDir: Path?, noSymlinks: Boolean = false) {
       CliDocGenerator(
@@ -115,14 +50,14 @@ class CliDocGeneratorTest {
             CliBaseOptions(
               sourceModules =
                 listOf(
-                  docsiteModule,
-                  package1PackageModule,
-                  package2PackageModule,
+                  helper.docsiteModule,
+                  helper.package1PackageModule,
+                  helper.package2PackageModule,
                   URI("package://localhost:0/birds@0.5.0"),
                   URI("package://localhost:0/fruit@1.1.0"),
                   URI("package://localhost:0/unlisted@1.0.0"),
                   URI("package://localhost:0/deprecated@1.0.0"),
-                ) + package1InputModules + package2InputModules,
+                ) + helper.package1InputModules + helper.package2InputModules,
               moduleCacheDir = cacheDir,
             ),
             outputDir = outputDir,
@@ -135,19 +70,7 @@ class CliDocGeneratorTest {
 
     @JvmStatic
     private fun generateDocs(): List<String> {
-      val cacheDir = Files.createTempDirectory("cli-doc-generator-test-cache")
-      PackageServer.populateCacheDir(cacheDir)
-      runDocGenerator(actualOutputDir, cacheDir)
-
-      val missingFiles = expectedRelativeOutputFiles - actualRelativeOutputFiles.toSet()
-      if (missingFiles.isNotEmpty()) {
-        Assertions.fail<Unit>(
-          "The following expected files were not actually generated:\n" +
-            missingFiles.joinToString("\n")
-        )
-      }
-
-      return actualRelativeOutputFiles
+      return helper.generateDocs()
     }
   }
 
@@ -158,6 +81,7 @@ class CliDocGeneratorTest {
         createParentDirectories()
         createFile()
       }
+
     val descriptor2 =
       tempFileSystem.getPath("/work/dir2/docsite-info.pkl").apply {
         createParentDirectories()
@@ -220,49 +144,20 @@ class CliDocGeneratorTest {
   @ParameterizedTest
   @MethodSource("generateDocs")
   fun test(relativeFilePath: String) {
-    val actualFile = actualOutputDir.resolve(relativeFilePath)
-    assertThat(actualFile)
-      .withFailMessage("Test bug: $actualFile should exist but does not.")
-      .exists()
-
-    // symlinks on Git and Windows is rather finnicky; they create shortcuts by default unless
-    // a core Git option is set. Also, by default, symlinks require administrator privileges to run.
-    // We'll just test that the symlink got created but skip verifying that it points to the right
-    // place.
-    if (actualFile.isSymbolicLink() && IoUtils.isWindows()) return
-    val expectedFile = expectedOutputDir.resolve(relativeFilePath)
-    if (expectedFile.exists()) {
-      when {
-        expectedFile.isSymbolicLink() -> {
-          assertThat(actualFile).isSymbolicLink
-          assertThat(expectedFile.readSymbolicLink().toString().toPath())
-            .isEqualTo(actualFile.readSymbolicLink().toString().toPath())
-        }
-        expectedFile.extension in binaryFileExtensions ->
-          assertThat(actualFile.readBytes()).isEqualTo(expectedFile.readBytes())
-        else -> assertThat(actualFile.readString()).isEqualTo(expectedFile.readString())
-      }
-    } else {
-      expectedFile.createParentDirectories()
-      if (actualFile.isSymbolicLink()) {
-        // needs special handling because `copyTo` can't copy symlinks between file systems
-        val linkTarget = actualFile.readSymbolicLink()
-        assertThat(linkTarget).isRelative
-        Files.createSymbolicLink(expectedFile, linkTarget.toString().toPath())
-      } else {
-        actualFile.copyTo(expectedFile)
-      }
-      Assertions.fail("Created missing expected file `$relativeFilePath`.")
-    }
+    DocTestUtils.testExpectedFile(
+      helper.expectedOutputDir,
+      helper.actualOutputDir,
+      relativeFilePath,
+    )
   }
 
   @Test
   fun `creates a symlink called current by default`(@TempDir tempDir: Path) {
     PackageServer.populateCacheDir(tempDir)
-    runDocGenerator(actualOutputDir, tempDir)
+    runDocGenerator(helper.actualOutputDir, tempDir)
 
-    val expectedSymlink = actualOutputDir.resolve("com.package1/current")
-    val expectedDestination = actualOutputDir.resolve("com.package1/1.2.3")
+    val expectedSymlink = helper.actualOutputDir.resolve("com.package1/current")
+    val expectedDestination = helper.actualOutputDir.resolve("com.package1/1.2.3")
 
     assertThat(expectedSymlink).isSymbolicLink().matches {
       Files.isSameFile(it, expectedDestination)
@@ -274,10 +169,10 @@ class CliDocGeneratorTest {
     @TempDir tempDir: Path
   ) {
     PackageServer.populateCacheDir(tempDir)
-    runDocGenerator(actualOutputDir, tempDir, noSymlinks = true)
+    runDocGenerator(helper.actualOutputDir, tempDir, noSymlinks = true)
 
-    val currentDirectory = actualOutputDir.resolve("com.package1/current")
-    val sourceDirectory = actualOutputDir.resolve("com.package1/1.2.3")
+    val currentDirectory = helper.actualOutputDir.resolve("com.package1/current")
+    val sourceDirectory = helper.actualOutputDir.resolve("com.package1/1.2.3")
 
     assertThat(currentDirectory).isDirectory()
     assertThat(currentDirectory.isSymbolicLink()).isFalse()

pkl-doc/src/test/kotlin/org/pkl/doc/DocGeneratorTestHelper.kt

@@ -0,0 +1,181 @@
+/*
+ * Copyright © 2025 Apple Inc. and the Pkl project authors. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.pkl.doc
+
+import java.net.URI
+import java.nio.file.Files
+import java.nio.file.Path
+import kotlin.io.path.createDirectories
+import kotlin.io.path.exists
+import org.assertj.core.api.Assertions
+import org.junit.jupiter.api.fail
+import org.pkl.commons.cli.CliBaseOptions
+import org.pkl.commons.test.FileTestUtils
+import org.pkl.commons.test.PackageServer
+import org.pkl.commons.test.listFilesRecursively
+import org.pkl.core.util.IoUtils
+
+class DocGeneratorTestHelper {
+  internal val tempDir by lazy { Files.createTempDirectory("ExecutableCliDocGeneratorTest") }
+
+  internal val projectDir = FileTestUtils.rootProjectDir.resolve("pkl-doc")
+
+  internal val inputDir: Path by lazy {
+    projectDir.resolve("src/test/files/DocGeneratorTest/input").apply { assert(exists()) }
+  }
+
+  internal val docsiteModule: URI by lazy {
+    inputDir.resolve("docsite-info.pkl").apply { assert(exists()) }.toUri()
+  }
+
+  internal val package1PackageModule: URI by lazy {
+    inputDir.resolve("com.package1/doc-package-info.pkl").apply { assert(exists()) }.toUri()
+  }
+
+  internal val package2PackageModule: URI by lazy {
+    inputDir.resolve("com.package2/doc-package-info.pkl").apply { assert(exists()) }.toUri()
+  }
+
+  internal val package1InputModules: List<URI> by lazy {
+    inputDir
+      .resolve("com.package1")
+      .listFilesRecursively()
+      .filter { it.fileName.toString() != "doc-package-info.pkl" }
+      .map { it.toUri() }
+  }
+
+  internal val package2InputModules: List<URI> by lazy {
+    inputDir
+      .resolve("com.package2")
+      .listFilesRecursively()
+      .filter { it.fileName.toString() != "doc-package-info.pkl" }
+      .map { it.toUri() }
+  }
+
+  internal val expectedOutputDir: Path by lazy {
+    projectDir.resolve("src/test/files/DocGeneratorTest/output").createDirectories()
+  }
+
+  internal val expectedOutputFiles: List<Path> by lazy { expectedOutputDir.listFilesRecursively() }
+
+  internal val actualOutputDir: Path by lazy {
+    tempDir.resolve("work/DocGeneratorTest").createDirectories()
+  }
+
+  internal val actualOutputFiles: List<Path> by lazy { actualOutputDir.listFilesRecursively() }
+
+  internal val cacheDir: Path by lazy { tempDir.resolve("cache") }
+
+  internal val sourceModules =
+    listOf(
+      docsiteModule,
+      package1PackageModule,
+      package2PackageModule,
+      URI("package://localhost:0/birds@0.5.0"),
+      URI("package://localhost:0/fruit@1.1.0"),
+      URI("package://localhost:0/unlisted@1.0.0"),
+      URI("package://localhost:0/deprecated@1.0.0"),
+    ) + package1InputModules + package2InputModules
+
+  internal val expectedRelativeOutputFiles: List<String> by lazy {
+    expectedOutputFiles.map { path ->
+      IoUtils.toNormalizedPathString(expectedOutputDir.relativize(path)).let { str ->
+        // Git will by default clone symlinks as shortcuts on Windows, and shortcuts have a
+        // `.lnk` extension.
+        if (IoUtils.isWindows() && str.endsWith(".lnk")) str.dropLast(4) else str
+      }
+    }
+  }
+
+  internal val actualRelativeOutputFiles: List<String> by lazy {
+    actualOutputFiles.map { IoUtils.toNormalizedPathString(actualOutputDir.relativize(it)) }
+  }
+
+  fun runPklDocCli(executable: Path, options: CliDocGeneratorOptions) {
+    val command = buildList {
+      add(executable.toString())
+      add("--output-dir")
+      add(options.normalizedOutputDir.toString())
+      add("--cache-dir")
+      add(options.base.normalizedModuleCacheDir.toString())
+      add("--test-mode")
+      addAll(sourceModules.map { it.toString() })
+    }
+    val process =
+      with(ProcessBuilder(command)) {
+        redirectErrorStream(true)
+        start()
+      }
+    try {
+      val out = process.inputStream.reader().readText()
+      val exitCode = process.waitFor()
+
+      if (exitCode != 0) {
+        fail(
+          """
+            Process exited with $exitCode.
+
+            Output:
+          """
+            .trimIndent() + out
+        )
+      }
+    } finally {
+      process.destroy()
+    }
+  }
+
+  private fun generateDocsWith(doGenerate: (CliDocGeneratorOptions) -> Unit): List<String> {
+    PackageServer.populateCacheDir(cacheDir)
+    val options =
+      CliDocGeneratorOptions(
+        CliBaseOptions(
+          sourceModules =
+            listOf(
+              docsiteModule,
+              package1PackageModule,
+              package2PackageModule,
+              URI("package://localhost:0/birds@0.5.0"),
+              URI("package://localhost:0/fruit@1.1.0"),
+              URI("package://localhost:0/unlisted@1.0.0"),
+              URI("package://localhost:0/deprecated@1.0.0"),
+            ) + package1InputModules + package2InputModules,
+          moduleCacheDir = cacheDir,
+        ),
+        outputDir = actualOutputDir,
+        isTestMode = true,
+        noSymlinks = false,
+      )
+    doGenerate(options)
+    val missingFiles = expectedRelativeOutputFiles - actualRelativeOutputFiles.toSet()
+    if (missingFiles.isNotEmpty()) {
+      Assertions.fail<Unit>(
+        "The following expected files were not actually generated:\n" +
+          missingFiles.joinToString("\n")
+      )
+    }
+
+    return actualRelativeOutputFiles
+  }
+
+  fun generateDocsWithCli(executable: Path): List<String> {
+    return generateDocsWith { runPklDocCli(executable, it) }
+  }
+
+  fun generateDocs(): List<String> {
+    return generateDocsWith { CliDocGenerator(it).run() }
+  }
+}

pkl-doc/src/test/kotlin/org/pkl/doc/DocTestUtils.kt

@@ -0,0 +1,73 @@
+/*
+ * Copyright © 2025 Apple Inc. and the Pkl project authors. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.pkl.doc
+
+import java.nio.file.Files
+import java.nio.file.Path
+import kotlin.io.path.copyTo
+import kotlin.io.path.createParentDirectories
+import kotlin.io.path.exists
+import kotlin.io.path.extension
+import kotlin.io.path.isSymbolicLink
+import kotlin.io.path.readBytes
+import kotlin.io.path.readSymbolicLink
+import org.assertj.core.api.Assertions
+import org.assertj.core.api.Assertions.assertThat
+import org.pkl.commons.readString
+import org.pkl.commons.toPath
+import org.pkl.core.util.IoUtils
+
+object DocTestUtils {
+
+  private val binaryFileExtensions = setOf("woff2", "png", "svg")
+
+  fun testExpectedFile(expectedOutputDir: Path, actualOutputDir: Path, relativeFilePath: String) {
+    val actualFile = actualOutputDir.resolve(relativeFilePath)
+    assertThat(actualFile)
+      .withFailMessage("Test bug: $actualFile should exist but does not.")
+      .exists()
+
+    // symlinks on Git and Windows is rather finnicky; they create shortcuts by default unless
+    // a core Git option is set. Also, by default, symlinks require administrator privileges to run.
+    // We'll just test that the symlink got created but skip verifying that it points to the right
+    // place.
+    if (actualFile.isSymbolicLink() && IoUtils.isWindows()) return
+    val expectedFile = expectedOutputDir.resolve(relativeFilePath)
+    if (expectedFile.exists()) {
+      when {
+        expectedFile.isSymbolicLink() -> {
+          assertThat(actualFile).isSymbolicLink
+          assertThat(expectedFile.readSymbolicLink().toString().toPath())
+            .isEqualTo(actualFile.readSymbolicLink().toString().toPath())
+        }
+        expectedFile.extension in binaryFileExtensions ->
+          assertThat(actualFile.readBytes()).isEqualTo(expectedFile.readBytes())
+        else -> assertThat(actualFile.readString()).isEqualTo(expectedFile.readString())
+      }
+    } else {
+      expectedFile.createParentDirectories()
+      if (actualFile.isSymbolicLink()) {
+        // needs special handling because `copyTo` can't copy symlinks between file systems
+        val linkTarget = actualFile.readSymbolicLink()
+        assertThat(linkTarget).isRelative
+        Files.createSymbolicLink(expectedFile, linkTarget.toString().toPath())
+      } else {
+        actualFile.copyTo(expectedFile)
+      }
+      Assertions.fail("Created missing expected file `$relativeFilePath`.")
+    }
+  }
+}

pkl-doc/src/test/kotlin/org/pkl/doc/JavaExecutableTest.kt

@@ -0,0 +1,41 @@
+/*
+ * Copyright © 2025 Apple Inc. and the Pkl project authors. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.pkl.doc
+
+import org.junit.jupiter.api.condition.DisabledIfSystemProperty
+import org.junit.jupiter.api.condition.EnabledIfSystemProperty
+import org.junit.jupiter.params.ParameterizedTest
+import org.junit.jupiter.params.provider.MethodSource
+import org.pkl.commons.test.Executables
+
+// need both annotations for this to work (see https://stackoverflow.com/a/63252081)
+@EnabledIfSystemProperty(named = "org.pkl.doc.JavaExecutableTest", matches = "true")
+@DisabledIfSystemProperty(named = "org.pkl.doc.JavaExecutableTest", matches = "(?!true)")
+class JavaExecutableTest {
+  companion object {
+    val helper = DocGeneratorTestHelper()
+
+    @JvmStatic
+    private fun generateDocs(): List<String> =
+      helper.generateDocsWithCli(Executables.pkldoc.javaExecutable)
+  }
+
+  @ParameterizedTest()
+  @MethodSource("generateDocs")
+  fun test(relativePath: String) {
+    DocTestUtils.testExpectedFile(helper.expectedOutputDir, helper.actualOutputDir, relativePath)
+  }
+}

pkl-doc/src/test/kotlin/org/pkl/doc/NativeExecutableTest.kt

@@ -0,0 +1,42 @@
+/*
+ * Copyright © 2025 Apple Inc. and the Pkl project authors. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.pkl.doc
+
+import org.junit.jupiter.api.condition.DisabledIfSystemProperty
+import org.junit.jupiter.api.condition.EnabledIfSystemProperty
+import org.junit.jupiter.params.ParameterizedTest
+import org.junit.jupiter.params.provider.MethodSource
+import org.pkl.commons.test.Executables
+
+// need both annotations for this to work (see https://stackoverflow.com/a/63252081)
+@EnabledIfSystemProperty(named = "org.pkl.doc.NativeExecutableTest", matches = "true")
+@DisabledIfSystemProperty(named = "org.pkl.doc.NativeExecutableTest", matches = "(?!true)")
+class NativeExecutableTest {
+  companion object {
+    val helper = DocGeneratorTestHelper()
+
+    @JvmStatic
+    private fun generateDocs(): List<String> {
+      return helper.generateDocsWithCli(Executables.pkldoc.firstExistingNative)
+    }
+  }
+
+  @ParameterizedTest()
+  @MethodSource("generateDocs")
+  fun test(relativePath: String) {
+    DocTestUtils.testExpectedFile(helper.expectedOutputDir, helper.actualOutputDir, relativePath)
+  }
+}

pkl-doc/src/test/kotlin/org/pkl/doc/SearchTest.kt

@@ -26,17 +26,17 @@ import org.junit.jupiter.api.AfterAll
 import org.junit.jupiter.api.Test
 import org.pkl.commons.cli.CliBaseOptions
 import org.pkl.commons.readString
-import org.pkl.doc.CliDocGeneratorTest.Companion.package1InputModules
-import org.pkl.doc.CliDocGeneratorTest.Companion.package1PackageModule
 
 class SearchTest {
   companion object {
     private val tempFileSystem = lazy { Jimfs.newFileSystem(Configuration.unix()) }
 
+    private val helper = DocGeneratorTestHelper()
+
     private val jsContext = lazy {
       // reuse CliDocGeneratorTest's input files (src/test/files/DocGeneratorTest/input)
-      val packageModule: URI = package1PackageModule
-      val inputModules: List<URI> = package1InputModules
+      val packageModule: URI = helper.package1PackageModule
+      val inputModules: List<URI> = helper.package1InputModules
 
       val pkldocDir = tempFileSystem.value.rootDirectories.first()
 

pkl-doc/src/test/kotlin/org/pkl/doc/TestUtils.kt

@@ -0,0 +1,93 @@
+/*
+ * Copyright © 2025 Apple Inc. and the Pkl project authors. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.pkl.doc
+
+import com.google.common.jimfs.Configuration
+import com.google.common.jimfs.Jimfs
+import java.net.URI
+import java.nio.file.FileSystem
+import java.nio.file.Path
+import kotlin.io.path.createDirectories
+import kotlin.io.path.exists
+import org.pkl.commons.test.FileTestUtils
+import org.pkl.commons.test.listFilesRecursively
+import org.pkl.core.util.IoUtils
+
+class TestUtils {
+  val tempFileSystem: FileSystem by lazy { Jimfs.newFileSystem(Configuration.unix()) }
+
+  val tmpOutputDir by lazy { tempFileSystem.getPath("/work/output").apply { createDirectories() } }
+
+  val projectDir = FileTestUtils.rootProjectDir.resolve("pkl-doc")
+
+  val inputDir: Path by lazy {
+    projectDir.resolve("src/test/files/DocGeneratorTest/input").apply { assert(exists()) }
+  }
+
+  val docsiteModule: URI by lazy {
+    inputDir.resolve("docsite-info.pkl").apply { assert(exists()) }.toUri()
+  }
+
+  internal val package1PackageModule: URI by lazy {
+    inputDir.resolve("com.package1/doc-package-info.pkl").apply { assert(exists()) }.toUri()
+  }
+
+  val package2PackageModule: URI by lazy {
+    inputDir.resolve("com.package2/doc-package-info.pkl").apply { assert(exists()) }.toUri()
+  }
+
+  internal val package1InputModules: List<URI> by lazy {
+    inputDir
+      .resolve("com.package1")
+      .listFilesRecursively()
+      .filter { it.fileName.toString() != "doc-package-info.pkl" }
+      .map { it.toUri() }
+  }
+
+  val package2InputModules: List<URI> by lazy {
+    inputDir
+      .resolve("com.package2")
+      .listFilesRecursively()
+      .filter { it.fileName.toString() != "doc-package-info.pkl" }
+      .map { it.toUri() }
+  }
+
+  val expectedOutputDir: Path by lazy {
+    projectDir.resolve("src/test/files/DocGeneratorTest/output").createDirectories()
+  }
+
+  val expectedOutputFiles: List<Path> by lazy { expectedOutputDir.listFilesRecursively() }
+
+  val actualOutputDir: Path by lazy { tempFileSystem.getPath("/work/DocGeneratorTest") }
+
+  val actualOutputFiles: List<Path> by lazy { actualOutputDir.listFilesRecursively() }
+
+  val expectedRelativeOutputFiles: List<String> by lazy {
+    expectedOutputFiles.map { path ->
+      IoUtils.toNormalizedPathString(expectedOutputDir.relativize(path)).let { str ->
+        // Git will by default clone symlinks as shortcuts on Windows, and shortcuts have a
+        // `.lnk` extension.
+        if (IoUtils.isWindows() && str.endsWith(".lnk")) str.dropLast(4) else str
+      }
+    }
+  }
+
+  val actualRelativeOutputFiles: List<String> by lazy {
+    actualOutputFiles.map { IoUtils.toNormalizedPathString(actualOutputDir.relativize(it)) }
+  }
+
+  val binaryFileExtensions = setOf("woff2", "png", "svg")
+}

pkl-server/src/test/kotlin/org/pkl/server/NativeServerTest.kt

@@ -17,7 +17,7 @@ package org.pkl.server
 
 import org.junit.jupiter.api.AfterEach
 import org.junit.jupiter.api.BeforeEach
-import org.pkl.commons.test.PklExecutablePaths
+import org.pkl.commons.test.Executables
 import org.pkl.core.messaging.MessageTransports
 
 class NativeServerTest : AbstractServerTest() {
@@ -26,7 +26,7 @@ class NativeServerTest : AbstractServerTest() {
 
   @BeforeEach
   fun beforeEach() {
-    val executable = PklExecutablePaths.firstExisting.toString()
+    val executable = Executables.pkl.firstExistingNative.toString()
     server = ProcessBuilder(executable, "server").start()
     client =
       TestTransport(