Improve configuration and tests for native-image (#509)

* Don't expose JDK internal classes; instead solve msgpack issue with `--initialize-at-run-time`.
* Use quick build mode for non-release builds:  40% faster compilation, 20% smaller executable.
* Remove options that were commented out.
* Also run ServerTest against native executable

.circleci/config.yml

@@ -12,7 +12,7 @@ jobs:
     - run:
         command: |-
           export PATH=~/staticdeps/bin:$PATH
-          ./gradlew --info --stacktrace -DtestReportsDir=${HOME}/test-results -DreleaseBuild=true pkl-cli:macExecutableAmd64 pkl-core:testMacExecutableAmd64
+          ./gradlew --info --stacktrace -DtestReportsDir=${HOME}/test-results -DreleaseBuild=true pkl-cli:macExecutableAmd64 pkl-core:testMacExecutableAmd64 pkl-server:testNative
         name: gradle buildNative
     - persist_to_workspace:
         root: '.'
@@ -88,7 +88,7 @@ jobs:
     - run:
         command: |-
           export PATH=~/staticdeps/bin:$PATH
-          ./gradlew --info --stacktrace -DtestReportsDir=${HOME}/test-results -DreleaseBuild=true pkl-cli:linuxExecutableAmd64 pkl-core:testLinuxExecutableAmd64
+          ./gradlew --info --stacktrace -DtestReportsDir=${HOME}/test-results -DreleaseBuild=true pkl-cli:linuxExecutableAmd64 pkl-core:testLinuxExecutableAmd64 pkl-server:testNative
         name: gradle buildNative
     - persist_to_workspace:
         root: '.'
@@ -108,7 +108,7 @@ jobs:
     - run:
         command: |-
           export PATH=~/staticdeps/bin:$PATH
-          ./gradlew --info --stacktrace -DtestReportsDir=${HOME}/test-results -DreleaseBuild=true pkl-cli:macExecutableAarch64 pkl-core:testMacExecutableAarch64
+          ./gradlew --info --stacktrace -DtestReportsDir=${HOME}/test-results -DreleaseBuild=true pkl-cli:macExecutableAarch64 pkl-core:testMacExecutableAarch64 pkl-server:testNative
         name: gradle buildNative
     - persist_to_workspace:
         root: '.'
@@ -168,7 +168,7 @@ jobs:
     - run:
         command: |-
           export PATH=~/staticdeps/bin:$PATH
-          ./gradlew --info --stacktrace -DtestReportsDir=${HOME}/test-results -DreleaseBuild=true pkl-cli:linuxExecutableAarch64 pkl-core:testLinuxExecutableAarch64
+          ./gradlew --info --stacktrace -DtestReportsDir=${HOME}/test-results -DreleaseBuild=true pkl-cli:linuxExecutableAarch64 pkl-core:testLinuxExecutableAarch64 pkl-server:testNative
         name: gradle buildNative
     - persist_to_workspace:
         root: '.'
@@ -245,7 +245,7 @@ jobs:
     - run:
         command: |-
           export PATH=~/staticdeps/bin:$PATH
-          ./gradlew --info --stacktrace -DtestReportsDir=${HOME}/test-results -DreleaseBuild=true pkl-cli:alpineExecutableAmd64 pkl-core:testAlpineExecutableAmd64
+          ./gradlew --info --stacktrace -DtestReportsDir=${HOME}/test-results -DreleaseBuild=true pkl-cli:alpineExecutableAmd64 pkl-core:testAlpineExecutableAmd64 pkl-server:testNative
         name: gradle buildNative
     - persist_to_workspace:
         root: '.'
@@ -265,7 +265,7 @@ jobs:
     - run:
         command: |-
           export PATH=~/staticdeps/bin:$PATH
-          ./gradlew --info --stacktrace -DtestReportsDir=${HOME}/test-results -DreleaseBuild=true pkl-cli:windowsExecutableAmd64 pkl-core:testWindowsExecutableAmd64
+          ./gradlew --info --stacktrace -DtestReportsDir=${HOME}/test-results -DreleaseBuild=true pkl-cli:windowsExecutableAmd64 pkl-core:testWindowsExecutableAmd64 pkl-server:testNative
         name: gradle buildNative
         shell: bash.exe
     - persist_to_workspace:
@@ -288,7 +288,7 @@ jobs:
     - run:
         command: |-
           export PATH=~/staticdeps/bin:$PATH
-          ./gradlew --info --stacktrace -DtestReportsDir=${HOME}/test-results pkl-cli:macExecutableAmd64 pkl-core:testMacExecutableAmd64
+          ./gradlew --info --stacktrace -DtestReportsDir=${HOME}/test-results pkl-cli:macExecutableAmd64 pkl-core:testMacExecutableAmd64 pkl-server:testNative
         name: gradle buildNative
     - persist_to_workspace:
         root: '.'
@@ -364,7 +364,7 @@ jobs:
     - run:
         command: |-
           export PATH=~/staticdeps/bin:$PATH
-          ./gradlew --info --stacktrace -DtestReportsDir=${HOME}/test-results pkl-cli:linuxExecutableAmd64 pkl-core:testLinuxExecutableAmd64
+          ./gradlew --info --stacktrace -DtestReportsDir=${HOME}/test-results pkl-cli:linuxExecutableAmd64 pkl-core:testLinuxExecutableAmd64 pkl-server:testNative
         name: gradle buildNative
     - persist_to_workspace:
         root: '.'
@@ -384,7 +384,7 @@ jobs:
     - run:
         command: |-
           export PATH=~/staticdeps/bin:$PATH
-          ./gradlew --info --stacktrace -DtestReportsDir=${HOME}/test-results pkl-cli:macExecutableAarch64 pkl-core:testMacExecutableAarch64
+          ./gradlew --info --stacktrace -DtestReportsDir=${HOME}/test-results pkl-cli:macExecutableAarch64 pkl-core:testMacExecutableAarch64 pkl-server:testNative
         name: gradle buildNative
     - persist_to_workspace:
         root: '.'
@@ -444,7 +444,7 @@ jobs:
     - run:
         command: |-
           export PATH=~/staticdeps/bin:$PATH
-          ./gradlew --info --stacktrace -DtestReportsDir=${HOME}/test-results pkl-cli:linuxExecutableAarch64 pkl-core:testLinuxExecutableAarch64
+          ./gradlew --info --stacktrace -DtestReportsDir=${HOME}/test-results pkl-cli:linuxExecutableAarch64 pkl-core:testLinuxExecutableAarch64 pkl-server:testNative
         name: gradle buildNative
     - persist_to_workspace:
         root: '.'
@@ -521,7 +521,7 @@ jobs:
     - run:
         command: |-
           export PATH=~/staticdeps/bin:$PATH
-          ./gradlew --info --stacktrace -DtestReportsDir=${HOME}/test-results pkl-cli:alpineExecutableAmd64 pkl-core:testAlpineExecutableAmd64
+          ./gradlew --info --stacktrace -DtestReportsDir=${HOME}/test-results pkl-cli:alpineExecutableAmd64 pkl-core:testAlpineExecutableAmd64 pkl-server:testNative
         name: gradle buildNative
     - persist_to_workspace:
         root: '.'
@@ -541,7 +541,7 @@ jobs:
     - run:
         command: |-
           export PATH=~/staticdeps/bin:$PATH
-          ./gradlew --info --stacktrace -DtestReportsDir=${HOME}/test-results pkl-cli:windowsExecutableAmd64 pkl-core:testWindowsExecutableAmd64
+          ./gradlew --info --stacktrace -DtestReportsDir=${HOME}/test-results pkl-cli:windowsExecutableAmd64 pkl-core:testWindowsExecutableAmd64 pkl-server:testNative
         name: gradle buildNative
         shell: bash.exe
     - persist_to_workspace:

.circleci/jobs/BuildNativeJob.pkl

@@ -126,7 +126,7 @@ steps {
     }
     command = #"""
       export PATH=~/staticdeps/bin:$PATH
-      ./gradlew \#(module.gradleArgs) pkl-cli:\#(jobName) pkl-core:test\#(jobName.capitalize())
+      ./gradlew \#(module.gradleArgs) pkl-cli:\#(jobName) pkl-core:test\#(jobName.capitalize()) pkl-server:testNative
       """#
   }
   new Config.PersistToWorkspaceStep {

pkl-cli/pkl-cli.gradle.kts

@@ -146,10 +146,17 @@ fun Exec.configureExecutable(
   outputFile: Provider<RegularFile>,
   extraArgs: List<String> = listOf()
 ) {
-  inputs.files(sourceSets.main.map { it.output }).withPropertyName("mainSourceSets").withPathSensitivity(PathSensitivity.RELATIVE)
+  inputs.files(sourceSets.main.map { it.output })
-  inputs.files(configurations.runtimeClasspath).withPropertyName("runtimeClasspath").withNormalizer(ClasspathNormalizer::class)
+    .withPropertyName("mainSourceSets")
+    .withPathSensitivity(PathSensitivity.RELATIVE)
+  inputs.files(configurations.runtimeClasspath)
+    .withPropertyName("runtimeClasspath")
+    .withNormalizer(ClasspathNormalizer::class)
   val nativeImageCommandName = if (buildInfo.os.isWindows) "native-image.cmd" else "native-image"
-  inputs.files(file(graalVm.baseDir).resolve("bin/$nativeImageCommandName")).withPropertyName("graalVmNativeImage").withPathSensitivity(PathSensitivity.ABSOLUTE)
+  inputs.files(file(graalVm.baseDir)
+    .resolve("bin/$nativeImageCommandName"))
+    .withPropertyName("graalVmNativeImage")
+    .withPathSensitivity(PathSensitivity.ABSOLUTE)
   outputs.file(outputFile)
   outputs.cacheIf { true }
 
@@ -165,47 +172,24 @@ fun Exec.configureExecutable(
         // that the "initialize everything at build time" *CLI* option is likely here to stay
         "--initialize-at-build-time="
         // needed for messagepack-java (see https://github.com/msgpack/msgpack-java/issues/600)
-        ,"--add-opens=java.base/java.nio=ALL-UNNAMED"
+        ,"--initialize-at-run-time=org.msgpack.core.buffer.DirectBufferAccess"
-        ,"--add-opens=java.base/sun.nio.ch=ALL-UNNAMED"
         ,"--no-fallback"
         ,"-H:IncludeResources=org/pkl/core/stdlib/.*\\.pkl"
         ,"-H:IncludeResources=org/jline/utils/.*"
         ,"-H:IncludeResources=org/pkl/certs/PklCARoots.pem"
-        //,"-H:IncludeResources=org/pkl/core/Release.properties"
         ,"-H:IncludeResourceBundles=org.pkl.core.errorMessages"
         ,"--macro:truffle"
         ,"-H:Class=org.pkl.cli.Main"
         ,"-H:Name=${outputFile.get().asFile.name}"
-        //,"--native-image-info"
-        //,"-Dpolyglot.image-build-time.PreinitializeContexts=pkl"
         // the actual limit (currently) used by native-image is this number + 1400 (idea is to compensate for Truffle's own nodes)
         ,"-H:MaxRuntimeCompileMethods=1800"
         ,"-H:+EnforceMaxRuntimeCompileMethods"
         ,"--enable-url-protocols=http,https"
-        //,"--install-exit-handlers"
         ,"-H:+ReportExceptionStackTraces"
-        ,"-H:-ParseRuntimeOptions" // disable automatic support for JVM CLI options (puts our main class in full control of argument parsing)
+        // disable automatic support for JVM CLI options (puts our main class in full control of argument parsing)
-        //,"-H:+PrintAnalysisCallTree"
+        ,"-H:-ParseRuntimeOptions"
-        //,"-H:PrintAnalysisCallTreeType=CSV"
+        // quick build mode: 40% faster compilation, 20% smaller (but presumably also slower) executable
-        //,"-H:+PrintImageObjectTree"
+        ,if (!buildInfo.isReleaseBuild) "-Ob" else ""
-        //,"--features=org.pkl.cli.svm.InitFeature"
-        //,"-H:Dump=:2"
-        //,"-H:MethodFilter=ModuleCache.getOrLoad*,VmLanguage.loadModule"
-        //,"-g"
-        //,"-verbose"
-        //,"--debug-attach"
-        //,"-H:+AllowVMInspection"
-        //,"-H:+PrintHeapHistogram"
-        //,"-H:+ReportDeletedElementsAtRuntime"
-        //,"-H:+PrintMethodHistogram"
-        //,"-H:+PrintRuntimeCompileMethods"
-        //,"-H:NumberOfThreads=1"
-        //,"-J-Dtruffle.TruffleRuntime=com.oracle.truffle.api.impl.DefaultTruffleRuntime"
-        //,"-J-Dcom.oracle.truffle.aot=true"
-        //,"-J:-ea"
-        //,"-J:-esa"
-        // for use with https://www.graalvm.org/docs/tools/dashboard/
-        //,"-H:DashboardDump=dashboard.dump", "-H:+DashboardAll"
         // native-image rejects non-existing class path entries -> filter
         ,"--class-path"
         ,((sourceSets.main.get().output + configurations.runtimeClasspath.get())

pkl-commons-test/src/main/kotlin/org/pkl/commons/test/PklExecutablePaths.kt

@@ -0,0 +1,47 @@
+/**
+ * Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.pkl.commons.test
+
+import java.nio.file.Files
+import java.nio.file.Path
+import org.pkl.commons.test.FileTestUtils.rootProjectDir
+
+object PklExecutablePaths {
+  val macAarch64: Path = executablePath("pkl-macos-aarch64")
+  val macAmd64: Path = executablePath("pkl-macos-amd64")
+  val linuxAarch64: Path = executablePath("pkl-linux-aarch64")
+  val linuxAmd64: Path = executablePath("pkl-linux-amd64")
+  val alpineAmd64: Path = executablePath("pkl-alpine-linux-amd64")
+  val windowsAmd64: Path = executablePath("pkl-windows-amd64.exe")
+
+  // order (aarch64 before amd64, linux before alpine) affects [firstExisting]
+  val all: List<Path> =
+    listOf(macAarch64, macAmd64, linuxAarch64, linuxAmd64, alpineAmd64, windowsAmd64)
+
+  val existing: List<Path>
+    get() = all.filter(Files::exists)
+
+  val firstExisting: Path
+    get() =
+      existing.firstOrNull()
+        ?: throw AssertionError(
+          "Native executable not found on system. " +
+            "To fix this problem, run `./gradlew assembleNative`."
+        )
+
+  private fun executablePath(name: String): Path =
+    rootProjectDir.resolve("pkl-cli/build/executable").resolve(name)
+}

pkl-core/pkl-core.gradle.kts

@@ -154,10 +154,7 @@ tasks.compileKotlin {
 }
 
 tasks.test {
-  inputs.dir("src/test/files/LanguageSnippetTests/input").withPropertyName("languageSnippetTestsInput").withPathSensitivity(PathSensitivity.RELATIVE)
+  configureTest()
-  inputs.dir("src/test/files/LanguageSnippetTests/input-helper").withPropertyName("languageSnippetTestsInputHelper").withPathSensitivity(PathSensitivity.RELATIVE)
-  inputs.dir("src/test/files/LanguageSnippetTests/output").withPropertyName("languageSnippetTestsOutput").withPathSensitivity(PathSensitivity.RELATIVE)
-
   useJUnitPlatform {
     excludeEngines("MacAmd64LanguageSnippetTestsEngine")
     excludeEngines("MacAarch64LanguageSnippetTestsEngine")
@@ -168,11 +165,7 @@ tasks.test {
 }
 
 val testJavaExecutable by tasks.registering(Test::class) {
-  inputs.dir("src/test/files/LanguageSnippetTests/input").withPropertyName("languageSnippetTestsInput").withPathSensitivity(PathSensitivity.RELATIVE)
+  configureExecutableTest("LanguageSnippetTestsEngine")
-  inputs.dir("src/test/files/LanguageSnippetTests/input-helper").withPropertyName("languageSnippetTestsInputHelper").withPathSensitivity(PathSensitivity.RELATIVE)
-  inputs.dir("src/test/files/LanguageSnippetTests/output").withPropertyName("languageSnippetTestsOutput").withPathSensitivity(PathSensitivity.RELATIVE)
-
-  testClassesDirs = files(tasks.test.get().testClassesDirs)
   classpath =
     // compiled test classes
     sourceSets.test.get().output +
@@ -182,10 +175,6 @@ val testJavaExecutable by tasks.registering(Test::class) {
     // (test dependencies that are also main dependencies must already be contained in java executable;
     // to verify that we don't want to include them here)
     (configurations.testRuntimeClasspath.get() - configurations.runtimeClasspath.get())
-
-  useJUnitPlatform {
-    includeEngines("LanguageSnippetTestsEngine")
-  }
 }
 
 tasks.check {
@@ -194,92 +183,32 @@ tasks.check {
 
 val testMacExecutableAmd64 by tasks.registering(Test::class) {
   dependsOn(":pkl-cli:macExecutableAmd64")
-
+  configureExecutableTest("MacAmd64LanguageSnippetTestsEngine")
-  inputs.dir("src/test/files/LanguageSnippetTests/input").withPropertyName("languageSnippetTestsInput").withPathSensitivity(PathSensitivity.RELATIVE)
-  inputs.dir("src/test/files/LanguageSnippetTests/input-helper").withPropertyName("languageSnippetTestsInputHelper").withPathSensitivity(PathSensitivity.RELATIVE)
-  inputs.dir("src/test/files/LanguageSnippetTests/output").withPropertyName("languageSnippetTestsOutput").withPathSensitivity(PathSensitivity.RELATIVE)
-
-  testClassesDirs = files(tasks.test.get().testClassesDirs)
-  classpath = tasks.test.get().classpath
-
-  useJUnitPlatform {
-    includeEngines("MacAmd64LanguageSnippetTestsEngine")
-  }
 }
 
 val testMacExecutableAarch64 by tasks.registering(Test::class) {
   dependsOn(":pkl-cli:macExecutableAarch64")
-
+  configureExecutableTest("MacAarch64LanguageSnippetTestsEngine")
-  inputs.dir("src/test/files/LanguageSnippetTests/input")
-  inputs.dir("src/test/files/LanguageSnippetTests/input-helper")
-  inputs.dir("src/test/files/LanguageSnippetTests/output")
-
-  testClassesDirs = files(tasks.test.get().testClassesDirs)
-  classpath = tasks.test.get().classpath
-
-  useJUnitPlatform {
-    includeEngines("MacAarch64LanguageSnippetTestsEngine")
-  }
 }
 
 val testLinuxExecutableAmd64 by tasks.registering(Test::class) {
   dependsOn(":pkl-cli:linuxExecutableAmd64")
-
+  configureExecutableTest("LinuxAmd64LanguageSnippetTestsEngine")
-  inputs.dir("src/test/files/LanguageSnippetTests/input")
-  inputs.dir("src/test/files/LanguageSnippetTests/input-helper")
-  inputs.dir("src/test/files/LanguageSnippetTests/output")
-
-  testClassesDirs = files(tasks.test.get().testClassesDirs)
-  classpath = tasks.test.get().classpath
-
-  useJUnitPlatform {
-    includeEngines("LinuxAmd64LanguageSnippetTestsEngine")
-  }
 }
 
 val testLinuxExecutableAarch64 by tasks.registering(Test::class) {
   dependsOn(":pkl-cli:linuxExecutableAarch64")
-
+  configureExecutableTest("LinuxAarch64LanguageSnippetTestsEngine")
-  inputs.dir("src/test/files/LanguageSnippetTests/input")
-  inputs.dir("src/test/files/LanguageSnippetTests/input-helper")
-  inputs.dir("src/test/files/LanguageSnippetTests/output")
-
-  testClassesDirs = files(tasks.test.get().testClassesDirs)
-  classpath = tasks.test.get().classpath
-
-  useJUnitPlatform {
-    includeEngines("LinuxAarch64LanguageSnippetTestsEngine")
-  }
 }
 
 val testAlpineExecutableAmd64 by tasks.registering(Test::class) {
   dependsOn(":pkl-cli:alpineExecutableAmd64")
-
+  configureExecutableTest("AlpineLanguageSnippetTestsEngine")
-  inputs.dir("src/test/files/LanguageSnippetTests/input")
-  inputs.dir("src/test/files/LanguageSnippetTests/input-helper")
-  inputs.dir("src/test/files/LanguageSnippetTests/output")
-
-  testClassesDirs = files(tasks.test.get().testClassesDirs)
-  classpath = tasks.test.get().classpath
-
-  useJUnitPlatform {
-    includeEngines("AlpineLanguageSnippetTestsEngine")
-  }
 }
 
 val testWindowsExecutableAmd64 by tasks.registering(Test::class) {
   dependsOn(":pkl-cli:windowsExecutableAmd64")
-
+  configureExecutableTest("WindowsLanguageSnippetTestsEngine")
-  inputs.dir("src/test/files/LanguageSnippetTests/input")
-  inputs.dir("src/test/files/LanguageSnippetTests/input-helper")
-  inputs.dir("src/test/files/LanguageSnippetTests/output")
-
-  testClassesDirs = files(tasks.test.get().testClassesDirs)
-  classpath = tasks.test.get().classpath
-
-  useJUnitPlatform {
-    includeEngines("WindowsLanguageSnippetTestsEngine")
-  }
 }
 
 tasks.testNative {
@@ -316,3 +245,24 @@ spotless {
     target(files("src/main/antlr/PklParser.g4", "src/main/antlr/PklLexer.g4"))
   }
 }
+
+private fun Test.configureTest() {
+  inputs.dir("src/test/files/LanguageSnippetTests/input")
+    .withPropertyName("languageSnippetTestsInput")
+    .withPathSensitivity(PathSensitivity.RELATIVE)
+  inputs.dir("src/test/files/LanguageSnippetTests/input-helper")
+    .withPropertyName("languageSnippetTestsInputHelper")
+    .withPathSensitivity(PathSensitivity.RELATIVE)
+  inputs.dir("src/test/files/LanguageSnippetTests/output")
+    .withPropertyName("languageSnippetTestsOutput")
+    .withPathSensitivity(PathSensitivity.RELATIVE)
+}
+
+private fun Test.configureExecutableTest(engineName: String) {
+  configureTest()
+  testClassesDirs = files(tasks.test.get().testClassesDirs)
+  classpath = tasks.test.get().classpath
+  useJUnitPlatform {
+    includeEngines(engineName)
+  }
+}

pkl-core/src/test/kotlin/org/pkl/core/LanguageSnippetTestsEngine.kt

@@ -7,6 +7,7 @@ import org.junit.platform.engine.support.descriptor.EngineDescriptor
 import org.pkl.commons.test.FileTestUtils
 import org.pkl.commons.test.InputOutputTestEngine
 import org.pkl.commons.test.PackageServer
+import org.pkl.commons.test.PklExecutablePaths
 import org.pkl.core.http.HttpClient
 import org.pkl.core.project.Project
 import org.pkl.core.util.IoUtils
@@ -236,31 +237,31 @@ abstract class AbstractNativeLanguageSnippetTestsEngine : AbstractLanguageSnippe
 }
 
 class MacAmd64LanguageSnippetTestsEngine : AbstractNativeLanguageSnippetTestsEngine() {
-  override val pklExecutablePath: Path = rootProjectDir.resolve("pkl-cli/build/executable/pkl-macos-amd64")
+  override val pklExecutablePath: Path = PklExecutablePaths.macAmd64
   override val testClass: KClass<*> = MacLanguageSnippetTests::class
 }
 
 class MacAarch64LanguageSnippetTestsEngine : AbstractNativeLanguageSnippetTestsEngine() {
-  override val pklExecutablePath: Path = rootProjectDir.resolve("pkl-cli/build/executable/pkl-macos-aarch64")
+  override val pklExecutablePath: Path = PklExecutablePaths.macAarch64
   override val testClass: KClass<*> = MacLanguageSnippetTests::class
 }
 
 class LinuxAmd64LanguageSnippetTestsEngine : AbstractNativeLanguageSnippetTestsEngine() {
-  override val pklExecutablePath: Path = rootProjectDir.resolve("pkl-cli/build/executable/pkl-linux-amd64")
+  override val pklExecutablePath: Path = PklExecutablePaths.linuxAmd64
   override val testClass: KClass<*> = LinuxLanguageSnippetTests::class
 }
 
 class LinuxAarch64LanguageSnippetTestsEngine : AbstractNativeLanguageSnippetTestsEngine() {
-  override val pklExecutablePath: Path = rootProjectDir.resolve("pkl-cli/build/executable/pkl-linux-aarch64")
+  override val pklExecutablePath: Path = PklExecutablePaths.linuxAarch64
   override val testClass: KClass<*> = LinuxLanguageSnippetTests::class
 }
 
 class AlpineLanguageSnippetTestsEngine : AbstractNativeLanguageSnippetTestsEngine() {
-  override val pklExecutablePath: Path = rootProjectDir.resolve("pkl-cli/build/executable/pkl-alpine-linux-amd64")
+  override val pklExecutablePath: Path = PklExecutablePaths.alpineAmd64
   override val testClass: KClass<*> = AlpineLanguageSnippetTests::class
 }
 
 class WindowsLanguageSnippetTestsEngine : AbstractNativeLanguageSnippetTestsEngine() {
-  override val pklExecutablePath: Path = rootProjectDir.resolve("pkl-cli/build/executable/pkl-windows-amd64.exe")
+  override val pklExecutablePath: Path = PklExecutablePaths.windowsAmd64
   override val testClass: KClass<*> = WindowsLanguageSnippetTests::class
 }

pkl-server/pkl-server.gradle.kts

@@ -2,6 +2,7 @@ plugins {
   pklAllProjects
   pklJavaLibrary
   pklKotlinLibrary
+  pklNativeBuild
 }
 
 dependencies {
@@ -14,6 +15,24 @@ dependencies {
 }
 
 tasks.test {
-  inputs.dir("src/test/files/SnippetTests/input").withPropertyName("snippetTestsInput").withPathSensitivity(PathSensitivity.RELATIVE)
+  inputs.dir("src/test/files/SnippetTests/input")
-  inputs.dir("src/test/files/SnippetTests/output").withPropertyName("snippetTestsOutput").withPathSensitivity(PathSensitivity.RELATIVE)
+    .withPropertyName("snippetTestsInput")
+    .withPathSensitivity(PathSensitivity.RELATIVE)
+  inputs.dir("src/test/files/SnippetTests/output")
+    .withPropertyName("snippetTestsOutput")
+    .withPathSensitivity(PathSensitivity.RELATIVE)
+  exclude("**/NativeServerTest.*")
 }
+
+val nativeTest by tasks.registering(Test::class) {
+  dependsOn(":pkl-cli:assembleNative")
+  testClassesDirs = files(tasks.test.get().testClassesDirs)
+  classpath = tasks.test.get().classpath
+  include("**/NativeServerTest.*")
+}
+
+val testNative by tasks.existing
+testNative {
+  dependsOn(nativeTest)
+}
+

pkl-server/src/test/kotlin/org/pkl/server/AbstractServerTest.kt

@@ -15,8 +15,6 @@
  */
 package org.pkl.server
 
-import java.io.PipedInputStream
-import java.io.PipedOutputStream
 import java.net.URI
 import java.nio.file.Path
 import java.util.concurrent.ExecutorService
@@ -27,21 +25,20 @@ import kotlin.io.path.outputStream
 import kotlin.io.path.writeText
 import org.assertj.core.api.Assertions.assertThat
 import org.junit.jupiter.api.AfterAll
-import org.junit.jupiter.api.AfterEach
-import org.junit.jupiter.api.BeforeEach
 import org.junit.jupiter.api.Test
 import org.junit.jupiter.api.io.TempDir
 import org.msgpack.core.MessagePack
 import org.pkl.commons.test.PackageServer
-import org.pkl.core.http.HttpClient
 import org.pkl.core.module.PathElement
 
-class ServerTest {
+abstract class AbstractServerTest {
-  companion object {
-    private const val useDirectTransport = false
 
-    private val executor: ExecutorService =
+  companion object {
-      if (useDirectTransport) {
+    /** Set to `true` to bypass messagepack serialization when running [JvmServerTest]. */
+    const val USE_DIRECT_TRANSPORT = false
+
+    val executor: ExecutorService =
+      if (USE_DIRECT_TRANSPORT) {
         createDirectExecutor()
       } else {
         Executors.newCachedThreadPool()
@@ -49,38 +46,12 @@ class ServerTest {
 
     @AfterAll
     @JvmStatic
-    @Suppress("unused")
     fun afterAll() {
       executor.shutdown()
     }
   }
 
-  private val transports: Pair<MessageTransport, MessageTransport> = run {
+  abstract val client: TestTransport
-    if (useDirectTransport) {
-      MessageTransports.direct()
-    } else {
-      val in1 = PipedInputStream()
-      val out1 = PipedOutputStream(in1)
-      val in2 = PipedInputStream()
-      val out2 = PipedOutputStream(in2)
-      MessageTransports.stream(in1, out2) to MessageTransports.stream(in2, out1)
-    }
-  }
-
-  private val client: TestTransport = TestTransport(transports.first)
-  private val server: Server = Server(transports.second, HttpClient.dummyClient())
-
-  @BeforeEach
-  fun before() {
-    executor.execute { server.start() }
-    executor.execute { client.start() }
-  }
-
-  @AfterEach
-  fun after() {
-    client.close()
-    server.close()
-  }
 
   @Test
   fun `create and close evaluator`() {

pkl-server/src/test/kotlin/org/pkl/server/JvmServerTest.kt

@@ -0,0 +1,51 @@
+/**
+ * Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.pkl.server
+
+import java.io.PipedInputStream
+import java.io.PipedOutputStream
+import org.junit.jupiter.api.AfterEach
+import org.junit.jupiter.api.BeforeEach
+import org.pkl.core.http.HttpClient
+
+class JvmServerTest : AbstractServerTest() {
+  private val transports: Pair<MessageTransport, MessageTransport> = run {
+    if (USE_DIRECT_TRANSPORT) {
+      MessageTransports.direct()
+    } else {
+      val in1 = PipedInputStream()
+      val out1 = PipedOutputStream(in1)
+      val in2 = PipedInputStream()
+      val out2 = PipedOutputStream(in2)
+      MessageTransports.stream(in1, out2) to MessageTransports.stream(in2, out1)
+    }
+  }
+
+  override val client: TestTransport = TestTransport(transports.first)
+  private val server: Server = Server(transports.second, HttpClient.dummyClient())
+
+  @BeforeEach
+  fun beforeEach() {
+    executor.execute { server.start() }
+    executor.execute { client.start() }
+  }
+
+  @AfterEach
+  fun after() {
+    client.close()
+    server.close()
+  }
+}

pkl-server/src/test/kotlin/org/pkl/server/NativeServerTest.kt

@@ -0,0 +1,39 @@
+/**
+ * Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.pkl.server
+
+import org.junit.jupiter.api.AfterEach
+import org.junit.jupiter.api.BeforeEach
+import org.pkl.commons.test.PklExecutablePaths
+
+class NativeServerTest : AbstractServerTest() {
+  private lateinit var server: Process
+  override lateinit var client: TestTransport
+
+  @BeforeEach
+  fun beforeEach() {
+    val executable = PklExecutablePaths.firstExisting.toString()
+    server = ProcessBuilder(executable, "server").start()
+    client = TestTransport(MessageTransports.stream(server.inputStream, server.outputStream))
+    executor.execute { client.start() }
+  }
+
+  @AfterEach
+  fun afterEach() {
+    client.close()
+    server.destroy()
+  }
+}

pkl-server/src/test/kotlin/org/pkl/server/TestTransport.kt

@@ -19,8 +19,8 @@ import java.util.concurrent.ArrayBlockingQueue
 import java.util.concurrent.BlockingQueue
 import org.assertj.core.api.Assertions.assertThat
 
-internal class TestTransport(private val delegate: MessageTransport) : AutoCloseable {
+class TestTransport(private val delegate: MessageTransport) : AutoCloseable {
-  private val incomingMessages: BlockingQueue<Message> = ArrayBlockingQueue(10)
+  val incomingMessages: BlockingQueue<Message> = ArrayBlockingQueue(10)
 
   fun start() {
     delegate.start({ incomingMessages.put(it) }, { incomingMessages.put(it) })