diff --git a/.circleci/config.yml b/.circleci/config.yml
index 29da17a1..51b74baf 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -12,7 +12,7 @@ jobs:
     - run:
         command: |-
           export PATH=~/staticdeps/bin:$PATH
-          ./gradlew --info --stacktrace -DtestReportsDir=${HOME}/test-results -DreleaseBuild=true pkl-cli:macExecutableAmd64 pkl-core:testMacExecutableAmd64 pkl-server:testNative
+          ./gradlew --info --stacktrace -DtestReportsDir=${HOME}/test-results -DreleaseBuild=true pkl-cli:macExecutableAmd64 pkl-core:testMacExecutableAmd64 pkl-server:testMacExecutableAmd64
         name: gradle buildNative
     - persist_to_workspace:
         root: '.'
@@ -88,7 +88,7 @@ jobs:
     - run:
         command: |-
           export PATH=~/staticdeps/bin:$PATH
-          ./gradlew --info --stacktrace -DtestReportsDir=${HOME}/test-results -DreleaseBuild=true pkl-cli:linuxExecutableAmd64 pkl-core:testLinuxExecutableAmd64 pkl-server:testNative
+          ./gradlew --info --stacktrace -DtestReportsDir=${HOME}/test-results -DreleaseBuild=true pkl-cli:linuxExecutableAmd64 pkl-core:testLinuxExecutableAmd64 pkl-server:testLinuxExecutableAmd64
         name: gradle buildNative
     - persist_to_workspace:
         root: '.'
@@ -108,7 +108,7 @@ jobs:
     - run:
         command: |-
           export PATH=~/staticdeps/bin:$PATH
-          ./gradlew --info --stacktrace -DtestReportsDir=${HOME}/test-results -DreleaseBuild=true pkl-cli:macExecutableAarch64 pkl-core:testMacExecutableAarch64 pkl-server:testNative
+          ./gradlew --info --stacktrace -DtestReportsDir=${HOME}/test-results -DreleaseBuild=true pkl-cli:macExecutableAarch64 pkl-core:testMacExecutableAarch64 pkl-server:testMacExecutableAarch64
         name: gradle buildNative
     - persist_to_workspace:
         root: '.'
@@ -168,7 +168,7 @@ jobs:
     - run:
         command: |-
           export PATH=~/staticdeps/bin:$PATH
-          ./gradlew --info --stacktrace -DtestReportsDir=${HOME}/test-results -DreleaseBuild=true pkl-cli:linuxExecutableAarch64 pkl-core:testLinuxExecutableAarch64 pkl-server:testNative
+          ./gradlew --info --stacktrace -DtestReportsDir=${HOME}/test-results -DreleaseBuild=true pkl-cli:linuxExecutableAarch64 pkl-core:testLinuxExecutableAarch64 pkl-server:testLinuxExecutableAarch64
         name: gradle buildNative
     - persist_to_workspace:
         root: '.'
@@ -245,7 +245,7 @@ jobs:
     - run:
         command: |-
           export PATH=~/staticdeps/bin:$PATH
-          ./gradlew --info --stacktrace -DtestReportsDir=${HOME}/test-results -DreleaseBuild=true pkl-cli:alpineExecutableAmd64 pkl-core:testAlpineExecutableAmd64 pkl-server:testNative
+          ./gradlew --info --stacktrace -DtestReportsDir=${HOME}/test-results -DreleaseBuild=true pkl-cli:alpineExecutableAmd64 pkl-core:testAlpineExecutableAmd64 pkl-server:testAlpineExecutableAmd64
         name: gradle buildNative
     - persist_to_workspace:
         root: '.'
@@ -265,7 +265,7 @@ jobs:
     - run:
         command: |-
           export PATH=~/staticdeps/bin:$PATH
-          ./gradlew --info --stacktrace -DtestReportsDir=${HOME}/test-results -DreleaseBuild=true pkl-cli:windowsExecutableAmd64 pkl-core:testWindowsExecutableAmd64 pkl-server:testNative
+          ./gradlew --info --stacktrace -DtestReportsDir=${HOME}/test-results -DreleaseBuild=true pkl-cli:windowsExecutableAmd64 pkl-core:testWindowsExecutableAmd64 pkl-server:testWindowsExecutableAmd64
         name: gradle buildNative
         shell: bash.exe
     - persist_to_workspace:
@@ -288,7 +288,7 @@ jobs:
     - run:
         command: |-
           export PATH=~/staticdeps/bin:$PATH
-          ./gradlew --info --stacktrace -DtestReportsDir=${HOME}/test-results pkl-cli:macExecutableAmd64 pkl-core:testMacExecutableAmd64 pkl-server:testNative
+          ./gradlew --info --stacktrace -DtestReportsDir=${HOME}/test-results pkl-cli:macExecutableAmd64 pkl-core:testMacExecutableAmd64 pkl-server:testMacExecutableAmd64
         name: gradle buildNative
     - persist_to_workspace:
         root: '.'
@@ -364,7 +364,7 @@ jobs:
     - run:
         command: |-
           export PATH=~/staticdeps/bin:$PATH
-          ./gradlew --info --stacktrace -DtestReportsDir=${HOME}/test-results pkl-cli:linuxExecutableAmd64 pkl-core:testLinuxExecutableAmd64 pkl-server:testNative
+          ./gradlew --info --stacktrace -DtestReportsDir=${HOME}/test-results pkl-cli:linuxExecutableAmd64 pkl-core:testLinuxExecutableAmd64 pkl-server:testLinuxExecutableAmd64
         name: gradle buildNative
     - persist_to_workspace:
         root: '.'
@@ -384,7 +384,7 @@ jobs:
     - run:
         command: |-
           export PATH=~/staticdeps/bin:$PATH
-          ./gradlew --info --stacktrace -DtestReportsDir=${HOME}/test-results pkl-cli:macExecutableAarch64 pkl-core:testMacExecutableAarch64 pkl-server:testNative
+          ./gradlew --info --stacktrace -DtestReportsDir=${HOME}/test-results pkl-cli:macExecutableAarch64 pkl-core:testMacExecutableAarch64 pkl-server:testMacExecutableAarch64
         name: gradle buildNative
     - persist_to_workspace:
         root: '.'
@@ -444,7 +444,7 @@ jobs:
     - run:
         command: |-
           export PATH=~/staticdeps/bin:$PATH
-          ./gradlew --info --stacktrace -DtestReportsDir=${HOME}/test-results pkl-cli:linuxExecutableAarch64 pkl-core:testLinuxExecutableAarch64 pkl-server:testNative
+          ./gradlew --info --stacktrace -DtestReportsDir=${HOME}/test-results pkl-cli:linuxExecutableAarch64 pkl-core:testLinuxExecutableAarch64 pkl-server:testLinuxExecutableAarch64
         name: gradle buildNative
     - persist_to_workspace:
         root: '.'
@@ -521,7 +521,7 @@ jobs:
     - run:
         command: |-
           export PATH=~/staticdeps/bin:$PATH
-          ./gradlew --info --stacktrace -DtestReportsDir=${HOME}/test-results pkl-cli:alpineExecutableAmd64 pkl-core:testAlpineExecutableAmd64 pkl-server:testNative
+          ./gradlew --info --stacktrace -DtestReportsDir=${HOME}/test-results pkl-cli:alpineExecutableAmd64 pkl-core:testAlpineExecutableAmd64 pkl-server:testAlpineExecutableAmd64
         name: gradle buildNative
     - persist_to_workspace:
         root: '.'
@@ -541,7 +541,7 @@ jobs:
     - run:
         command: |-
           export PATH=~/staticdeps/bin:$PATH
-          ./gradlew --info --stacktrace -DtestReportsDir=${HOME}/test-results pkl-cli:windowsExecutableAmd64 pkl-core:testWindowsExecutableAmd64 pkl-server:testNative
+          ./gradlew --info --stacktrace -DtestReportsDir=${HOME}/test-results pkl-cli:windowsExecutableAmd64 pkl-core:testWindowsExecutableAmd64 pkl-server:testWindowsExecutableAmd64
         name: gradle buildNative
         shell: bash.exe
     - persist_to_workspace:
diff --git a/.circleci/jobs/BuildNativeJob.pkl b/.circleci/jobs/BuildNativeJob.pkl
index 734c7987..5d48d029 100644
--- a/.circleci/jobs/BuildNativeJob.pkl
+++ b/.circleci/jobs/BuildNativeJob.pkl
@@ -126,7 +126,7 @@ steps {
     }
     command = #"""
       export PATH=~/staticdeps/bin:$PATH
-      ./gradlew \#(module.gradleArgs) pkl-cli:\#(jobName) pkl-core:test\#(jobName.capitalize()) pkl-server:testNative
+      ./gradlew \#(module.gradleArgs) pkl-cli:\#(jobName) pkl-core:test\#(jobName.capitalize()) pkl-server:test\#(jobName.capitalize())
       """#
   }
   new Config.PersistToWorkspaceStep {
diff --git a/pkl-server/pkl-server.gradle.kts b/pkl-server/pkl-server.gradle.kts
index 1173619b..b7c32902 100644
--- a/pkl-server/pkl-server.gradle.kts
+++ b/pkl-server/pkl-server.gradle.kts
@@ -24,15 +24,62 @@ tasks.test {
   exclude("**/NativeServerTest.*")
 }
 
-val nativeTest by tasks.registering(Test::class) {
-  dependsOn(":pkl-cli:assembleNative")
+private fun Test.configureNativeTest() {
   testClassesDirs = files(tasks.test.get().testClassesDirs)
   classpath = tasks.test.get().classpath
   include("**/NativeServerTest.*")
 }
 
-val testNative by tasks.existing
-testNative {
-  dependsOn(nativeTest)
+val testMacExecutableAarch64 by tasks.registering(Test::class) {
+  dependsOn(":pkl-cli:macExecutableAarch64")
+  configureNativeTest()
 }
 
+val testMacExecutableAmd64 by tasks.registering(Test::class) {
+  dependsOn(":pkl-cli:macExecutableAmd64")
+  configureNativeTest()
+}
+
+val testLinuxExecutableAmd64 by tasks.registering(Test::class) {
+  dependsOn(":pkl-cli:linuxExecutableAmd64")
+  configureNativeTest()
+}
+
+val testLinuxExecutableAarch64 by tasks.registering(Test::class) {
+  dependsOn(":pkl-cli:linuxExecutableAarch64")
+  configureNativeTest()
+}
+
+val testAlpineExecutableAmd64 by tasks.registering(Test::class) {
+  dependsOn(":pkl-cli:alpineExecutableAmd64")
+  configureNativeTest()
+}
+
+val testWindowsExecutableAmd64 by tasks.registering(Test::class) {
+  dependsOn(":pkl-cli:windowsExecutableAmd64")
+  configureNativeTest()
+}
+
+val testNative by tasks.existing
+testNative {
+  when {
+    buildInfo.os.isMacOsX -> {
+      dependsOn(testMacExecutableAmd64)
+      if (buildInfo.arch == "aarch64") {
+        dependsOn(testMacExecutableAarch64)
+      }
+    }
+    buildInfo.os.isWindows -> {
+      dependsOn(testWindowsExecutableAmd64)
+    }
+    buildInfo.os.isLinux && buildInfo.arch == "aarch64" -> {
+      dependsOn(testLinuxExecutableAarch64)
+    }
+    buildInfo.os.isLinux && buildInfo.arch == "amd64" -> {
+      dependsOn(testLinuxExecutableAmd64)
+      if (buildInfo.hasMuslToolchain) {
+        dependsOn(testAlpineExecutableAmd64)
+      }
+    }
+  }
+}