Upgrade GraalVM and Truffle, set up multi-JDK testing, bump development Java to 21 (#876)

This updates the GraalVM and Truffle libraries to 2024.1.2.

This also updates the build logic to compile Java sources using Java 21, due to some compile-only dependencies within GraalVM/Truffle using class file version 65. However, the produced artifact is still compatible with Java 17.

This also changes the Gradle build logic to use toolchains, and to test the Java libraries with JDK 17 and 21.

One consequence of this change is that Truffle is no longer shaded within the fat jars.

feat: support for jvm21+ toolchain
feat: support for gradle toolchains
feat: pass -PnativeArch=native to build with -march=native
test: multi-jdk testing support
test: support for jvm-test-suite plugin
test: add tasks to run jpkl eval on multiple jdks
test: make jdk exec tests respect multi-jdk flags and ranges
fix: remove mrjar classes at >jvm17 from fatjars
fix: use jdk21 to run the tests (needed for Unsafe.ensureInitialized)
fix: truffle svm dependency is required after graalvm 24.0.0
fix: warnings for gvm flag usage, renamed truffle svm macro
fix: build with --add-modules=jdk.unsupported where needed
fix: don't use gu tool for modern graalvm versions
fix: catch Throwable instead of deprecated-for-removal ThreadDeath
chore: buildinfo changes for JVM targets, toolchains
chore: enforce testing at exactly jdk21
chore: enforce build tooling at jdk21+
chore: bump graalvm/truffle libs → 24.1.2
chore: toolchains for buildSrc

Signed-off-by: Sam Gammon <sam@elide.dev>

pkl-cli/gradle.lockfile

@@ -54,13 +54,21 @@ org.eclipse.jetty:jetty-util:11.0.24=testCompileClasspath,testImplementationDepe
 org.eclipse.jetty:jetty-webapp:11.0.24=testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath
 org.eclipse.jetty:jetty-xml:11.0.24=testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath
 org.fusesource.jansi:jansi:2.4.1=compileClasspath,implementationDependenciesMetadata,runtimeClasspath,testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath
-org.graalvm.compiler:compiler:23.0.6=compileClasspath,compileOnlyDependenciesMetadata
-org.graalvm.nativeimage:native-image-base:23.0.6=compileClasspath,compileOnlyDependenciesMetadata
-org.graalvm.nativeimage:objectfile:23.0.6=compileClasspath,compileOnlyDependenciesMetadata
-org.graalvm.nativeimage:pointsto:23.0.6=compileClasspath,compileOnlyDependenciesMetadata
-org.graalvm.nativeimage:svm:23.0.6=compileClasspath,compileOnlyDependenciesMetadata
-org.graalvm.sdk:graal-sdk:23.0.6=compileClasspath,compileOnlyDependenciesMetadata,runtimeClasspath,testRuntimeClasspath
-org.graalvm.truffle:truffle-api:23.0.6=compileClasspath,compileOnlyDependenciesMetadata,runtimeClasspath,testRuntimeClasspath
+org.graalvm.compiler:compiler:24.1.2=compileClasspath,compileOnlyDependenciesMetadata
+org.graalvm.nativeimage:native-image-base:24.1.2=compileClasspath,compileOnlyDependenciesMetadata
+org.graalvm.nativeimage:objectfile:24.1.2=compileClasspath,compileOnlyDependenciesMetadata
+org.graalvm.nativeimage:pointsto:24.1.2=compileClasspath,compileOnlyDependenciesMetadata
+org.graalvm.nativeimage:svm:24.1.2=compileClasspath,compileOnlyDependenciesMetadata
+org.graalvm.nativeimage:truffle-runtime-svm:24.1.2=compileClasspath,compileOnlyDependenciesMetadata
+org.graalvm.polyglot:polyglot:24.1.2=compileClasspath,compileOnlyDependenciesMetadata,implementationDependenciesMetadata,runtimeClasspath,testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath
+org.graalvm.sdk:collections:24.1.2=compileClasspath,compileOnlyDependenciesMetadata,implementationDependenciesMetadata,runtimeClasspath,testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath
+org.graalvm.sdk:graal-sdk:24.1.2=compileClasspath,compileOnlyDependenciesMetadata,runtimeClasspath,testRuntimeClasspath
+org.graalvm.sdk:jniutils:24.1.2=compileClasspath,compileOnlyDependenciesMetadata,implementationDependenciesMetadata,runtimeClasspath,testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath
+org.graalvm.sdk:nativeimage:24.1.2=compileClasspath,compileOnlyDependenciesMetadata,implementationDependenciesMetadata,runtimeClasspath,testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath
+org.graalvm.sdk:word:24.1.2=compileClasspath,compileOnlyDependenciesMetadata,implementationDependenciesMetadata,runtimeClasspath,testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath
+org.graalvm.truffle:truffle-api:24.1.2=compileClasspath,compileOnlyDependenciesMetadata,implementationDependenciesMetadata,runtimeClasspath,testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath
+org.graalvm.truffle:truffle-compiler:24.1.2=compileClasspath,compileOnlyDependenciesMetadata,implementationDependenciesMetadata,runtimeClasspath,testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath
+org.graalvm.truffle:truffle-runtime:24.1.2=compileClasspath,compileOnlyDependenciesMetadata,implementationDependenciesMetadata,runtimeClasspath,testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath
 org.hamcrest:hamcrest-core:2.2=testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath
 org.hamcrest:hamcrest:2.2=testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath
 org.jetbrains.intellij.deps:trove4j:1.0.20200330=kotlinBuildToolsApiClasspath,kotlinCompilerClasspath,kotlinKlibCommonizerClasspath

pkl-cli/pkl-cli.gradle.kts

@@ -13,6 +13,9 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+import java.io.ByteArrayOutputStream
+import org.gradle.kotlin.dsl.support.serviceOf
+
 plugins {
   pklAllProjects
   pklKotlinLibrary
@@ -50,6 +53,9 @@ val stagedWindowsAmd64Executable: Configuration by configurations.creating
 
 dependencies {
   compileOnly(libs.svm)
+  compileOnly(libs.truffleSvm)
+  implementation(libs.truffleRuntime)
+  compileOnly(libs.graalSdk)
 
   // CliEvaluator exposes PClass
   api(projects.pklCore)
@@ -81,7 +87,10 @@ dependencies {
   stagedWindowsAmd64Executable(executableDir("pkl-windows-amd64.exe"))
 }
 
-tasks.jar { manifest { attributes += mapOf("Main-Class" to "org.pkl.cli.Main") } }
+tasks.jar {
+  manifest.attributes +=
+    mapOf("Main-Class" to "org.pkl.cli.Main", "Add-Exports" to buildInfo.jpmsExportsForJarManifest)
+}
 
 tasks.javadoc { enabled = false }
 
@@ -121,33 +130,95 @@ val testJavaExecutable by
         (configurations.testRuntimeClasspath.get() - configurations.runtimeClasspath.get())
   }
 
-tasks.check { dependsOn(testJavaExecutable) }
+// Setup `testJavaExecutable` tasks for multi-JDK testing.
+val testJavaExecutableOnOtherJdks =
+  if (buildInfo.multiJdkTesting) {
+    buildInfo.multiJdkTestingWith(testJavaExecutable)
+  } else {
+    emptyList()
+  }
 
-// 0.14 Java executable was broken because javaExecutable.jvmArgs wasn't commented out.
-// To catch this and similar problems, test that Java executable starts successfully.
-val testStartJavaExecutable by
-  tasks.registering(Exec::class) {
+// Prepare a run of the fat JAR, optionally with a specific Java launcher.
+private fun setupJavaExecutableRun(
+  name: String,
+  args: Array<String>,
+  launcher: Provider<JavaLauncher>? = null,
+  configurator: Exec.() -> Unit = {},
+) =
+  tasks.register(name, Exec::class) {
     dependsOn(javaExecutable)
-    val outputFile =
-      layout.buildDirectory.file(
-        "testStartJavaExecutable"
-      ) // dummy output to satisfy up-to-date check
+    val outputFile = layout.buildDirectory.file(name) // dummy output to satisfy up-to-date check
     outputs.file(outputFile)
 
-    if (buildInfo.os.isWindows) {
-      executable = "java"
-      args("-jar", javaExecutable.get().outputs.files.singleFile.toString(), "--version")
-    } else {
-      executable = javaExecutable.get().outputs.files.singleFile.toString()
-      args("--version")
-    }
+    executable =
+      when (launcher) {
+        null -> "java"
+        else -> launcher.get().executablePath.asFile.absolutePath
+      }
+
+    args("-jar", javaExecutable.get().outputs.files.singleFile.toString(), *args)
 
     doFirst { outputFile.get().asFile.delete() }
 
     doLast { outputFile.get().asFile.writeText("OK") }
+
+    configurator()
   }
 
-tasks.check { dependsOn(testStartJavaExecutable) }
+// 0.14 Java executable was broken because javaExecutable.jvmArgs wasn't commented out.
+// To catch this and similar problems, test that Java executable starts successfully.
+val testStartJavaExecutable by
+  setupJavaExecutableRun("testStartJavaExecutable", arrayOf("--version"))
+
+// Setup `testStartJavaExecutable` tasks for multi-JDK testing.
+val testStartJavaExecutableOnOtherJdks =
+  if (buildInfo.multiJdkTesting) {
+    buildInfo.jdkTestRange.map { jdkTarget ->
+      setupJavaExecutableRun(
+        "testStartJavaExecutableJdk${jdkTarget.asInt()}",
+        arrayOf("--version"),
+        serviceOf<JavaToolchainService>().launcherFor { languageVersion = jdkTarget },
+      )
+    }
+  } else {
+    emptyList()
+  }
+
+val evalTestFlags = arrayOf("eval", "./.circleci/config.pkl")
+
+fun Exec.useRootDirAndSuppressOutput() {
+  workingDir = rootProject.layout.projectDirectory.asFile
+  standardOutput = ByteArrayOutputStream() // we only care that this exec doesn't fail
+}
+
+// 0.28 Preparing for JDK21 toolchains revealed that `testStartJavaExecutable` may pass, even though
+// the evaluator fails. To catch this, we need to test the evaluator. We render the CircleCI config
+// as a realistic test of the fat JAR.
+val testEvalJavaExecutable by
+  setupJavaExecutableRun("testEvalJavaExecutable", evalTestFlags) { useRootDirAndSuppressOutput() }
+
+// Run the same evaluator tests on all configured JDK test versions.
+val testEvalJavaExecutableOnOtherJdks =
+  buildInfo.jdkTestRange.map { jdkTarget ->
+    setupJavaExecutableRun(
+      "testEvalJavaExecutableJdk${jdkTarget.asInt()}",
+      evalTestFlags,
+      serviceOf<JavaToolchainService>().launcherFor { languageVersion = jdkTarget },
+    ) {
+      useRootDirAndSuppressOutput()
+    }
+  }
+
+tasks.check {
+  dependsOn(
+    testJavaExecutable,
+    testStartJavaExecutable,
+    testJavaExecutableOnOtherJdks,
+    testStartJavaExecutableOnOtherJdks,
+    testEvalJavaExecutable,
+    testEvalJavaExecutableOnOtherJdks,
+  )
+}
 
 fun Exec.configureExecutable(
   graalVm: BuildInfo.GraalVm,
@@ -174,11 +245,13 @@ fun Exec.configureExecutable(
   executable = "${graalVm.baseDir}/bin/$nativeImageCommandName"
 
   // JARs to exclude from the class path for the native-image build.
-  val exclusions = listOf(libs.truffleApi, libs.graalSdk).map { it.get().module.name }
+  val exclusions = listOf(libs.graalSdk).map { it.get().module.name }
   // https://www.graalvm.org/22.0/reference-manual/native-image/Options/
   argumentProviders.add(
     CommandLineArgumentProvider {
       buildList {
+        // must be emitted before any experimental options are used
+        add("-H:+UnlockExperimentalVMOptions")
         // currently gives a deprecation warning, but we've been told
         // that the "initialize everything at build time" *CLI* option is likely here to stay
         add("--initialize-at-build-time=")
@@ -189,9 +262,9 @@ fun Exec.configureExecutable(
         add("-H:IncludeResources=org/jline/utils/.*")
         add("-H:IncludeResourceBundles=org.pkl.core.errorMessages")
         add("-H:IncludeResources=org/pkl/commons/cli/PklCARoots.pem")
-        add("--macro:truffle")
         add("-H:Class=org.pkl.cli.Main")
-        add("-H:Name=${outputFile.get().asFile.name}")
+        add("-o")
+        add(outputFile.get().asFile.name)
         // the actual limit (currently) used by native-image is this number + 1400 (idea is to
         // compensate for Truffle's own nodes)
         add("-H:MaxRuntimeCompileMethods=1800")
@@ -206,7 +279,11 @@ fun Exec.configureExecutable(
         if (!buildInfo.isReleaseBuild) {
           add("-Ob")
         }
-        add("-march=compatibility")
+        if (buildInfo.isNativeArch) {
+          add("-march=native")
+        } else {
+          add("-march=compatibility")
+        }
         // native-image rejects non-existing class path entries -> filter
         add("--class-path")
         val pathInput =