From d062312df9c294e6abf7dad79cea9518fc9f22c9 Mon Sep 17 00:00:00 2001 From: Daniel Chao Date: Wed, 14 Oct 2026 10:03:45 -0700 Subject: [PATCH] Add `sourceBundle` task to download and prepare source bundles (#1902) * Add tasks to generate source jars for java executables * native source bundles * source bundles in separate plugin * Wire up sourceBundle task * Produce bundles on a per-variant basis (Java fat jar, each native variant) * Introduce `pklNativeSourceBundle` and `pklJvmSourceBundle` plugins for handling fat jar vs. GraalVM bits * Download compiled classes list, and exclude Java sources for executable stuff specifically * Include native sources when appropriate (GraalVM, JDK, zlib, musl, libpkl) --------- Co-authored-by: Jen Basch --- .../src/main/kotlin/BuildSourceBundle.kt | 111 ++++++ .../src/main/kotlin/NativeImageSourcesJar.kt | 109 ++++++ .../src/main/kotlin/SourceBundleSpec.kt | 43 +++ .../src/main/kotlin/pklFatJar.gradle.kts | 17 +- .../main/kotlin/pklJavaExecutable.gradle.kts | 1 + .../main/kotlin/pklJvmSourceBundle.gradle.kts | 35 ++ .../kotlin/pklNativeExecutable.gradle.kts | 4 + .../kotlin/pklNativeSourceBundle.gradle.kts | 322 ++++++++++++++++++ .../main/kotlin/pklSourceBundle.gradle.kts | 49 +++ libpkl/libpkl.gradle.kts | 38 ++- pkl-cli/pkl-cli.gradle.kts | 9 + pkl-codegen-java/pkl-codegen-java.gradle.kts | 7 + .../pkl-codegen-kotlin.gradle.kts | 7 + pkl-config-java/pkl-config-java.gradle.kts | 2 + pkl-doc/pkl-doc.gradle.kts | 6 + pkl-tools/pkl-tools.gradle.kts | 5 + settings.gradle.kts | 35 +- 17 files changed, 777 insertions(+), 23 deletions(-) create mode 100644 build-logic/src/main/kotlin/BuildSourceBundle.kt create mode 100644 build-logic/src/main/kotlin/NativeImageSourcesJar.kt create mode 100644 build-logic/src/main/kotlin/SourceBundleSpec.kt create mode 100644 build-logic/src/main/kotlin/pklJvmSourceBundle.gradle.kts create mode 100644 build-logic/src/main/kotlin/pklNativeSourceBundle.gradle.kts create mode 100644 build-logic/src/main/kotlin/pklSourceBundle.gradle.kts diff --git a/build-logic/src/main/kotlin/BuildSourceBundle.kt b/build-logic/src/main/kotlin/BuildSourceBundle.kt new file mode 100644 index 000000000..c5802c9e5 --- /dev/null +++ b/build-logic/src/main/kotlin/BuildSourceBundle.kt @@ -0,0 +1,111 @@ +/* + * Copyright © 2024-2026 Apple Inc. and the Pkl project authors. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import java.io.File +import java.util.zip.ZipEntry +import java.util.zip.ZipOutputStream +import org.gradle.api.DefaultTask +import org.gradle.api.file.ConfigurableFileCollection +import org.gradle.api.file.RegularFileProperty +import org.gradle.api.tasks.InputFile +import org.gradle.api.tasks.InputFiles +import org.gradle.api.tasks.Optional +import org.gradle.api.tasks.OutputFile +import org.gradle.api.tasks.TaskAction + +open class BuildSourceBundle : DefaultTask() { + /** Sources jars that are included in full. */ + @get:InputFiles val inputJars: ConfigurableFileCollection = project.objects.fileCollection() + + /** + * Sources jars that are only included in part, if [compiledClasses] is set. + * + * Only `.java` files are dropped, if their class is not in [compiledClasses]. Everything else is + * kept: other languages (e.g. Kotlin) can't be matched by name, because the classes that they + * compile to don't necessarily relate to the file name or package (e.g. with `@file:JvmName` or + * `@file:JvmPackageName`), and licenses and other metadata must be retained. + * + * If [compiledClasses] is not set, these jars are included in full. + */ + @get:InputFiles + val filteredInputJars: ConfigurableFileCollection = project.objects.fileCollection() + + /** Fully qualified names of top-level classes, one per line. */ + @get:[InputFile Optional] + val compiledClasses: RegularFileProperty = project.objects.fileProperty() + + @get:OutputFile val outputZip: RegularFileProperty = project.objects.fileProperty() + + @TaskAction + @Suppress("unused") + fun merge() { + val zipFile = outputZip.asFile.get() + if (!zipFile.getParentFile().exists()) zipFile.getParentFile().mkdirs() + val classes = + if (compiledClasses.isPresent) { + compiledClasses + .get() + .asFile + .readLines() + .map { it.trim() } + .filterTo(hashSetOf()) { it.isNotEmpty() } + } else null + val packages = classes?.mapTo(hashSetOf()) { it.substringBeforeLast('.', "").replace('.', '/') } + var dropped = 0 + ZipOutputStream(zipFile.outputStream()).use { zip -> + fun copyJar(jar: File, keep: (String) -> Boolean) { + val jarName = jar.toPath().fileName.toString().removeSuffix(".jar") + project.zipTree(jar).visit { + if (isDirectory) return@visit + val path = relativePath.pathString + if (!keep(path)) { + dropped++ + return@visit + } + zip.putNextEntry(ZipEntry("$jarName/$path")) + copyTo(zip) + zip.closeEntry() + } + } + for (jar in inputJars) copyJar(jar) { true } + for (jar in filteredInputJars) { + copyJar(jar) { path -> + classes == null || packages == null || isEvident(path, classes, packages) + } + } + } + if (classes != null) { + logger.lifecycle("Dropped $dropped source files that are not in ${compiledClasses.get()}") + } + } + + private fun isEvident(path: String, classes: Set, packages: Set): Boolean { + // sources of multi-release jars have the same classes as the unversioned ones + val sourcePath = path.replace(multiReleasePrefix, "") + val packagePath = sourcePath.substringBeforeLast('/', "") + val fileName = sourcePath.substringAfterLast('/') + return when (sourcePath.substringAfterLast('.', "")) { + "java" -> + when (fileName) { + "package-info.java" -> packagePath in packages + "module-info.java" -> true + else -> sourcePath.removeSuffix(".java").replace('/', '.') in classes + } + else -> true + } + } + + private val multiReleasePrefix = Regex("^META-INF/versions/\\d+/") +} diff --git a/build-logic/src/main/kotlin/NativeImageSourcesJar.kt b/build-logic/src/main/kotlin/NativeImageSourcesJar.kt new file mode 100644 index 000000000..95daee04a --- /dev/null +++ b/build-logic/src/main/kotlin/NativeImageSourcesJar.kt @@ -0,0 +1,109 @@ +/* + * Copyright © 2026 Apple Inc. and the Pkl project authors. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import java.util.zip.ZipEntry +import java.util.zip.ZipOutputStream +import org.gradle.api.DefaultTask +import org.gradle.api.GradleException +import org.gradle.api.file.ConfigurableFileCollection +import org.gradle.api.file.RegularFileProperty +import org.gradle.api.provider.SetProperty +import org.gradle.api.tasks.Input +import org.gradle.api.tasks.InputFile +import org.gradle.api.tasks.InputFiles +import org.gradle.api.tasks.OutputFile +import org.gradle.api.tasks.TaskAction + +/** + * Collects the JDK, SubstrateVM, and Graal Java sources for every class that has methods compiled + * into a native image. + * + * The classes are read from [compiledClasses], and their sources are looked up in the OpenJDK and + * Graal repositories' tagged source archives. These archives contain the sources of every platform + * and architecture, so this task does not need to run on the platform that it collects sources for. + */ +abstract class NativeImageSourcesJar : DefaultTask() { + /** + * Fully qualified top-level names of the classes compiled into the image, one per line. + * + * This is the list that native-image reports (via `-H:+PrintUniverse`) when building the image. + */ + @get:InputFile abstract val compiledClasses: RegularFileProperty + + /** The gzipped tarball of the OpenJDK repository (e.g. `jdk25u`) at the JDK's release tag. */ + @get:InputFiles abstract val openJdkTarball: ConfigurableFileCollection + + /** The gzipped tarball of the `oracle/graal` repository at the GraalVM's release tag. */ + @get:InputFiles abstract val graalTarball: ConfigurableFileCollection + + /** + * The platform-specific source roots to include from OpenJDK; any of `share`, `unix`, `linux`, + * `macosx`, and `windows`. + */ + @get:Input abstract val openJdkPlatforms: SetProperty + + @get:OutputFile abstract val outputJar: RegularFileProperty + + @TaskAction + @Suppress("unused") + fun collect() { + val wanted = + compiledClasses + .get() + .asFile + .readLines() + .filter { it.isNotBlank() } + .mapTo(hashSetOf()) { it.trim().replace('.', '/') + ".java" } + if (wanted.isEmpty()) { + throw GradleException("No compiled classes found in ${compiledClasses.get()}") + } + + val platforms = openJdkPlatforms.get() + // -/src///classes/.java + val openJdkSource = Regex("""^[^/]+/src/[^/]+/([^/]+)/classes/(.+\.java)$""") + // -//src//src/.java + // + // Truffle sources are intentionally omitted; they are already published as sources jars. + val graalSource = Regex("""^[^/]+/(?:substratevm|compiler|sdk)/src/[^/]+/src/(.+\.java)$""") + + val out = outputJar.get().asFile + out.parentFile.mkdirs() + val written = hashSetOf() + ZipOutputStream(out.outputStream().buffered()).use { zip -> + fun collectFrom(tarball: java.io.File, sourcePathOf: (String) -> String?) { + project.tarTree(project.resources.gzip(tarball)).visit { + if (isDirectory) return@visit + val sourcePath = sourcePathOf(relativePath.pathString) ?: return@visit + if (sourcePath !in wanted || !written.add(sourcePath)) return@visit + zip.putNextEntry(ZipEntry(sourcePath)) + copyTo(zip) + zip.closeEntry() + } + } + collectFrom(openJdkTarball.singleFile) { path -> + openJdkSource + .matchEntire(path) + ?.takeIf { it.groupValues[1] in platforms } + ?.let { it.groupValues[2] } + } + collectFrom(graalTarball.singleFile) { path -> + graalSource.matchEntire(path)?.groupValues?.get(1) + } + } + logger.lifecycle( + "Collected sources of ${written.size} of ${wanted.size} compiled classes into $out" + ) + } +} diff --git a/build-logic/src/main/kotlin/SourceBundleSpec.kt b/build-logic/src/main/kotlin/SourceBundleSpec.kt new file mode 100644 index 000000000..b432e2102 --- /dev/null +++ b/build-logic/src/main/kotlin/SourceBundleSpec.kt @@ -0,0 +1,43 @@ +/* + * Copyright © 2026 Apple Inc. and the Pkl project authors. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import org.gradle.api.file.ConfigurableFileCollection +import org.gradle.api.provider.Property + +/** + * The inputs that source bundles of a project are made of. + * + * This is configured by the `pklSourceBundle` plugin; the plugins that register the actual source + * bundles (`pklJvmSourceBundle` and `pklNativeSourceBundle`) consume it. + */ +abstract class SourceBundleSpec { + /** Sources jars of the project itself, and of the projects that it depends on. */ + abstract val firstPartyJars: ConfigurableFileCollection + + /** Sources jars of third-party dependencies, as resolved from the runtime classpath. */ + abstract val dependencyJars: ConfigurableFileCollection + + /** + * Additional sources to include in source bundles of native targets (e.g. C sources of the + * project itself). + */ + abstract val extraNativeSources: ConfigurableFileCollection + + /** + * Whether musl and zlib are statically linked into the project's native executable for musl + * targets, in which case their sources are included in the source bundle of those targets. + */ + abstract val staticMuslLibc: Property +} diff --git a/build-logic/src/main/kotlin/pklFatJar.gradle.kts b/build-logic/src/main/kotlin/pklFatJar.gradle.kts index 971293f51..9c1815b96 100644 --- a/build-logic/src/main/kotlin/pklFatJar.gradle.kts +++ b/build-logic/src/main/kotlin/pklFatJar.gradle.kts @@ -24,6 +24,7 @@ plugins { `java-library` `maven-publish` id("com.gradleup.shadow") + id("pklJvmSourceBundle") } // make fat Jar available to other subprojects @@ -31,10 +32,6 @@ val fatJarConfiguration: Configuration = configurations.create("fatJar") val fatJarPublication: MavenPublication = publishing.publications.create("fatJar") -// ideally we'd configure this automatically based on project dependencies -val firstPartySourcesJarsConfiguration: Configuration = - configurations.create("firstPartySourcesJars") - val relocations = mapOf( // pkl-core dependencies @@ -182,17 +179,13 @@ val validateFatJar = tasks.check { dependsOn(validateFatJar) } -val resolveSourcesJars = - tasks.register("resolveSourcesJars") { - configuration.set(configurations.runtimeClasspath) - outputDir.set(layout.buildDirectory.dir("resolveSourcesJars")) - } - val fatSourcesJar = tasks.register("fatSourcesJar") { plugins.withId("pklJavaLibrary") { inputJars.from(tasks.named("sourcesJar")) } - inputJars.from(firstPartySourcesJarsConfiguration) - inputJars.from(resolveSourcesJars.map { fileTree(it.outputDir) }) + inputJars.from(configurations.named("firstPartySourcesJars")) + inputJars.from( + tasks.named("resolveSourcesJars").map { fileTree(it.outputDir) } + ) mergedBinaryJars.from(tasks.shadowJar) relocatedPackages.set(relocations) diff --git a/build-logic/src/main/kotlin/pklJavaExecutable.gradle.kts b/build-logic/src/main/kotlin/pklJavaExecutable.gradle.kts index e2fd0aeb9..521d299cc 100644 --- a/build-logic/src/main/kotlin/pklJavaExecutable.gradle.kts +++ b/build-logic/src/main/kotlin/pklJavaExecutable.gradle.kts @@ -21,6 +21,7 @@ plugins { id("pklJavaLibrary") // id("pklPublishLibrary") id("com.gradleup.shadow") + id("pklJvmSourceBundle") } val executableSpec = project.extensions.create("executable", ExecutableSpec::class.java) diff --git a/build-logic/src/main/kotlin/pklJvmSourceBundle.gradle.kts b/build-logic/src/main/kotlin/pklJvmSourceBundle.gradle.kts new file mode 100644 index 000000000..cc0fd6e08 --- /dev/null +++ b/build-logic/src/main/kotlin/pklJvmSourceBundle.gradle.kts @@ -0,0 +1,35 @@ +/* + * Copyright © 2026 Apple Inc. and the Pkl project authors. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +// Source bundle for the JVM variant of a project: its fat JAR, or its executable JAR. +// +// These contain the complete sources of all dependencies, because fat JARs contain the complete +// classes of all dependencies. +plugins { id("pklSourceBundle") } + +val sourceBundleSpec = extensions.getByType() + +val sourceBundleJava = + tasks.register("sourceBundleJava") { + group = "build" + description = "Builds the source bundle of ${project.name}'s JVM variant." + inputJars.from(sourceBundleSpec.firstPartyJars, sourceBundleSpec.dependencyJars) + outputZip = + layout.buildDirectory.file( + "sourceBundle/${project.name}-${project.version}-java-sourcebundle.zip" + ) + } + +tasks.named("sourceBundle") { dependsOn(sourceBundleJava) } diff --git a/build-logic/src/main/kotlin/pklNativeExecutable.gradle.kts b/build-logic/src/main/kotlin/pklNativeExecutable.gradle.kts index 87ab8b9b5..514106336 100644 --- a/build-logic/src/main/kotlin/pklNativeExecutable.gradle.kts +++ b/build-logic/src/main/kotlin/pklNativeExecutable.gradle.kts @@ -22,6 +22,7 @@ plugins { id("pklGraalVm") id("pklJavaLibrary") id("pklNativeLifecycle") + id("pklNativeSourceBundle") id("pklPublishLibrary") id("com.gradleup.shadow") } @@ -30,6 +31,9 @@ plugins { val executableSpec = project.extensions.getByType() val buildInfo = project.extensions.getByType() +// musl executables are linked with `--static`, so musl and zlib become part of the executable +extensions.getByType().staticMuslLibc = true + val stagedMacAarch64Executable: Configuration = configurations.create("stagedMacAarch64Executable") val stagedLinuxAmd64Executable: Configuration = configurations.create("stagedLinuxAmd64Executable") val stagedLinuxAarch64Executable: Configuration = diff --git a/build-logic/src/main/kotlin/pklNativeSourceBundle.gradle.kts b/build-logic/src/main/kotlin/pklNativeSourceBundle.gradle.kts new file mode 100644 index 000000000..b76fe1851 --- /dev/null +++ b/build-logic/src/main/kotlin/pklNativeSourceBundle.gradle.kts @@ -0,0 +1,322 @@ +/* + * Copyright © 2026 Apple Inc. and the Pkl project authors. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import javax.inject.Inject +import org.gradle.accessors.dm.LibrariesForLibs +import org.gradle.api.tasks.options.Option +import org.gradle.process.ExecOperations + +// Source bundles for the native variants of a project, one for each `Target`. +// +// Each bundle consists of the sources of everything that is compiled into the native image or +// library of that target: +// * the project's own sources, and those of its first-party dependencies; +// * the sources of third-party dependencies that have classes in the image; +// * the Java sources of the JDK, SubstrateVM, and Graal classes that are in the image; +// * the C sources of the JDK and SubstrateVM native libraries that are linked into the image; +// * for statically linked musl executables, the sources of musl and zlib. +// +// Which classes are in the image is read from the lists in `build/compiled-classes` (download them +// with the `downloadCompiledClasses` task). native-image produces these lists, and CI uploads them +// as `compiled-classes--*` artifacts. This way, the source bundle of any target can be +// built on any machine. +// +// Sources of the JDK, SubstrateVM, and Graal come from the OpenJDK and Graal repositories at the +// tags matching the GraalVM that is used to build, because GraalVM doesn't ship all of the Java +// sources (or any of the C sources). +plugins { id("pklSourceBundle") } + +val libs = the() +val sourceBundleSpec = extensions.getByType() + +val openJdkSourcesTarball: Configuration = configurations.create("openJdkSourcesTarball") +val graalSourcesTarball: Configuration = configurations.create("graalSourcesTarball") + +val muslSourcesTarball: Configuration = configurations.create("muslSourcesTarball") +val zlibSourcesTarball: Configuration = configurations.create("zlibSourcesTarball") + +// CI builds musl and zlib from source for statically linked executables; use the same versions. +fun muslToolchainVersion(name: String): String { + val script = rootProject.file(".github/scripts/install_musl.sh").readText() + return Regex("${name}_VERSION=\"([^\"]+)\"").find(script)?.groupValues?.get(1) + ?: throw GradleException("Could not find ${name}_VERSION in .github/scripts/install_musl.sh") +} + +// The "GitHub tag archives" and "musl releases" repositories are declared in settings.gradle.kts; +// declaring repositories in a project would stop it from using the ones declared in settings. +dependencies { + val jdkVersion = libs.versions.graalVmJdkVersion.get() + openJdkSourcesTarball("openjdk:jdk${jdkVersion.substringBefore('.')}u:jdk-$jdkVersion-ga") + graalSourcesTarball("oracle:graal:vm-${libs.versions.graalVm.get()}") + muslSourcesTarball("musl:musl:${muslToolchainVersion("MUSL")}") + zlibSourcesTarball("madler:zlib:v${muslToolchainVersion("ZLIB")}") +} + +// native-image's class path can differ from the runtime class path (e.g. `nativeImageClasspath` in +// `pklNativeExecutable`), so resolve the sources of what's actually compiled in. +val resolveNativeSourcesJars = + tasks.register("resolveNativeSourcesJars") { + configuration.set( + provider { + configurations.findByName("nativeImageClasspath") + ?: configurations.getByName("runtimeClasspath") + } + ) + outputDir.set(layout.buildDirectory.dir("resolveNativeSourcesJars")) + } + +// The JDK native libraries that native-image links into the image. +val openJdkNativeLibraries = listOf("java", "net", "nio", "zip", "extnet", "management_ext") + +// Platform-specific directories in OpenJDK's `src///`. +val Target.OS.openJdkPlatforms: List + get() = buildList { + add("share") + if (!isWindows) add("unix") + add( + when { + isMacOS -> "macosx" + isWindows -> "windows" + else -> "linux" + } + ) + } + +// The `substratevm/src/com.oracle.svm.native.*` projects that apply to the OS. +val Target.OS.svmNativeProjects: List + get() = buildList { + add("libchelper") + when { + isMacOS -> { + add("darwin") + add("jvm.posix") + } + isLinux -> { + add("libcontainer") + add("jvm.posix") + } + isWindows -> add("jvm.windows") + } + } + +val sourceBundleTasks = + Target.entries.map { target -> + val variant = target.name + val outputDir = layout.buildDirectory.dir("tmp/nativeImageSources/${target.targetName}") + val compiledClassesFile = + layout.buildDirectory.file("compiled-classes/${target.targetName}.txt") + + val nativeImageSourcesJar = + tasks.register("nativeImageSourcesJar$variant") { + compiledClasses = compiledClassesFile + openJdkTarball.from(openJdkSourcesTarball) + graalTarball.from(graalSourcesTarball) + openJdkPlatforms.addAll(target.os.openJdkPlatforms) + outputJar = outputDir.map { it.file("native-image-sources.jar") } + } + + val openJdkNativeSourcesJar = + tasks.register("openJdkNativeSourcesJar$variant") { + archiveFileName = "openjdk-native-sources.jar" + destinationDirectory = outputDir + includeEmptyDirs = false + + from(provider { tarTree(resources.gzip(openJdkSourcesTarball.singleFile)) }) { + for (platform in target.os.openJdkPlatforms) { + include("*/src/*/$platform/native/include/**") + for (library in openJdkNativeLibraries) { + include("*/src/*/$platform/native/lib$library/**") + } + } + // drop the `-/` root directory + eachFile { + relativePath = RelativePath(true, *relativePath.segments.drop(1).toTypedArray()) + } + } + } + + val graalNativeSourcesJar = + tasks.register("graalNativeSourcesJar$variant") { + archiveFileName = "graal-native-sources.jar" + destinationDirectory = outputDir + includeEmptyDirs = false + + from(provider { tarTree(resources.gzip(graalSourcesTarball.singleFile)) }) { + for (svmProject in target.os.svmNativeProjects) { + include("*/substratevm/src/com.oracle.svm.native.$svmProject/**") + } + eachFile { + relativePath = RelativePath(true, *relativePath.segments.drop(1).toTypedArray()) + } + } + } + + // Executables for musl targets are statically linked, so musl and zlib are part of them. + val staticLibcSources = + if (target.musl) { + val muslArch = + when (target.arch) { + Target.Arch.AMD64 -> "x86_64" + Target.Arch.AARCH64 -> "aarch64" + } + + val muslSourcesJar = + tasks.register("muslSourcesJar$variant") { + archiveFileName = "musl-sources.jar" + destinationDirectory = outputDir + includeEmptyDirs = false + + from(provider { tarTree(resources.gzip(muslSourcesTarball.singleFile)) }) { + // skip the other architectures + include("*/src/**", "*/include/**", "*/crt/**", "*/ldso/**", "*/compat/**") + include("*/arch/generic/**", "*/arch/$muslArch/**") + include("*/COPYRIGHT", "*/VERSION", "*/WHATSNEW") + eachFile { + relativePath = RelativePath(true, *relativePath.segments.drop(1).toTypedArray()) + } + } + } + + val zlibSourcesJar = + tasks.register("zlibSourcesJar$variant") { + archiveFileName = "zlib-sources.jar" + destinationDirectory = outputDir + includeEmptyDirs = false + + from(provider { tarTree(resources.gzip(zlibSourcesTarball.singleFile)) }) { + exclude("*/.github/**") + eachFile { + relativePath = RelativePath(true, *relativePath.segments.drop(1).toTypedArray()) + } + } + } + + sourceBundleSpec.staticMuslLibc.map { isStatic -> + if (isStatic) files(muslSourcesJar, zlibSourcesJar) else files() + } + } else { + provider { files() } + } + + tasks.register("sourceBundle$variant") { + group = "build" + description = "Builds the source bundle of ${project.name} for ${target.targetName}." + inputJars.from( + sourceBundleSpec.firstPartyJars, + sourceBundleSpec.extraNativeSources, + nativeImageSourcesJar, + openJdkNativeSourcesJar, + graalNativeSourcesJar, + staticLibcSources, + ) + // Only include sources of dependency classes that are compiled into the image. + filteredInputJars.from(resolveNativeSourcesJars.map { fileTree(it.outputDir) }) + compiledClasses = compiledClassesFile + outputZip = + layout.buildDirectory.file( + "sourceBundle/${project.name}-${project.version}-${target.targetName}-sourcebundle.zip" + ) + } + } + +tasks.named("sourceBundle") { dependsOn(sourceBundleTasks) } + +/** + * Downloads the lists of classes compiled into native images, which CI uploads as + * `compiled-classes---` artifacts, into `build/compiled-classes`. + * + * Requires the GitHub CLI (`gh`) to be installed and authenticated. + */ +abstract class DownloadCompiledClasses : DefaultTask() { + @get:Option(option = "run-id", description = "ID of the GitHub Actions run to download from.") + @get:Input + abstract val runId: Property + + @get:Option(option = "repo", description = "GitHub repository that the run belongs to.") + @get:Input + abstract val repo: Property + + /** The project whose artifacts to download (named `compiled-classes--*`). */ + @get:Input abstract val projectName: Property + + @get:OutputDirectory abstract val destinationDir: DirectoryProperty + + @get:Inject protected abstract val execOperations: ExecOperations + + init { + group = "build" + description = "Downloads the compiled classes lists that CI produced into the build directory." + repo.convention("apple/pkl") + // always re-download; this task is meant to be run manually + outputs.upToDateWhen { false } + } + + @TaskAction + @Suppress("unused") + fun download() { + if (!runId.isPresent) { + throw GradleException("Specify the run to download from, e.g. `--run-id=`") + } + val name = projectName.get() + val downloadDir = temporaryDir.resolve("download") + downloadDir.deleteRecursively() + downloadDir.mkdirs() + + execOperations.exec { + commandLine( + "gh", + "run", + "download", + runId.get(), + "--repo", + repo.get(), + "--pattern", + "compiled-classes-$name-*", + "--dir", + downloadDir.absolutePath, + ) + } + + // Files are named `--.txt`; keep only the target name. + val fileName = Regex("^${Regex.escape(name)}-\\d+(?:\\.\\d+)*(?:-SNAPSHOT)?-(.+\\.txt)$") + val downloaded = + downloadDir + .walkTopDown() + .filter { it.isFile } + .associate { file -> + val target = + fileName.matchEntire(file.name)?.groupValues?.get(1) + ?: throw GradleException("Unexpected file in downloaded artifacts: $file") + target to file + } + if (downloaded.isEmpty()) { + throw GradleException("Run ${runId.get()} has no `compiled-classes-$name-*` artifacts") + } + + val destination = destinationDir.get().asFile + destination.mkdirs() + // remove lists of targets that are no longer built + destination.listFiles { f -> f.extension == "txt" }?.forEach { it.delete() } + for ((target, file) in downloaded) { + file.copyTo(destination.resolve(target)) + } + logger.lifecycle("Downloaded ${downloaded.keys.sorted()} into $destination") + } +} + +tasks.register("downloadCompiledClasses") { + projectName = project.name + destinationDir = layout.buildDirectory.dir("compiled-classes") +} diff --git a/build-logic/src/main/kotlin/pklSourceBundle.gradle.kts b/build-logic/src/main/kotlin/pklSourceBundle.gradle.kts new file mode 100644 index 000000000..fbca37a4d --- /dev/null +++ b/build-logic/src/main/kotlin/pklSourceBundle.gradle.kts @@ -0,0 +1,49 @@ +/* + * Copyright © 2026 Apple Inc. and the Pkl project authors. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +plugins { id("pklAllProjects") } + +// Sources jars of the first-party projects that this project depends on. +// Ideally we'd configure this automatically based on project dependencies. +val firstPartySourcesJars: Configuration = configurations.create("firstPartySourcesJars") + +// Sources jars of third-party dependencies. Project dependencies are not covered by this; they +// are added to `firstPartySourcesJars` instead. +val resolveSourcesJars = + tasks.register("resolveSourcesJars") { + configuration.set(configurations.named("runtimeClasspath")) + outputDir.set(layout.buildDirectory.dir("resolveSourcesJars")) + } + +val sourceBundleSpec = extensions.create("sourceBundleSpec") + +sourceBundleSpec.staticMuslLibc.convention(false) + +plugins.withId("pklJavaLibrary") { sourceBundleSpec.firstPartyJars.from(tasks.named("sourcesJar")) } + +plugins.withId("pklKotlinLibrary") { + sourceBundleSpec.firstPartyJars.from(tasks.named("sourcesJar")) +} + +sourceBundleSpec.firstPartyJars.from(firstPartySourcesJars) + +sourceBundleSpec.dependencyJars.from(resolveSourcesJars.map { fileTree(it.outputDir) }) + +// Builds every source bundle of this project; there is one for each variant that the project is +// distributed as. The variants are registered by `pklJvmSourceBundle` and `pklNativeSourceBundle`. +tasks.register("sourceBundle") { + group = "build" + description = "Builds the source bundles of all variants of this project." +} diff --git a/libpkl/libpkl.gradle.kts b/libpkl/libpkl.gradle.kts index f2e773e69..cd7d6b3c2 100644 --- a/libpkl/libpkl.gradle.kts +++ b/libpkl/libpkl.gradle.kts @@ -23,6 +23,7 @@ plugins { id("pklGraalVm") id("pklJavaLibrary") id("pklNativeLifecycle") + id("pklNativeSourceBundle") } val stagedMacAarch64NativeLibrary: Configuration = @@ -66,6 +67,11 @@ dependencies { nativeTestRuntimeOnly("org.junit.platform:junit-platform-launcher") externalReaderFixtureConfiguration(project(":pkl-core", "externalReaderFixture")) + + // firstPartySourcesJars(project(":pkl-formatter", "sourcesJar")) // TODO when syntax merges + firstPartySourcesJars(project(":pkl-core", "sourcesJar")) + firstPartySourcesJars(project(":pkl-parser", "sourcesJar")) + firstPartySourcesJars(project(":pkl-server", "sourcesJar")) } tasks.withType(CCompile::class) { @@ -146,19 +152,20 @@ val windowsNativeImageAmd64 = extraNativeImageArgs.addAll("-Dfile.encoding=UTF-8", "-H:-CheckToolchain") } +val nativeImageBuildTask = + when (buildInfo.targetMachine) { + Target.MacosAarch64 -> macNativeImageAarch64 + Target.LinuxAarch64 -> linuxNativeImageAarch64 + Target.LinuxAmd64 -> linuxNativeImageAmd64 + Target.WindowsAmd64 -> windowsNativeImageAmd64 + Target.AlpineLinuxAmd64 -> alpineLinuxNativeImageAmd64 + } + val buildNativeImageLibrary = tasks.register("buildNativeImageLibrary") { group = "build" - val underlyingTask = - when (buildInfo.targetMachine) { - Target.MacosAarch64 -> macNativeImageAarch64 - Target.LinuxAarch64 -> linuxNativeImageAarch64 - Target.LinuxAmd64 -> linuxNativeImageAmd64 - Target.WindowsAmd64 -> windowsNativeImageAmd64 - Target.AlpineLinuxAmd64 -> alpineLinuxNativeImageAmd64 - } - dependsOn(underlyingTask) - outputs.files(underlyingTask) + dependsOn(nativeImageBuildTask) + outputs.files(nativeImageBuildTask) } val buildPklObjectFile = @@ -282,6 +289,17 @@ val buildSharedLibrary = } } +// Source bundles for each target are registered by `pklNativeSourceBundle`. libpkl additionally +// includes its own C sources, which are the same for all targets. +val libpklNativeSourcesJar = + tasks.register("libpklNativeSourcesJar") { + archiveFileName = "libpkl-native-sources.jar" + destinationDirectory = layout.buildDirectory.dir("tmp/nativeImageSources") + from("src/main/c") + } + +sourceBundleSpec { extraNativeSources.from(libpklNativeSourcesJar) } + val Target.outputDir get() = layout.buildDirectory.dir("native-libs/$targetName") diff --git a/pkl-cli/pkl-cli.gradle.kts b/pkl-cli/pkl-cli.gradle.kts index c461bd811..79f5d9422 100644 --- a/pkl-cli/pkl-cli.gradle.kts +++ b/pkl-cli/pkl-cli.gradle.kts @@ -65,6 +65,15 @@ dependencies { testImplementation(projects.pklCommonsTest) testImplementation(libs.wiremock) + + // used by `pklJavaExecutable` plugin (ideally this would be inferred automatically) + firstPartySourcesJars(project(":pkl-cli", "sourcesJar")) + firstPartySourcesJars(project(":pkl-commons", "sourcesJar")) + firstPartySourcesJars(project(":pkl-commons-cli", "sourcesJar")) + firstPartySourcesJars(project(":pkl-core", "sourcesJar")) + firstPartySourcesJars(project(":pkl-formatter", "sourcesJar")) + firstPartySourcesJars(project(":pkl-parser", "sourcesJar")) + firstPartySourcesJars(project(":pkl-server", "sourcesJar")) } tasks.jar { diff --git a/pkl-codegen-java/pkl-codegen-java.gradle.kts b/pkl-codegen-java/pkl-codegen-java.gradle.kts index f8746eed5..4a96bd66c 100644 --- a/pkl-codegen-java/pkl-codegen-java.gradle.kts +++ b/pkl-codegen-java/pkl-codegen-java.gradle.kts @@ -30,6 +30,13 @@ dependencies { testImplementation(projects.pklConfigJava) testImplementation(projects.pklCommonsTest) + + // used by `pklJavaExecutable` plugin (ideally this would be inferred automatically) + firstPartySourcesJars(project(":pkl-codegen-java", "sourcesJar")) + firstPartySourcesJars(project(":pkl-commons", "sourcesJar")) + firstPartySourcesJars(project(":pkl-commons-cli", "sourcesJar")) + firstPartySourcesJars(project(":pkl-core", "sourcesJar")) + firstPartySourcesJars(project(":pkl-parser", "sourcesJar")) } executable { diff --git a/pkl-codegen-kotlin/pkl-codegen-kotlin.gradle.kts b/pkl-codegen-kotlin/pkl-codegen-kotlin.gradle.kts index 6f6bef9a2..0917d9aae 100644 --- a/pkl-codegen-kotlin/pkl-codegen-kotlin.gradle.kts +++ b/pkl-codegen-kotlin/pkl-codegen-kotlin.gradle.kts @@ -53,6 +53,13 @@ dependencies { testImplementation(projects.pklConfigKotlin) testImplementation(projects.pklCommonsTest) testRuntimeOnly(libs.kotlinScripting) + + // used by `pklJavaExecutable` plugin (ideally this would be inferred automatically) + firstPartySourcesJars(project(":pkl-codegen-kotlin", "sourcesJar")) + firstPartySourcesJars(project(":pkl-commons", "sourcesJar")) + firstPartySourcesJars(project(":pkl-commons-cli", "sourcesJar")) + firstPartySourcesJars(project(":pkl-core", "sourcesJar")) + firstPartySourcesJars(project(":pkl-parser", "sourcesJar")) } executable { diff --git a/pkl-config-java/pkl-config-java.gradle.kts b/pkl-config-java/pkl-config-java.gradle.kts index f2538b631..bb189543f 100644 --- a/pkl-config-java/pkl-config-java.gradle.kts +++ b/pkl-config-java/pkl-config-java.gradle.kts @@ -87,7 +87,9 @@ dependencies { testImplementation(libs.javaxInject) + // firstPartySourcesJars(project(":pkl-formatter", "sourcesJar")) // TODO when syntax merges firstPartySourcesJars(project(":pkl-core", "sourcesJar")) + firstPartySourcesJars(project(":pkl-parser", "sourcesJar")) pklCodegenJava(projects.pklCodegenJava) } diff --git a/pkl-doc/pkl-doc.gradle.kts b/pkl-doc/pkl-doc.gradle.kts index 6ac1381bb..fe07a1871 100644 --- a/pkl-doc/pkl-doc.gradle.kts +++ b/pkl-doc/pkl-doc.gradle.kts @@ -56,6 +56,12 @@ dependencies { // Graal.JS testImplementation(libs.graalSdk) testImplementation(libs.graalJs) + + // used by `pklJavaExecutable` plugin (ideally this would be inferred automatically) + firstPartySourcesJars(project(":pkl-commons", "sourcesJar")) + firstPartySourcesJars(project(":pkl-commons-cli", "sourcesJar")) + firstPartySourcesJars(project(":pkl-core", "sourcesJar")) + firstPartySourcesJars(project(":pkl-parser", "sourcesJar")) } publishing { diff --git a/pkl-tools/pkl-tools.gradle.kts b/pkl-tools/pkl-tools.gradle.kts index f912a7f10..0371014c4 100644 --- a/pkl-tools/pkl-tools.gradle.kts +++ b/pkl-tools/pkl-tools.gradle.kts @@ -48,9 +48,14 @@ dependencies { firstPartySourcesJars(project(":pkl-cli", "sourcesJar")) firstPartySourcesJars(project(":pkl-codegen-java", "sourcesJar")) firstPartySourcesJars(project(":pkl-codegen-kotlin", "sourcesJar")) + firstPartySourcesJars(project(":pkl-commons", "sourcesJar")) + firstPartySourcesJars(project(":pkl-commons-cli", "sourcesJar")) firstPartySourcesJars(project(":pkl-config-java", "sourcesJar")) firstPartySourcesJars(project(":pkl-core", "sourcesJar")) firstPartySourcesJars(project(":pkl-doc", "sourcesJar")) + firstPartySourcesJars(project(":pkl-formatter", "sourcesJar")) + firstPartySourcesJars(project(":pkl-parser", "sourcesJar")) + firstPartySourcesJars(project(":pkl-server", "sourcesJar")) } // TODO: need to figure out how to properly generate javadoc here. diff --git a/settings.gradle.kts b/settings.gradle.kts index eb699f0c0..b0c6ee3dd 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -68,7 +68,40 @@ pluginManagement { plugins { id("org.gradle.toolchains.foojay-resolver-convention") version ("1.0.0") } -@Suppress("UnstableApiUsage") dependencyResolutionManagement { repositories { mavenCentral() } } +@Suppress("UnstableApiUsage") +dependencyResolutionManagement { + repositories { + mavenCentral() + // Source archives of tagged GitHub releases (used to bundle sources of native executables). + exclusiveContent { + forRepository { + ivy { + name = "GitHub tag archives" + url = uri("https://github.com/") + patternLayout { artifact("[organisation]/[module]/archive/refs/tags/[revision].tar.gz") } + metadataSources { artifact() } + } + } + filter { + includeGroup("madler") + includeGroup("openjdk") + includeGroup("oracle") + } + } + // musl's release tarballs (used to bundle sources of statically linked musl executables). + exclusiveContent { + forRepository { + ivy { + name = "musl releases" + url = uri("https://musl.libc.org/releases/") + patternLayout { artifact("[module]-[revision].tar.gz") } + metadataSources { artifact() } + } + } + filter { includeGroup("musl") } + } + } +} for (prj in rootProject.children) { prj.buildFileName = "${prj.name}.gradle.kts"