diff --git a/.idea/inspectionProfiles/Project_Default.xml b/.idea/inspectionProfiles/Project_Default.xml index 8cc562db..45a03ea6 100644 --- a/.idea/inspectionProfiles/Project_Default.xml +++ b/.idea/inspectionProfiles/Project_Default.xml @@ -27,6 +27,12 @@ @@ -98,4 +104,4 @@ - \ No newline at end of file + diff --git a/docs/modules/ROOT/partials/component-attributes.adoc b/docs/modules/ROOT/partials/component-attributes.adoc index 7ad1a656..c7caac5c 100644 --- a/docs/modules/ROOT/partials/component-attributes.adoc +++ b/docs/modules/ROOT/partials/component-attributes.adoc @@ -78,8 +78,10 @@ endif::[] :uri-stdlib-pklbinaryModule: {uri-pkl-stdlib-docs}/pklbinary :uri-stdlib-yamlModule: {uri-pkl-stdlib-docs}/yaml :uri-stdlib-YamlParser: {uri-stdlib-yamlModule}/Parser +:uri-stdlib-projectModule: {uri-pkl-stdlib-docs}/Project :uri-stdlib-evaluatorSettingsModule: {uri-pkl-stdlib-docs}/EvaluatorSettings :uri-stdlib-evaluatorSettingsHttpClass: {uri-stdlib-evaluatorSettingsModule}/Http +:uri-stdlib-evaluatorSettingsExternalReaderClass: {uri-stdlib-evaluatorSettingsModule}/ExternalReader :uri-stdlib-Boolean: {uri-stdlib-baseModule}/Boolean :uri-stdlib-xor: {uri-stdlib-baseModule}/Boolean#xor() :uri-stdlib-implies: {uri-stdlib-baseModule}/Boolean#implies() diff --git a/docs/modules/release-notes/pages/0.32.adoc b/docs/modules/release-notes/pages/0.32.adoc index 53595679..f2687322 100644 --- a/docs/modules/release-notes/pages/0.32.adoc +++ b/docs/modules/release-notes/pages/0.32.adoc @@ -21,17 +21,31 @@ XXX Ready when you need them. -.XXX -[%collapsible] -==== -XXX -==== +=== Standard Library Changes + +==== `pkl:EvaluatorSettings` + +**Additions** + +* New method: link:{uri-stdlib-evaluatorSettingsModule}#resolve()[`EvaluatorSettings.resolve()`] +* New method: link:{uri-stdlib-evaluatorSettingsModule}#resolveForOs()[`EvaluatorSettings.resolveForOs()`] +* New property: link:{uri-stdlib-evaluatorSettingsExternalReaderClass}#workingDir[`EvaluatorSettings#ExternalReader.workingDir`] + +==== `pkl:Project` + +**Additions** + +* New property: link:{uri-stdlib-projectModule}#resolvedEvaluatorSettings[`Project.resolvedEvaluatorSettings`] == Breaking Changes [small]#💔# Things to watch out for when upgrading. -=== Removed Java APIs +=== Java API changes + +Changes have been made to the Java API. + +==== Removals and deprecations The following APIs have been removed without replacement. @@ -41,11 +55,26 @@ The following APIs have been deprecated for removal. * `org.pkl.config.java.mapper.NonNull` (https://github.com/apple/pkl/pull/1607[#1607]). -.XXX -[%collapsible] -==== -XXX -==== +==== Changes + +* `org.pkl.core.evaluatorSettings.PklEvaluatorSettings.parse` no longer accepts a `pathNormalizer` argument. + +=== Loading rule changes in `pkl:EvaluatorSettings` + +Breaking changes have been made to how evaluator settings are loaded (https://github.com/apple/pkl/pull/1394[#1394]). + +==== Loading rule changes for the external reader executable + +The following changes have been made for the `executable` property in an external reader: + +* If the executable does not contain a slash (`/` on POSIX, `\` on Windows) character, it is always resolved against the `PATH` environment variable. +* If it does contain a slash, it is resolved relative to the enclosing PklProject directory, instead of the current working directory. + +=== Changes to `--external-module-reader` and `--external-resource-reader` CLI flags + +The `--external-module-reader` and `--external-resource-reader` CLI flags will _replace_ any external readers otherwise configured within a PklProject, instead of add to it (https://github.com/apple/pkl/pull/1394[#1394]). + +This makes this behavior consistent with how other settings work. == Work In Progress [small]#🚆# diff --git a/pkl-commons-cli/src/main/kotlin/org/pkl/commons/cli/CliBaseOptions.kt b/pkl-commons-cli/src/main/kotlin/org/pkl/commons/cli/CliBaseOptions.kt index af58ad48..0a7274fe 100644 --- a/pkl-commons-cli/src/main/kotlin/org/pkl/commons/cli/CliBaseOptions.kt +++ b/pkl-commons-cli/src/main/kotlin/org/pkl/commons/cli/CliBaseOptions.kt @@ -148,10 +148,10 @@ data class CliBaseOptions( val httpHeaders: Map>>? = null, /** External module reader process specs */ - val externalModuleReaders: Map = mapOf(), + val externalModuleReaders: Map? = null, /** External resource reader process specs */ - val externalResourceReaders: Map = mapOf(), + val externalResourceReaders: Map? = null, /** Defines options for the formatting of calls to the trace() method. */ val traceMode: TraceMode? = null, diff --git a/pkl-commons-cli/src/main/kotlin/org/pkl/commons/cli/CliCommand.kt b/pkl-commons-cli/src/main/kotlin/org/pkl/commons/cli/CliCommand.kt index c944a0ee..c34611d9 100644 --- a/pkl-commons-cli/src/main/kotlin/org/pkl/commons/cli/CliCommand.kt +++ b/pkl-commons-cli/src/main/kotlin/org/pkl/commons/cli/CliCommand.kt @@ -110,7 +110,7 @@ abstract class CliCommand(protected val cliOptions: CliBaseOptions) { private val evaluatorSettings: PklEvaluatorSettings? by lazy { @Suppress("PklCliDirectProjectEvaluatorSettingsAccess") - if (cliOptions.omitProjectSettings) null else project?.evaluatorSettings + if (cliOptions.omitProjectSettings) null else project?.resolvedEvaluatorSettings } protected val allowedModules: List by lazy { @@ -193,11 +193,11 @@ abstract class CliCommand(protected val cliOptions: CliBaseOptions) { } protected val externalModuleReaders: Map by lazy { - (evaluatorSettings?.externalModuleReaders ?: emptyMap()) + cliOptions.externalModuleReaders + cliOptions.externalModuleReaders ?: evaluatorSettings?.externalModuleReaders ?: mapOf() } protected val externalResourceReaders: Map by lazy { - (evaluatorSettings?.externalResourceReaders ?: emptyMap()) + cliOptions.externalResourceReaders + cliOptions.externalResourceReaders ?: evaluatorSettings?.externalResourceReaders ?: mapOf() } private val externalProcesses: diff --git a/pkl-commons-cli/src/main/kotlin/org/pkl/commons/cli/commands/BaseOptions.kt b/pkl-commons-cli/src/main/kotlin/org/pkl/commons/cli/commands/BaseOptions.kt index 62ba7653..359ae10c 100644 --- a/pkl-commons-cli/src/main/kotlin/org/pkl/commons/cli/commands/BaseOptions.kt +++ b/pkl-commons-cli/src/main/kotlin/org/pkl/commons/cli/commands/BaseOptions.kt @@ -55,7 +55,8 @@ class BaseOptions : OptionGroup() { if (IoUtils.isUriLike(moduleName)) URI(moduleName) // Can't just use URI constructor, because URI(null, null, "C:/foo/bar", null) turns // into `URI("C", null, "/foo/bar", null)`. - else if (IoUtils.isWindowsAbsolutePath(moduleName)) Path.of(moduleName).toUri() + else if (IoUtils.isWindows() && IoUtils.isWindowsAbsolutePath(moduleName)) + Path.of(moduleName).toUri() else URI(null, null, IoUtils.toNormalizedPathString(Path.of(moduleName)), null) } catch (e: URISyntaxException) { val message = buildString { @@ -91,7 +92,7 @@ class BaseOptions : OptionGroup() { > { return splitPair(delimiter).convert { val cmd = shlex(it.second) - Pair(it.first, ExternalReader(cmd.first(), cmd.drop(1))) + Pair(it.first, ExternalReader(cmd.first(), cmd.drop(1), null)) } } @@ -386,8 +387,8 @@ class BaseOptions : OptionGroup() { httpNoProxy = noProxy, httpRewrites = httpRewrites.ifEmpty { null }, httpHeaders = httpHeaders.ifEmpty { null }, - externalModuleReaders = externalModuleReaders, - externalResourceReaders = externalResourceReaders, + externalModuleReaders = externalModuleReaders.ifEmpty { null }, + externalResourceReaders = externalResourceReaders.ifEmpty { null }, traceMode = traceMode, powerAssertionsEnabled = powerAssertionsEnabled, ) diff --git a/pkl-commons-cli/src/test/kotlin/org/pkl/commons/cli/BaseCommandTest.kt b/pkl-commons-cli/src/test/kotlin/org/pkl/commons/cli/BaseCommandTest.kt index ac7ed75a..15b3dd10 100644 --- a/pkl-commons-cli/src/test/kotlin/org/pkl/commons/cli/BaseCommandTest.kt +++ b/pkl-commons-cli/src/test/kotlin/org/pkl/commons/cli/BaseCommandTest.kt @@ -1,5 +1,5 @@ /* - * Copyright © 2024-2025 Apple Inc. and the Pkl project authors. All rights reserved. + * Copyright © 2024-2026 Apple Inc. and the Pkl project authors. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -98,17 +98,17 @@ class BaseCommandTest { assertThat(cmd.baseOptions.externalModuleReaders) .isEqualTo( mapOf( - "scheme3" to ExternalReader("reader3", emptyList()), - "scheme4" to ExternalReader("reader4", listOf("with", "args")), - "scheme+ext" to ExternalReader("reader5", listOf("with", "args")), + "scheme3" to ExternalReader("reader3", emptyList(), null), + "scheme4" to ExternalReader("reader4", listOf("with", "args"), null), + "scheme+ext" to ExternalReader("reader5", listOf("with", "args"), null), ) ) assertThat(cmd.baseOptions.externalResourceReaders) .isEqualTo( mapOf( - "scheme1" to ExternalReader("reader1", emptyList()), - "scheme2" to ExternalReader("reader2", listOf("with", "args")), - "scheme+ext" to ExternalReader("reader5", listOf("with", "args")), + "scheme1" to ExternalReader("reader1", emptyList(), null), + "scheme2" to ExternalReader("reader2", listOf("with", "args"), null), + "scheme+ext" to ExternalReader("reader5", listOf("with", "args"), null), ) ) } diff --git a/pkl-commons-cli/src/test/kotlin/org/pkl/commons/cli/CliCommandTest.kt b/pkl-commons-cli/src/test/kotlin/org/pkl/commons/cli/CliCommandTest.kt index 07a5f981..d6c1ab75 100644 --- a/pkl-commons-cli/src/test/kotlin/org/pkl/commons/cli/CliCommandTest.kt +++ b/pkl-commons-cli/src/test/kotlin/org/pkl/commons/cli/CliCommandTest.kt @@ -28,12 +28,12 @@ import org.pkl.commons.cli.commands.BaseCommand import org.pkl.commons.cli.commands.ProjectOptions import org.pkl.commons.writeString import org.pkl.core.SecurityManagers +import org.pkl.core.evaluatorSettings.PklEvaluatorSettings import org.pkl.core.evaluatorSettings.TraceMode import org.pkl.core.util.IoUtils @OptIn(ExperimentalPathApi::class) class CliCommandTest { - private val cmd = object : BaseCommand("test", "") { val projectOptions: ProjectOptions by ProjectOptions() @@ -172,4 +172,32 @@ class CliCommandTest { .isNotNull } } + + @Test + fun `--external-module-reader blows away PklProject externalModuleReaders`( + @TempDir tempDir: Path + ) { + tempDir + .resolve("PklProject") + .writeString( + // language=pkl + """ + amends "pkl:Project" + + evaluatorSettings { + externalModuleReaders { + ["foo"] { + executable = "foo" + } + } + } + """ + .trimIndent() + ) + cmd.parse(arrayOf("--external-module-reader", "bar=bar")) + val opts = cmd.baseOptions.baseOptions(emptyList(), null, true) + val cliTest = CliTest(opts) + assertThat(cliTest.myExternalModuleReaders) + .isEqualTo(mapOf("bar" to PklEvaluatorSettings.ExternalReader("bar", listOf(), null))) + } } diff --git a/pkl-core/pkl-core.gradle.kts b/pkl-core/pkl-core.gradle.kts index 243e94f4..d715386d 100644 --- a/pkl-core/pkl-core.gradle.kts +++ b/pkl-core/pkl-core.gradle.kts @@ -27,7 +27,17 @@ plugins { idea } -val generatorSourceSet = sourceSets.register("generator") +val generatorSourceSet: NamedDomainObjectProvider = sourceSets.register("generator") + +val externalReaderFixtureSourceSet: NamedDomainObjectProvider = + sourceSets.register("externalReaderFixture") { + compileClasspath += sourceSets.test.get().output + sourceSets.test.get().compileClasspath + runtimeClasspath += sourceSets.test.get().output + sourceSets.test.get().runtimeClasspath + } + +val externalReaderFixtureImplementation: Configuration by configurations.getting { + extendsFrom(configurations.testImplementation.get()) +} idea { module { @@ -126,8 +136,45 @@ tasks.withType().configureEach { tasks.compileKotlin { enabled = false } +val externalReaderFixture by tasks.registering { + group = "build" + dependsOn(tasks.named("compileExternalReaderFixtureJava")) + inputs.files(externalReaderFixtureSourceSet.map { it.output }) + val fileName = if (buildInfo.os.isWindows) "externalreader.bat" else "externalreader" + val outputFile = layout.buildDirectory.file("fixtures/$fileName") + outputs.file(outputFile) + doLast { + val classpath = externalReaderFixtureSourceSet.get().runtimeClasspath.asPath + val scriptContent = + if (buildInfo.os.isWindows) { + """ + @echo off + java -cp $classpath org.pkl.core.externalreaderfixture.Main + """ + .trimIndent() + } else { + """ + #!/usr/bin/env bash + + java -cp $classpath org.pkl.core.externalreaderfixture.Main + """ + .trimIndent() + } + + outputFile.get().asFile.writeText(scriptContent) + outputFile.get().asFile.setExecutable(true) + println("Created external reader ${outputFile.get().asFile.absolutePath}") + } +} + tasks.test { configureTest() + dependsOn(externalReaderFixture) + environment( + "PATH", + listOf(System.getenv("PATH"), layout.buildDirectory.dir("fixtures/").get()) + .joinToString(File.pathSeparator), + ) useJUnitPlatform { excludeEngines("MacAmd64LanguageSnippetTestsEngine") excludeEngines("MacAarch64LanguageSnippetTestsEngine") @@ -139,6 +186,11 @@ tasks.test { // testing very large lists requires more memory than the default 512m! maxHeapSize = "1g" + + systemProperty( + "org.pkl.core.testExternalReaderPath", + externalReaderFixture.map { it.outputs.files.singleFile.absolutePath }, + ) } val generateBaseModuleMembers by diff --git a/pkl-core/src/externalReaderFixture/kotlin/org/pkl/core/externalreader/util.kt b/pkl-core/src/externalReaderFixture/kotlin/org/pkl/core/externalreader/util.kt new file mode 100644 index 00000000..c321eed9 --- /dev/null +++ b/pkl-core/src/externalReaderFixture/kotlin/org/pkl/core/externalreader/util.kt @@ -0,0 +1,24 @@ +/* + * Copyright © 2026 Apple Inc. and the Pkl project authors. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.pkl.core.externalreader + +import org.pkl.core.messaging.MessageTransports + +private val systemInDecoder = ExternalReaderMessagePackDecoder(System.`in`) + +private val systemOutEncoder = ExternalReaderMessagePackEncoder(System.out) + +val stdioTransport = MessageTransports.stream(systemInDecoder, systemOutEncoder) {} diff --git a/pkl-core/src/externalReaderFixture/kotlin/org/pkl/core/externalreaderfixture/Main.kt b/pkl-core/src/externalReaderFixture/kotlin/org/pkl/core/externalreaderfixture/Main.kt new file mode 100644 index 00000000..478466a7 --- /dev/null +++ b/pkl-core/src/externalReaderFixture/kotlin/org/pkl/core/externalreaderfixture/Main.kt @@ -0,0 +1,60 @@ +/* + * Copyright © 2026 Apple Inc. and the Pkl project authors. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +@file:JvmName("Main") + +package org.pkl.core.externalreaderfixture + +import java.net.URI +import org.pkl.core.externalreader.ExternalModuleReader +import org.pkl.core.externalreader.ExternalReaderClient +import org.pkl.core.externalreader.ExternalResourceReader +import org.pkl.core.externalreader.stdioTransport +import org.pkl.core.module.PathElement + +object ModuleReader : ExternalModuleReader { + override val isLocal: Boolean = true + + override fun read(uri: URI): String = "hello" + + override val scheme: String = "foo" + + override val hasHierarchicalUris: Boolean = false + + override val isGlobbable: Boolean = false + + override fun listElements(uri: URI): List { + throw NotImplementedError() + } +} + +object ResourceReader : ExternalResourceReader { + override fun read(uri: URI): ByteArray = "hello".toByteArray() + + override val scheme: String = "foo" + + override val hasHierarchicalUris: Boolean = false + + override val isGlobbable: Boolean = false + + override fun listElements(uri: URI): List { + throw NotImplementedError() + } +} + +fun main() { + val client = ExternalReaderClient(listOf(ModuleReader), listOf(ResourceReader), stdioTransport) + client.run() +} diff --git a/pkl-core/src/main/java/org/pkl/core/EvaluatorBuilder.java b/pkl-core/src/main/java/org/pkl/core/EvaluatorBuilder.java index 8e1d9c4f..3f163616 100644 --- a/pkl-core/src/main/java/org/pkl/core/EvaluatorBuilder.java +++ b/pkl-core/src/main/java/org/pkl/core/EvaluatorBuilder.java @@ -488,7 +488,7 @@ public final class EvaluatorBuilder { */ public EvaluatorBuilder applyFromProject(Project project) { this.dependencies = project.getDependencies(); - var settings = project.getEvaluatorSettings(); + var settings = project.getResolvedEvaluatorSettings(); if (securityManager != null) { throw new IllegalStateException( "Cannot call both `setSecurityManager` and `setProject`, because both define security manager settings. Call `setProjectOnly` if the security manager is desired."); diff --git a/pkl-core/src/main/java/org/pkl/core/evaluatorSettings/PklEvaluatorSettings.java b/pkl-core/src/main/java/org/pkl/core/evaluatorSettings/PklEvaluatorSettings.java index 255f9284..4fdfdecb 100644 --- a/pkl-core/src/main/java/org/pkl/core/evaluatorSettings/PklEvaluatorSettings.java +++ b/pkl-core/src/main/java/org/pkl/core/evaluatorSettings/PklEvaluatorSettings.java @@ -26,7 +26,6 @@ import java.util.Locale; import java.util.Map; import java.util.Map.Entry; import java.util.Objects; -import java.util.function.BiFunction; import java.util.regex.Pattern; import java.util.stream.Collectors; import org.jspecify.annotations.Nullable; @@ -57,17 +56,13 @@ public record PklEvaluatorSettings( /** Initializes a {@link PklEvaluatorSettings} from a raw object representation. */ @SuppressWarnings("unchecked") - public static PklEvaluatorSettings parse( - Value input, BiFunction pathNormalizer) { + public static PklEvaluatorSettings parse(Value input) { if (!(input instanceof PObject pSettings)) { throw PklBugException.unreachableCode(); } var moduleCacheDirStr = (String) pSettings.get("moduleCacheDir"); - var moduleCacheDir = - moduleCacheDirStr == null - ? null - : pathNormalizer.apply(moduleCacheDirStr, "moduleCacheDir"); + var moduleCacheDir = moduleCacheDirStr == null ? null : Path.of(moduleCacheDirStr); var allowedModulesStrs = (List) pSettings.get("allowedModules"); var allowedModules = @@ -82,13 +77,10 @@ public record PklEvaluatorSettings( : allowedResourcesStrs.stream().map(Pattern::compile).toList(); var modulePathStrs = (List) pSettings.get("modulePath"); - var modulePath = - modulePathStrs == null - ? null - : modulePathStrs.stream().map(it -> pathNormalizer.apply(it, "modulePath")).toList(); + var modulePath = modulePathStrs == null ? null : modulePathStrs.stream().map(Path::of).toList(); var rootDirStr = (String) pSettings.get("rootDir"); - var rootDir = rootDirStr == null ? null : pathNormalizer.apply(rootDirStr, "rootDir"); + var rootDir = rootDirStr == null ? null : Path.of(rootDirStr); var externalModuleReadersRaw = (Map) pSettings.get("externalModuleReaders"); var externalModuleReaders = @@ -219,13 +211,16 @@ public record PklEvaluatorSettings( } } - public record ExternalReader(String executable, @Nullable List arguments) { + public record ExternalReader( + String executable, @Nullable List arguments, @Nullable String workingDir) { @SuppressWarnings("unchecked") public static ExternalReader parse(Value input) { if (input instanceof PObject externalReader) { var executable = (String) externalReader.getProperty("executable"); var arguments = (List) externalReader.get("arguments"); - return new ExternalReader(executable, arguments); + var workingDir = externalReader.getProperty("workingDir"); + return new ExternalReader( + executable, arguments, workingDir instanceof PNull ? null : (String) workingDir); } throw PklBugException.unreachableCode(); } diff --git a/pkl-core/src/main/java/org/pkl/core/externalreader/ExternalReaderProcessImpl.java b/pkl-core/src/main/java/org/pkl/core/externalreader/ExternalReaderProcessImpl.java index fc5fa8fc..1c81b5ae 100644 --- a/pkl-core/src/main/java/org/pkl/core/externalreader/ExternalReaderProcessImpl.java +++ b/pkl-core/src/main/java/org/pkl/core/externalreader/ExternalReaderProcessImpl.java @@ -16,6 +16,7 @@ package org.pkl.core.externalreader; import com.google.errorprone.annotations.concurrent.GuardedBy; +import java.io.File; import java.io.IOException; import java.lang.ProcessBuilder.Redirect; import java.time.Duration; @@ -105,6 +106,10 @@ final class ExternalReaderProcessImpl implements ExternalReaderProcess { } var builder = new ProcessBuilder(command); + var workingDir = spec.workingDir(); + if (workingDir != null) { + builder.directory(new File(workingDir)); + } builder.redirectError(Redirect.INHERIT); // inherit stderr from this pkl process try { process = builder.start(); diff --git a/pkl-core/src/main/java/org/pkl/core/project/Project.java b/pkl-core/src/main/java/org/pkl/core/project/Project.java index a004d0b8..99214346 100644 --- a/pkl-core/src/main/java/org/pkl/core/project/Project.java +++ b/pkl-core/src/main/java/org/pkl/core/project/Project.java @@ -62,6 +62,7 @@ public final class Project { private final @Nullable Package pkg; private final DeclaredDependencies dependencies; private final PklEvaluatorSettings evaluatorSettings; + private final PklEvaluatorSettings resolvedEvaluatorSettings; private final URI projectFileUri; private final URI projectBaseUri; private final List tests; @@ -290,7 +291,7 @@ public final class Project { public static Project parseProject(PObject module) throws URISyntaxException { var pkgObj = getNullableProperty(module, "package"); - var projectFileUri = URI.create((String) module.getProperty("projectFileUri")); + var projectFileUri = new URI((String) module.getProperty("projectFileUri")); var dependencies = parseDependencies(module, projectFileUri, null); var projectBaseUri = IoUtils.resolve(projectFileUri, "."); Package pkg = null; @@ -301,9 +302,14 @@ public final class Project { getProperty( module, "evaluatorSettings", - (settings) -> - PklEvaluatorSettings.parse( - (Value) settings, (it, name) -> resolveNullablePath(it, projectBaseUri, name))); + (settings) -> PklEvaluatorSettings.parse((Value) settings)); + + var resolvedEvaluatorSettings = + getProperty( + module, + "resolvedEvaluatorSettings", + (settings) -> PklEvaluatorSettings.parse((Value) settings)); + @SuppressWarnings("unchecked") var testPathStrs = (List) getProperty(module, "tests"); var tests = @@ -316,6 +322,7 @@ public final class Project { pkg, dependencies, evaluatorSettings, + resolvedEvaluatorSettings, projectFileUri, projectBaseUri, tests, @@ -371,24 +378,6 @@ public final class Project { return new URI((String) value); } - /** - * Resolve a path string against projectBaseUri. - * - * @throws PackageLoadError if projectBaseUri is not a {@code file:} URI. - */ - private static @Nullable Path resolveNullablePath( - @Nullable String path, URI projectBaseUri, String propertyName) { - if (path == null) { - return null; - } - try { - return Path.of(projectBaseUri).resolve(path).normalize(); - } catch (FileSystemNotFoundException e) { - throw new PackageLoadError( - "relativePathPropertyDefinedByProjectFromNonFileUri", projectBaseUri, propertyName); - } - } - @SuppressWarnings("unchecked") private static Package parsePackage(PObject pObj) throws URISyntaxException { var name = (String) pObj.getProperty("name"); @@ -430,6 +419,7 @@ public final class Project { @Nullable Package pkg, DeclaredDependencies dependencies, PklEvaluatorSettings evaluatorSettings, + PklEvaluatorSettings resolvedEvaluatorSettings, URI projectFileUri, URI projectBaseUri, List tests, @@ -438,6 +428,7 @@ public final class Project { this.pkg = pkg; this.dependencies = dependencies; this.evaluatorSettings = evaluatorSettings; + this.resolvedEvaluatorSettings = resolvedEvaluatorSettings; this.projectFileUri = projectFileUri; this.projectBaseUri = projectBaseUri; this.tests = tests; @@ -459,6 +450,15 @@ public final class Project { return evaluatorSettings; } + /** + * The evaluator settings whose paths have been resolved against the project dir. + * + * @since 0.32.0 + */ + public PklEvaluatorSettings getResolvedEvaluatorSettings() { + return resolvedEvaluatorSettings; + } + public URI getProjectFileUri() { return projectFileUri; } @@ -488,6 +488,7 @@ public final class Project { return Objects.equals(pkg, project.pkg) && dependencies.equals(project.dependencies) && evaluatorSettings.equals(project.evaluatorSettings) + && resolvedEvaluatorSettings.equals(project.resolvedEvaluatorSettings) && projectFileUri.equals(project.projectFileUri) && tests.equals(project.tests) && annotations.equals(project.annotations); @@ -495,7 +496,14 @@ public final class Project { @Override public int hashCode() { - return Objects.hash(pkg, dependencies, evaluatorSettings, projectFileUri, tests, annotations); + return Objects.hash( + pkg, + dependencies, + evaluatorSettings, + resolvedEvaluatorSettings, + projectFileUri, + tests, + annotations); } public DeclaredDependencies getDependencies() { @@ -506,6 +514,7 @@ public final class Project { return localProjectDependencies; } + @SuppressWarnings("unused") public URI getProjectBaseUri() { return projectBaseUri; } diff --git a/pkl-core/src/main/java/org/pkl/core/runtime/CommandSpecParser.java b/pkl-core/src/main/java/org/pkl/core/runtime/CommandSpecParser.java index 400a4e1a..720dc894 100644 --- a/pkl-core/src/main/java/org/pkl/core/runtime/CommandSpecParser.java +++ b/pkl-core/src/main/java/org/pkl/core/runtime/CommandSpecParser.java @@ -1160,7 +1160,7 @@ public final class CommandSpecParser { var uri = IoUtils.isUriLike(moduleName) ? new URI(moduleName) - : IoUtils.isWindowsAbsolutePath(moduleName) + : IoUtils.isWindows() && IoUtils.isWindowsAbsolutePath(moduleName) ? Path.of(moduleName).toUri() : new URI(null, null, IoUtils.toNormalizedPathString(Path.of(moduleName)), null); uriString = diff --git a/pkl-core/src/main/java/org/pkl/core/stdlib/evaluatorsettings/EvaluatorSettingsNodes.java b/pkl-core/src/main/java/org/pkl/core/stdlib/evaluatorsettings/EvaluatorSettingsNodes.java new file mode 100644 index 00000000..734ed89c --- /dev/null +++ b/pkl-core/src/main/java/org/pkl/core/stdlib/evaluatorsettings/EvaluatorSettingsNodes.java @@ -0,0 +1,54 @@ +/* + * Copyright © 2026 Apple Inc. and the Pkl project authors. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.pkl.core.stdlib.evaluatorsettings; + +import com.oracle.truffle.api.CompilerDirectives.TruffleBoundary; +import com.oracle.truffle.api.dsl.Specialization; +import java.net.URI; +import java.net.URISyntaxException; +import org.pkl.core.runtime.VmTyped; +import org.pkl.core.stdlib.ExternalMethod3Node; +import org.pkl.core.util.PathResolver; +import org.pkl.core.util.PathResolvers; + +public class EvaluatorSettingsNodes { + + public abstract static class resolvePath extends ExternalMethod3Node { + @TruffleBoundary + private URI toUri(String baseUri) { + try { + var uri = new URI(baseUri); + // guaranteed by Pkl + assert uri.getScheme().equals("file"); + return uri; + } catch (URISyntaxException e) { + throw exceptionBuilder().evalError("invalidUri", baseUri).build(); + } + } + + private PathResolver getPathResolver(boolean forWindows) { + return forWindows ? PathResolvers.forWindows() : PathResolvers.forPosix(); + } + + @Specialization + protected String eval(VmTyped ignored, String uriStr, String path, boolean forWindows) { + var uri = toUri(uriStr); + var baseUri = uri.resolve("."); + var resolver = getPathResolver(forWindows); + return resolver.resolvePath(baseUri, path); + } + } +} diff --git a/pkl-core/src/main/java/org/pkl/core/stdlib/evaluatorsettings/package-info.java b/pkl-core/src/main/java/org/pkl/core/stdlib/evaluatorsettings/package-info.java new file mode 100644 index 00000000..78455282 --- /dev/null +++ b/pkl-core/src/main/java/org/pkl/core/stdlib/evaluatorsettings/package-info.java @@ -0,0 +1,6 @@ +@NullMarked +@PklName("EvaluatorSettings") +package org.pkl.core.stdlib.evaluatorsettings; + +import org.jspecify.annotations.NullMarked; +import org.pkl.core.stdlib.PklName; diff --git a/pkl-core/src/main/java/org/pkl/core/util/IoUtils.java b/pkl-core/src/main/java/org/pkl/core/util/IoUtils.java index 9ef9a44f..c784a078 100644 --- a/pkl-core/src/main/java/org/pkl/core/util/IoUtils.java +++ b/pkl-core/src/main/java/org/pkl/core/util/IoUtils.java @@ -16,6 +16,7 @@ package org.pkl.core.util; import com.oracle.truffle.api.TruffleOptions; +import java.io.File; import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStream; @@ -52,7 +53,7 @@ public final class IoUtils { // Don't match paths like `C:\`, which are drive letters on Windows. private static final Pattern uriLike = Pattern.compile("[\\w+.-]+:[^\\\\].*"); - private static final Pattern windowsPathLike = Pattern.compile("\\w:\\\\.*"); + public static final Pattern windowsDriveLetterLike = Pattern.compile("[a-zA-Z]:.*"); private static final Pattern headerNameLike = Pattern.compile("[a-zA-Z0-9!#$%&'*+-.^_`|~]+"); @@ -107,8 +108,7 @@ public final class IoUtils { } public static boolean isWindowsAbsolutePath(String str) { - if (!isWindows()) return false; - return windowsPathLike.matcher(str).matches(); + return windowsDriveLetterLike.matcher(str).matches(); } /** @@ -935,4 +935,44 @@ public final class IoUtils { ErrorMessages.create("invalidHttpHeaderValueTooLong", headerValue)); } } + + private static @Nullable String getFilenameExtension(String fileName) { + var dotIndex = fileName.lastIndexOf('.'); + // 0 if hidden file (e.g. `.gitignore`); not an extension + if (dotIndex == -1 || dotIndex == 0) { + return null; + } + return fileName.substring(dotIndex + 1); + } + + public static @Nullable Path findExecutableOnPath(String executable) { + var pathEnvVar = System.getenv("PATH"); + if (pathEnvVar == null) { + return null; + } + var executableExtension = getFilenameExtension(executable); + List extensions; + if (executableExtension != null || !isWindows()) { + extensions = List.of(""); + } else { + extensions = List.of(".exe", ".cmd", ".bat", ".com"); + } + var pathDirs = pathEnvVar.split(File.pathSeparator); + for (var dir : pathDirs) { + for (var extension : extensions) { + var candidate = Path.of(dir, executable + extension); + if (Files.exists(candidate) && Files.isExecutable(candidate)) { + return candidate; + } + } + } + return null; + } + + public static URI fixFileUri(URI uri) { + if ("file".equals(uri.getScheme()) && !uri.getSchemeSpecificPart().startsWith("//")) { + return URI.create("file://" + uri.getSchemeSpecificPart()); + } + return uri; + } } diff --git a/pkl-core/src/main/java/org/pkl/core/util/PathResolver.java b/pkl-core/src/main/java/org/pkl/core/util/PathResolver.java new file mode 100644 index 00000000..731349a8 --- /dev/null +++ b/pkl-core/src/main/java/org/pkl/core/util/PathResolver.java @@ -0,0 +1,198 @@ +/* + * Copyright © 2026 Apple Inc. and the Pkl project authors. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.pkl.core.util; + +import java.net.URI; +import java.util.ArrayList; +import java.util.regex.Pattern; + +// These are implemented by hand instead of relying on the NIO Path API because the JDK does not +// provide libraries for cross-platform resolvers. +// For example, from POSIX systems, you cannot resolve Windows-style paths. +// The alternative to this approach is to depend on a library (e.g. Apache Commons IO or jimfs). +public abstract sealed class PathResolver { + public final String resolvePath(URI uri, String path) { + if (isAbsolute(path)) { + return normalize(path); + } + var basePath = uriToPath(uri); + var resolved = join(basePath, path); + return normalize(resolved); + } + + protected abstract String uriToPath(URI uri); + + protected abstract String join(String base, String path); + + protected abstract boolean isAbsolute(String path); + + protected abstract String getRoot(String path); + + protected abstract char getSeparator(); + + protected String normalize(String path) { + var root = getRoot(path); + var separator = getSeparator(); + var remainder = path.substring(root.length()); + + var parts = remainder.split(Pattern.quote(((Character) separator).toString())); + var stack = new ArrayList<>(); + + for (var part : parts) { + if (part.equals("..")) { + if (!stack.isEmpty()) { + stack.remove(stack.size() - 1); + } + } else if (!part.isEmpty() && !part.equals(".")) { + stack.add(part); + } + } + + if (stack.isEmpty()) { + return root; + } + + var sb = new StringBuilder(root); + + for (var i = 0; i < stack.size(); i++) { + if (i > 0) { + sb.append(separator); + } + sb.append(stack.get(i)); + } + // path ends with trailing separator + if (parts[parts.length - 1].isEmpty()) { + sb.append(separator); + } + + return sb.toString(); + } + + static final class WindowsPathResolver extends PathResolver { + @Override + protected String uriToPath(URI uri) { + var host = uri.getHost(); + var path = uri.getPath(); + if (host != null) { + // UNC path: \\server\share\path + return "\\\\%s%s".formatted(host, path.replace('/', '\\')); + } + var ret = path.matches("/[A-Z]:/.*") ? path.substring(1) : path; + return ret.replace('/', '\\'); + } + + @Override + protected boolean isAbsolute(String path) { + // Normalize forward slashes first + path = path.replace('/', '\\'); + return isDriveLetter(path) || path.startsWith("\\\\"); + } + + @Override + protected String join(String base, String path) { + path = path.replace('/', '\\'); + if (isAbsolute(path)) { + return path; + } + if (path.startsWith("\\")) { + // Root-relative path: skip the leading backslash to avoid double backslash + return getRoot(base) + path; + } + if (base.endsWith("\\")) { + return base + path; + } + return base + '\\' + path; + } + + private boolean isDriveLetter(String path) { + return path.length() >= 2 && isAlpha(path.charAt(0)) && path.charAt(1) == ':'; + } + + private boolean isAlpha(char character) { + return character >= 65 && character <= 90 || character >= 97 && character <= 122; + } + + @Override + protected String getRoot(String path) { + // UNC path, e.g. \\server\share + if (path.startsWith("\\\\")) { + var firstBackslash = path.indexOf('\\', 2); + if (firstBackslash == -1) { + // Malformed UNC, just return what we have + return path; + } + var secondBackslash = path.indexOf('\\', firstBackslash + 1); + if (secondBackslash == -1) { + return path + "\\"; + } + return path.substring(0, secondBackslash + 1); + } else if (isDriveLetter(path)) { + // drive letter without leading slash, e.g. `C:foo\bar` (uncommon but valid) + if (path.length() > 2 && path.charAt(2) == '\\') { + return path.substring(0, 3); + } + return path.substring(0, 2); + } else if (path.startsWith("\\")) { + // drive-relative path + return "\\"; + } + return ""; + } + + @Override + protected char getSeparator() { + return '\\'; + } + + @Override + protected String normalize(String path) { + return super.normalize(path.replace('/', '\\')); + } + } + + static final class PosixPathResolver extends PathResolver { + @Override + protected String uriToPath(URI uri) { + return uri.getPath(); + } + + @Override + protected boolean isAbsolute(String path) { + return path.startsWith("/"); + } + + @Override + protected String join(String base, String path) { + if (isAbsolute(path)) { + return path; + } + if (base.endsWith("/")) { + return base + path; + } + return base + '/' + path; + } + + @Override + protected String getRoot(String ignored) { + return "/"; + } + + @Override + protected char getSeparator() { + return '/'; + } + } +} diff --git a/pkl-core/src/main/java/org/pkl/core/util/PathResolvers.java b/pkl-core/src/main/java/org/pkl/core/util/PathResolvers.java new file mode 100644 index 00000000..e58b2fee --- /dev/null +++ b/pkl-core/src/main/java/org/pkl/core/util/PathResolvers.java @@ -0,0 +1,35 @@ +/* + * Copyright © 2026 Apple Inc. and the Pkl project authors. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.pkl.core.util; + +import org.pkl.core.util.PathResolver.PosixPathResolver; +import org.pkl.core.util.PathResolver.WindowsPathResolver; + +public final class PathResolvers { + private PathResolvers() {} + + private static final PathResolver WINDOWS = new WindowsPathResolver(); + + private static final PathResolver POSIX = new PosixPathResolver(); + + public static PathResolver forWindows() { + return WINDOWS; + } + + public static PathResolver forPosix() { + return POSIX; + } +} diff --git a/pkl-core/src/main/resources/org/pkl/core/errorMessages.properties b/pkl-core/src/main/resources/org/pkl/core/errorMessages.properties index 98fcd8ba..37cb3ce0 100644 --- a/pkl-core/src/main/resources/org/pkl/core/errorMessages.properties +++ b/pkl-core/src/main/resources/org/pkl/core/errorMessages.properties @@ -619,6 +619,9 @@ Cannot analyze relative module URI `{0}`. invalidModuleUri=\ Module URI `{0}` has invalid syntax. +invalidUri=\ +URI `{0}` has invalid syntax. + invalidModuleUriMissingSlash=\ Module URI `{0}` is missing a `/` after `{1}:`. diff --git a/pkl-core/src/test/files/LanguageSnippetTests/input/api/evaluatorSettingsModulePosix.pkl b/pkl-core/src/test/files/LanguageSnippetTests/input/api/evaluatorSettingsModulePosix.pkl new file mode 100644 index 00000000..cdcdbcfe --- /dev/null +++ b/pkl-core/src/test/files/LanguageSnippetTests/input/api/evaluatorSettingsModulePosix.pkl @@ -0,0 +1,130 @@ +amends "../snippetTest.pkl" + +import "pkl:EvaluatorSettings" +import "pkl:platform" + +// macOS and linux have the same impl; both POSIX systems +local macOS = new platform.OperatingSystem { name = "macOS" } + +local function resolve(base: String, path: String) = + new EvaluatorSettings { rootDir = path }.resolveForOs(base, macOS).rootDir + +examples { + ["resolve()"] { + local settings: EvaluatorSettings = new { + modulePath { + "first/module/path" + "second/module/path" + } + moduleCacheDir = "my/cache/dir" + rootDir = "my/root/dir" + externalModuleReaders { + ["foo"] { + executable = "path/to/my/executable" + } + } + externalResourceReaders { + ["foo"] { + executable = "path/to/my/executable" + } + } + } + settings.resolveForOs("file:///path/to/dir/PklProject", macOS) + } + + ["relative path"] { + resolve("file:///path/to/dir/PklProject", "foo/bar") + } + + ["absolute path"] { + resolve("file:///path/to/dir/PklProject", "/foo/bar") + } + + ["enclosing URI has spaces"] { + resolve("file:///path/to/dir%20with%20spaces/PklProject", "foo/bar") + } + + ["relative path with dot segments"] { + resolve("file:///path/to/dir/PklProject", "../my/module/path") + } + + ["relative path with dot segments 2"] { + resolve("file:///path/to/dir/PklProject", "../../my/module/path") + } + + ["relative path with dot segments 3"] { + resolve("file:///path/to/dir/PklProject", ".") + } + + ["relative path with dot segments 4"] { + resolve("file:///path/to/dir/PklProject", "./") + } + + ["relative path with dot segments 5"] { + resolve("file:///path/to/dir/PklProject", "../../../../../../../../") + } + + ["file:/ instead of file:///"] { + resolve("file:/path/to/dir/PklProject", "foo/bar") + } + + ["executable with simple name is not resolved"] { + local settings: EvaluatorSettings = new { + externalModuleReaders { + ["foo"] { + executable = "my-reader" + } + } + } + settings.resolveForOs("file:///path/to/dir/PklProject", macOS).externalModuleReaders!!["foo"] + .executable + } + + ["executable with path segments is resolved against enclosingUri"] { + local settings: EvaluatorSettings = new { + externalModuleReaders { + ["foo"] { + executable = "path/to/reader" + } + } + } + settings.resolveForOs("file:///path/to/dir/PklProject", macOS).externalModuleReaders!!["foo"] + .executable + } + + ["workingDir defaults to enclosingUri"] { + local settings: EvaluatorSettings = new { + externalModuleReaders { + ["foo"] { + workingDir = null + } + } + } + settings.resolveForOs("file:///path/to/dir/PklProject", macOS).externalModuleReaders!!["foo"] + .workingDir + } + + ["workingDir with relative path"] { + local settings: EvaluatorSettings = new { + externalModuleReaders { + ["foo"] { + workingDir = "." + } + } + } + settings.resolveForOs("file:///path/to/dir/PklProject", macOS).externalModuleReaders!!["foo"] + .workingDir + } + + ["workingDir with absolute path"] { + local settings: EvaluatorSettings = new { + externalModuleReaders { + ["foo"] { + workingDir = "/foo/bar" + } + } + } + settings.resolveForOs("file:///path/to/dir/PklProject", macOS).externalModuleReaders!!["foo"] + .workingDir + } +} diff --git a/pkl-core/src/test/files/LanguageSnippetTests/input/api/evaluatorSettingsModuleWindows.pkl b/pkl-core/src/test/files/LanguageSnippetTests/input/api/evaluatorSettingsModuleWindows.pkl new file mode 100644 index 00000000..137e34af --- /dev/null +++ b/pkl-core/src/test/files/LanguageSnippetTests/input/api/evaluatorSettingsModuleWindows.pkl @@ -0,0 +1,243 @@ +amends "../snippetTest.pkl" + +import "pkl:EvaluatorSettings" +import "pkl:platform" + +local windows = new platform.OperatingSystem { name = "Windows" } + +local function resolve(base: String, path: String) = + new EvaluatorSettings { rootDir = path } + .resolveForOs(base, windows) + .rootDir + +examples { + ["resolve()"] { + local settings: EvaluatorSettings = new { + modulePath { + "first/module/path" + "second/module/path" + } + moduleCacheDir = "my/cache/dir" + rootDir = "my/root/dir" + externalModuleReaders { + ["foo"] { + executable = "path/to/my/executable" + } + } + externalResourceReaders { + ["foo"] { + executable = "path/to/my/executable" + } + } + } + settings.resolveForOs("file:///C:/path/to/dir/PklProject", windows) + } + + ["relative path"] { + resolve("file:///C:/path/to/dir/PklProject", "foo\\bar") + } + + ["relative path 2"] { + resolve("file:///C:/path/to/dir/PklProject", "foo/bar") + } + + ["absolute path"] { + resolve("file:///C:/path/to/dir/PklProject", #"\foo\bar"#) + } + + ["absolute path with drive letter"] { + resolve("file:///C:/path/to/dir/PklProject", #"C:\foo\bar"#) + } + + ["absolute path with drive letter 2"] { + resolve("file:///C:/path/to/dir/PklProject", #"C:/foo/bar"#) + } + + ["absolute path with drive letter 3"] { + resolve("file:///C:/path/to/dir/PklProject", #"C:/"#) + } + + ["absolute path with drive letter 4"] { + resolve("file:///C:/path/to/dir/PklProject", #"C:"#) + } + + ["absolute path with drive letter 5"] { + resolve("file:///C:", #"\path\to\foo"#) + } + + ["base path with drive letter"] { + resolve("file:///C:", #"path\to\foo"#) + } + + ["enclosing URI has spaces"] { + resolve("file:///C:/path/to/dir%20with%20spaces/PklProject", "foo/bar") + } + + ["relative path with dot segments"] { + resolve("file:///C:/path/to/dir/PklProject", "../my/module/path") + } + + ["relative path with dot segments 2"] { + resolve("file:///C:/path/to/dir/PklProject", "../../my/module/path") + } + + ["relative path with dot segments 3"] { + resolve("file:///C:/path/to/dir/PklProject", ".") + } + + ["relative path with dot segments 4"] { + resolve("file:///C:/path/to/dir/PklProject", "./") + } + + ["relative path with dot segments 5"] { + resolve("file:///C:/path/to/dir/PklProject", #"..\..\..\..\..\..\..\..\"#) + } + + ["file URI with no drive"] { + resolve("file:///path/to/dir/PklProject", "foo\\bar") + } + + ["executable with simple name is not resolved"] { + local settings: EvaluatorSettings = new { + externalModuleReaders { + ["foo"] { + executable = "my-reader" + } + } + } + settings + .resolveForOs("file:///C:/path/to/dir/PklProject", windows) + .externalModuleReaders!!["foo"].executable + } + + ["executable with path segments is resolved against enclosingUri"] { + local settings: EvaluatorSettings = new { + externalModuleReaders { + ["foo"] { + executable = "path/to/reader" + } + } + } + settings + .resolveForOs("file:///C:/path/to/dir/PklProject", windows) + .externalModuleReaders!!["foo"].executable + } + + ["workingDir defaults to enclosingUri"] { + local settings: EvaluatorSettings = new { + externalModuleReaders { + ["foo"] { + workingDir = null + } + } + } + settings + .resolveForOs("file:///C:/path/to/dir/PklProject", windows) + .externalModuleReaders!!["foo"].workingDir + } + + ["workingDir with relative path"] { + local settings: EvaluatorSettings = new { + externalModuleReaders { + ["foo"] { + workingDir = "." + } + } + } + settings + .resolveForOs("file:///C:/path/to/dir/PklProject", windows) + .externalModuleReaders!!["foo"].workingDir + } + + ["workingDir with absolute path"] { + local settings: EvaluatorSettings = new { + externalModuleReaders { + ["foo"] { + workingDir = "/foo/bar" + } + } + } + settings + .resolveForOs("file:///C:/path/to/dir/PklProject", windows) + .externalModuleReaders!!["foo"].workingDir + } + + ["UNC path"] { + resolve("file:///path/to/foo", #"\\server\share\path\to\foo"#) + } + + ["UNC path 2"] { + resolve(#"file://server/share/path/to/foo"#, #"\new\path"#) + } + + ["empty path"] { + resolve("file:///C:/path/to/dir/PklProject", "") + } + + ["multiple consecutive slashes"] { + resolve("file:///C:/path/to/dir/PklProject", #"my\\\\path"#) + } + + ["different drive letter"] { + resolve("file:///C:/path/to/dir/PklProject", #"D:\my\path"#) + } + + ["UNC path as base with relative path"] { + resolve(#"file://server/share/dir/PklProject"#, #"my\path"#) + } + + ["UNC path as base with .."] { + resolve(#"file://server/share/dir/sub/PklProject"#, #"..\my\path"#) + } + + ["UNC path as base with ."] { + resolve(#"file://server/share/dir/PklProject"#, ".") + } + + ["UNC path as base, cannot remove root"] { + resolve(#"file://server/share/PklProject"#, #"..\..\..\"#) + } + + ["UNC path as base, cannot remove root 2"] { + resolve(#"file://server/share/dir/PklProject"#, #"../../.."#) + } + + ["UNC path as base with drive letter path"] { + resolve(#"file://server/share/dir/PklProject"#, #"C:\local\path"#) + } + + ["UNC path as base with no drive"] { + resolve(#"file://server/"#, #"/my/path"#) + } + + ["trailing separator preserved"] { + resolve("file:///C:/path/to/dir/PklProject", #"my\path\"#) + } + + ["trailing separator preserved with normalization"] { + resolve("file:///C:/path/to/dir/PklProject", #"..\my\path\"#) + } + + ["no trailing separator"] { + resolve("file:///C:/path/to/dir/PklProject", #"my\path"#) + } + + ["complex normalization"] { + resolve("file:///C:/path/to/dir/PklProject", #".\foo\..\bar\.\baz"#) + } + + ["UNC with forward slashes"] { + resolve(#"file://server/share/dir/PklProject"#, "my/path") + } + + ["absolute path with different drive and forward slashes"] { + resolve("file:///C:/path/to/dir/PklProject", "D:/my/path") + } +} + +output { + renderer = new PcfRenderer { + useCustomStringDelimiters = true + omitNullProperties = true + } +} diff --git a/pkl-core/src/test/files/LanguageSnippetTests/input/api/project1.pkl b/pkl-core/src/test/files/LanguageSnippetTests/input/api/project1.pkl new file mode 100644 index 00000000..a4db49bb --- /dev/null +++ b/pkl-core/src/test/files/LanguageSnippetTests/input/api/project1.pkl @@ -0,0 +1,32 @@ +amends "../snippetTest.pkl" + +import "pkl:platform" +import "pkl:Project" + +examples { + local macos = new platform.OperatingSystem { name = "macOS" } + + ["resolvedEvaluatorSettings"] { + new Project { + projectFileUri = "file:///path/to/PklProject" + evaluatorSettings { + modulePath { + "modulepath/first" + "modulepath/second" + } + moduleCacheDir = "path/to/cache" + externalModuleReaders { + ["relative path"] { + executable = "foo/executable" + } + ["absolute path"] { + executable = "/path/to/executable" + } + ["command name"] { + executable = "foo" + } + } + } + }.evaluatorSettings.resolveForOs("file:///path/to/PklProject", macos) + } +} diff --git a/pkl-core/src/test/files/LanguageSnippetTests/input/projects/badPklProject4/PklProject b/pkl-core/src/test/files/LanguageSnippetTests/input/projects/badPklProject4/PklProject new file mode 100644 index 00000000..ef0dc083 --- /dev/null +++ b/pkl-core/src/test/files/LanguageSnippetTests/input/projects/badPklProject4/PklProject @@ -0,0 +1,7 @@ +amends "pkl:Project" + +projectFileUri = "file:///not a valid uri" + +evaluatorSettings { + moduleCacheDir = "foo" +} diff --git a/pkl-core/src/test/files/LanguageSnippetTests/input/projects/badPklProject4/bug.pkl b/pkl-core/src/test/files/LanguageSnippetTests/input/projects/badPklProject4/bug.pkl new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/pkl-core/src/test/files/LanguageSnippetTests/input/projects/badPklProject4/bug.pkl @@ -0,0 +1 @@ + diff --git a/pkl-core/src/test/files/LanguageSnippetTests/input/projects/badPklProject5/PklProject b/pkl-core/src/test/files/LanguageSnippetTests/input/projects/badPklProject5/PklProject new file mode 100644 index 00000000..0b16faa3 --- /dev/null +++ b/pkl-core/src/test/files/LanguageSnippetTests/input/projects/badPklProject5/PklProject @@ -0,0 +1,11 @@ +amends "pkl:Project" + +projectFileUri = "modulepath:/foo/bar/PklProject" + +evaluatorSettings { + externalModuleReaders { + ["foo"] { + executable = "foo/bar" + } + } +} diff --git a/pkl-core/src/test/files/LanguageSnippetTests/input/projects/badPklProject5/bug.pkl b/pkl-core/src/test/files/LanguageSnippetTests/input/projects/badPklProject5/bug.pkl new file mode 100644 index 00000000..e69de29b diff --git a/pkl-core/src/test/files/LanguageSnippetTests/input/projects/evaluatorSettings2/PklProject b/pkl-core/src/test/files/LanguageSnippetTests/input/projects/evaluatorSettings2/PklProject new file mode 100644 index 00000000..9cc526b3 --- /dev/null +++ b/pkl-core/src/test/files/LanguageSnippetTests/input/projects/evaluatorSettings2/PklProject @@ -0,0 +1,5 @@ +amends "pkl:Project" + +evaluatorSettings { + rootDir = "." +} diff --git a/pkl-core/src/test/files/LanguageSnippetTests/input/projects/evaluatorSettings2/badRead.pkl b/pkl-core/src/test/files/LanguageSnippetTests/input/projects/evaluatorSettings2/badRead.pkl new file mode 100644 index 00000000..131d2c9a --- /dev/null +++ b/pkl-core/src/test/files/LanguageSnippetTests/input/projects/evaluatorSettings2/badRead.pkl @@ -0,0 +1 @@ +res = read("file:///file.txt") diff --git a/pkl-core/src/test/files/LanguageSnippetTests/input/projects/evaluatorSettings2/goodRead.pkl b/pkl-core/src/test/files/LanguageSnippetTests/input/projects/evaluatorSettings2/goodRead.pkl new file mode 100644 index 00000000..37c61567 --- /dev/null +++ b/pkl-core/src/test/files/LanguageSnippetTests/input/projects/evaluatorSettings2/goodRead.pkl @@ -0,0 +1 @@ +res = read("badRead.pkl").text diff --git a/pkl-core/src/test/files/LanguageSnippetTests/output/api/evaluatorSettingsModulePosix.pcf b/pkl-core/src/test/files/LanguageSnippetTests/output/api/evaluatorSettingsModulePosix.pcf new file mode 100644 index 00000000..df39a01d --- /dev/null +++ b/pkl-core/src/test/files/LanguageSnippetTests/output/api/evaluatorSettingsModulePosix.pcf @@ -0,0 +1,66 @@ +examples { + ["resolve()"] { + new { + modulePath { + "/path/to/dir/first/module/path" + "/path/to/dir/second/module/path" + } + moduleCacheDir = "/path/to/dir/my/cache/dir" + rootDir = "/path/to/dir/my/root/dir" + externalModuleReaders { + ["foo"] { + executable = "/path/to/dir/path/to/my/executable" + workingDir = "/path/to/dir" + } + } + externalResourceReaders { + ["foo"] { + executable = "/path/to/dir/path/to/my/executable" + workingDir = "/path/to/dir" + } + } + } + } + ["relative path"] { + "/path/to/dir/foo/bar" + } + ["absolute path"] { + "/foo/bar" + } + ["enclosing URI has spaces"] { + "/path/to/dir with spaces/foo/bar" + } + ["relative path with dot segments"] { + "/path/to/my/module/path" + } + ["relative path with dot segments 2"] { + "/path/my/module/path" + } + ["relative path with dot segments 3"] { + "/path/to/dir" + } + ["relative path with dot segments 4"] { + "/path/to/dir" + } + ["relative path with dot segments 5"] { + "/" + } + ["file:/ instead of file:///"] { + "/path/to/dir/foo/bar" + } + ["executable with simple name is not resolved"] { + "my-reader" + } + ["executable with path segments is resolved against enclosingUri"] { + "/path/to/dir/path/to/reader" + } + ["workingDir defaults to enclosingUri"] { + "/path/to/dir" + } + ["workingDir with relative path"] { + "/path/to/dir" + } + ["workingDir with absolute path"] { + "/foo/bar" + } +} diff --git a/pkl-core/src/test/files/LanguageSnippetTests/output/api/evaluatorSettingsModuleWindows.pcf b/pkl-core/src/test/files/LanguageSnippetTests/output/api/evaluatorSettingsModuleWindows.pcf new file mode 100644 index 00000000..d88f294c --- /dev/null +++ b/pkl-core/src/test/files/LanguageSnippetTests/output/api/evaluatorSettingsModuleWindows.pcf @@ -0,0 +1,141 @@ +examples { + ["resolve()"] { + new { + modulePath { + #"C:\path\to\dir\first\module\path"# + #"C:\path\to\dir\second\module\path"# + } + moduleCacheDir = #"C:\path\to\dir\my\cache\dir"# + rootDir = #"C:\path\to\dir\my\root\dir"# + externalModuleReaders { + ["foo"] { + executable = #"C:\path\to\dir\path\to\my\executable"# + workingDir = #"C:\path\to\dir"# + } + } + externalResourceReaders { + ["foo"] { + executable = #"C:\path\to\dir\path\to\my\executable"# + workingDir = #"C:\path\to\dir"# + } + } + } + } + ["relative path"] { + #"C:\path\to\dir\foo\bar"# + } + ["relative path 2"] { + #"C:\path\to\dir\foo\bar"# + } + ["absolute path"] { + #"C:\foo\bar"# + } + ["absolute path with drive letter"] { + #"C:\foo\bar"# + } + ["absolute path with drive letter 2"] { + #"C:\foo\bar"# + } + ["absolute path with drive letter 3"] { + #"C:\"# + } + ["absolute path with drive letter 4"] { + "C:" + } + ["absolute path with drive letter 5"] { + #"\\path\to\foo"# + } + ["base path with drive letter"] { + #"\path\to\foo"# + } + ["enclosing URI has spaces"] { + #"C:\path\to\dir with spaces\foo\bar"# + } + ["relative path with dot segments"] { + #"C:\path\to\my\module\path"# + } + ["relative path with dot segments 2"] { + #"C:\path\my\module\path"# + } + ["relative path with dot segments 3"] { + #"C:\path\to\dir"# + } + ["relative path with dot segments 4"] { + #"C:\path\to\dir"# + } + ["relative path with dot segments 5"] { + #"C:\"# + } + ["file URI with no drive"] { + #"\path\to\dir\foo\bar"# + } + ["executable with simple name is not resolved"] { + "my-reader" + } + ["executable with path segments is resolved against enclosingUri"] { + #"C:\path\to\dir\path\to\reader"# + } + ["workingDir defaults to enclosingUri"] { + #"C:\path\to\dir"# + } + ["workingDir with relative path"] { + #"C:\path\to\dir"# + } + ["workingDir with absolute path"] { + #"C:\foo\bar"# + } + ["UNC path"] { + #"\\server\share\path\to\foo"# + } + ["UNC path 2"] { + #"\\server\share\new\path"# + } + ["empty path"] { + #"C:\path\to\dir"# + } + ["multiple consecutive slashes"] { + #"C:\path\to\dir\my\path"# + } + ["different drive letter"] { + #"D:\my\path"# + } + ["UNC path as base with relative path"] { + #"\\server\share\dir\my\path"# + } + ["UNC path as base with .."] { + #"\\server\share\dir\my\path"# + } + ["UNC path as base with ."] { + #"\\server\share\dir"# + } + ["UNC path as base, cannot remove root"] { + #"\\server\share\"# + } + ["UNC path as base, cannot remove root 2"] { + #"\\server\share\"# + } + ["UNC path as base with drive letter path"] { + #"C:\local\path"# + } + ["UNC path as base with no drive"] { + #"\\server\\my\path"# + } + ["trailing separator preserved"] { + #"C:\path\to\dir\my\path"# + } + ["trailing separator preserved with normalization"] { + #"C:\path\to\my\path"# + } + ["no trailing separator"] { + #"C:\path\to\dir\my\path"# + } + ["complex normalization"] { + #"C:\path\to\dir\bar\baz"# + } + ["UNC with forward slashes"] { + #"\\server\share\dir\my\path"# + } + ["absolute path with different drive and forward slashes"] { + #"D:\my\path"# + } +} diff --git a/pkl-core/src/test/files/LanguageSnippetTests/output/api/project1.pcf b/pkl-core/src/test/files/LanguageSnippetTests/output/api/project1.pcf new file mode 100644 index 00000000..c207fbc6 --- /dev/null +++ b/pkl-core/src/test/files/LanguageSnippetTests/output/api/project1.pcf @@ -0,0 +1,25 @@ +examples { + ["resolvedEvaluatorSettings"] { + new { + modulePath { + "/path/to/modulepath/first" + "/path/to/modulepath/second" + } + moduleCacheDir = "/path/to/path/to/cache" + externalModuleReaders { + ["relative path"] { + executable = "/path/to/foo/executable" + workingDir = "/path/to" + } + ["absolute path"] { + executable = "/path/to/executable" + workingDir = "/path/to" + } + ["command name"] { + executable = "foo" + workingDir = "/path/to" + } + } + } + } +} diff --git a/pkl-core/src/test/files/LanguageSnippetTests/output/projects/badPklProject4/bug.err b/pkl-core/src/test/files/LanguageSnippetTests/output/projects/badPklProject4/bug.err new file mode 100644 index 00000000..4cfca8b8 --- /dev/null +++ b/pkl-core/src/test/files/LanguageSnippetTests/output/projects/badPklProject4/bug.err @@ -0,0 +1,6 @@ +–– Pkl Error –– +URI `file:///not a valid uri` has invalid syntax. + +xxx | moduleCacheDir = resolvePath(enclosingUri, module.moduleCacheDir!!, forWindows) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +at pkl.EvaluatorSettings#resolveForOs.moduleCacheDir (pkl:EvaluatorSettings) diff --git a/pkl-core/src/test/files/LanguageSnippetTests/output/projects/badPklProject5/bug.err b/pkl-core/src/test/files/LanguageSnippetTests/output/projects/badPklProject5/bug.err new file mode 100644 index 00000000..35d10ef2 --- /dev/null +++ b/pkl-core/src/test/files/LanguageSnippetTests/output/projects/badPklProject5/bug.err @@ -0,0 +1,16 @@ +–– Pkl Error –– +Type constraint `(externalModuleReaders != null).implies(isFileBasedProject)` violated. +Value: new ModuleClass { externalProperties = ?; env = ?; allowedModules = ?; allowe... + + (externalModuleReaders != null).implies(isFileBasedProject) + │ │ │ │ + │ true false false + new Mapping { ["foo"] { executable = ?; arguments = ?; workingDir = ? } } + +xxx | (externalModuleReaders != null).implies(isFileBasedProject), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +at pkl.Project#evaluatorSettings (pkl:Project) + +x | evaluatorSettings { + ^^^^^^^^^^^^^^^^^^^ +at PklProject#evaluatorSettings (file:///$snippetsDir/input/projects/badPklProject5/PklProject) diff --git a/pkl-core/src/test/files/LanguageSnippetTests/output/projects/evaluatorSettings2/badRead.err b/pkl-core/src/test/files/LanguageSnippetTests/output/projects/evaluatorSettings2/badRead.err new file mode 100644 index 00000000..0a9c223b --- /dev/null +++ b/pkl-core/src/test/files/LanguageSnippetTests/output/projects/evaluatorSettings2/badRead.err @@ -0,0 +1,14 @@ +–– Pkl Error –– +Refusing to read resource `file:///file.txt` because it is not within the root directory (`--root-dir`). + +x | res = read("file:///file.txt") + ^^^^^^^^^^^^^^^^^^^^^^^^ +at badRead#res (file:///$snippetsDir/input/projects/evaluatorSettings2/badRead.pkl) + +xxx | renderer.renderDocument(value) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +at pkl.base#Module.output.text (pkl:base) + +xxx | if (renderer is BytesRenderer) renderer.renderDocument(value) else text.encodeToBytes("UTF-8") + ^^^^ +at pkl.base#Module.output.bytes (pkl:base) diff --git a/pkl-core/src/test/files/LanguageSnippetTests/output/projects/evaluatorSettings2/goodRead.pcf b/pkl-core/src/test/files/LanguageSnippetTests/output/projects/evaluatorSettings2/goodRead.pcf new file mode 100644 index 00000000..383df4c2 --- /dev/null +++ b/pkl-core/src/test/files/LanguageSnippetTests/output/projects/evaluatorSettings2/goodRead.pcf @@ -0,0 +1,4 @@ +res = """ + res = read("file:///file.txt") + + """ diff --git a/pkl-core/src/test/kotlin/org/pkl/core/module/ModuleKeyFactoriesTest.kt b/pkl-core/src/test/kotlin/org/pkl/core/module/ModuleKeyFactoriesTest.kt index 58ada865..0fb5167b 100644 --- a/pkl-core/src/test/kotlin/org/pkl/core/module/ModuleKeyFactoriesTest.kt +++ b/pkl-core/src/test/kotlin/org/pkl/core/module/ModuleKeyFactoriesTest.kt @@ -1,5 +1,5 @@ /* - * Copyright © 2024-2025 Apple Inc. and the Pkl project authors. All rights reserved. + * Copyright © 2024-2026 Apple Inc. and the Pkl project authors. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,20 +15,55 @@ */ package org.pkl.core.module +import java.io.File import java.net.URI +import java.nio.file.Files import java.nio.file.Path +import java.util.regex.Pattern import kotlin.io.path.createDirectories import kotlin.io.path.createParentDirectories import kotlin.io.path.outputStream import org.assertj.core.api.Assertions.assertThat +import org.junit.jupiter.api.Assumptions.assumeTrue import org.junit.jupiter.api.Test import org.junit.jupiter.api.io.TempDir +import org.pkl.commons.test.FileTestUtils import org.pkl.commons.toPath import org.pkl.commons.writeString +import org.pkl.core.EvaluatorBuilder +import org.pkl.core.ModuleSource import org.pkl.core.SecurityManagers -import org.pkl.core.externalreader.* +import org.pkl.core.evaluatorSettings.PklEvaluatorSettings +import org.pkl.core.externalreader.ExternalReaderProcess +import org.pkl.core.externalreader.TestExternalModuleReader +import org.pkl.core.externalreader.TestExternalReaderProcess +import org.pkl.core.resource.ResourceReaders +import org.pkl.core.util.IoUtils class ModuleKeyFactoriesTest { + companion object { + private val externalReaderFixture by lazy { + val readerPath = + "pkl-core/build/fixtures/externalreader".let { if (IoUtils.isWindows()) "$it.bat" else it } + + FileTestUtils.rootProjectDir.resolve(readerPath).also { path -> + if (!Files.exists(path)) { + throw AssertionError( + "Fixture `externalreader` not found. To fix this problem, first run" + + " `./gradlew pkl-core:externalReaderFixture`." + ) + } + } + } + + @JvmStatic + private fun pathEnvIsSet(): Boolean { + return System.getenv("PATH") + ?.split(File.pathSeparator) + ?.contains(externalReaderFixture.toAbsolutePath().toString()) ?: false + } + } + @Test fun `standard library`() { val factory = ModuleKeyFactories.standardLibrary @@ -146,4 +181,34 @@ class ModuleKeyFactoriesTest { proc.close() runtime.close() } + + @Test + fun `external process -- spawning an executable using a path`() { + testExternalReader(externalReaderFixture.toAbsolutePath().toString()) + } + + @Test + fun `external process -- spawning an executable using a simple name off PATH`() { + assumeTrue(pathEnvIsSet(), "PATH contains fixtures dir") + testExternalReader("externalreader") + } + + private fun testExternalReader(executable: String) { + val evaluator = makeEvaluatorWithExternalReader(executable) + val result = evaluator.use { + evaluator.evaluateExpression(ModuleSource.uri("pkl:base"), "read(\"foo:foo\").text") + } + assertThat(result).isEqualTo("hello") + } + + private fun makeEvaluatorWithExternalReader(reader: String) = + with(EvaluatorBuilder.preconfigured()) { + val process = + ExternalReaderProcess.of(PklEvaluatorSettings.ExternalReader(reader, listOf(), null)) + addModuleKeyFactory(ModuleKeyFactories.externalProcess("foo", process)) + addResourceReader(ResourceReaders.externalProcess("foo", process)) + setAllowedModules(allowedModules + listOf(Pattern.compile("foo:"))) + setAllowedResources(allowedResources + listOf(Pattern.compile("foo:"))) + build() + } } diff --git a/pkl-core/src/test/kotlin/org/pkl/core/project/ProjectTest.kt b/pkl-core/src/test/kotlin/org/pkl/core/project/ProjectTest.kt index fac9b02b..8ba576b1 100644 --- a/pkl-core/src/test/kotlin/org/pkl/core/project/ProjectTest.kt +++ b/pkl-core/src/test/kotlin/org/pkl/core/project/ProjectTest.kt @@ -18,6 +18,7 @@ package org.pkl.core.project import java.net.URI import java.nio.file.Path import java.util.regex.Pattern +import kotlin.io.path.createDirectories import org.assertj.core.api.Assertions.assertThat import org.assertj.core.api.Assertions.assertThatCode import org.junit.jupiter.api.Test @@ -153,12 +154,57 @@ class ProjectTest { ) val project = Project.loadFromPath(projectPath) assertThat(project.`package`).isEqualTo(expectedPackage) - assertThat(project.evaluatorSettings).isEqualTo(expectedSettings) + assertThat(project.resolvedEvaluatorSettings).isEqualTo(expectedSettings) assertThat(project.annotations).isEqualTo(expectedAnnotations) assertThat(project.tests) .isEqualTo(listOf(path.resolve("test1.pkl"), path.resolve("test2.pkl"))) } + @Test + fun `loadFromPath() resolvedEvaluatorSettings`(@TempDir path: Path) { + val projectPath = + path.resolve("PklProject").also { + it.writeString( + """ + amends "pkl:Project" + + projectFileUri = "file:///path/to/PklProject" + + evaluatorSettings { + rootDir = "." + moduleCacheDir = "cache/" + modulePath { + "modulepath1/" + "modulepath2/" + } + } + """ + .trimIndent() + ) + } + val project = Project.loadFromPath(projectPath) + assertThat(project.resolvedEvaluatorSettings) + .usingRecursiveComparison() + .isEqualTo( + PklEvaluatorSettings( + null, + null, + null, + null, + null, + null, + Path.of("/path/to/cache/"), + listOf(Path.of("/path/to/modulepath1/"), Path.of("/path/to/modulepath2/")), + null, + Path.of("/path/to"), + null, + null, + null, + null, + ) + ) + } + @Test fun `load wrong type`(@TempDir path: Path) { val projectPath = path.resolve("PklProject") @@ -261,4 +307,59 @@ class ProjectTest { .trimIndent() ) } + + @Test + fun `external readers -- executable path is relative to project dir`(@TempDir tempDir: Path) { + val projectDir = tempDir.resolve("project").also { it.createDirectories() } + val pklProject = + projectDir.resolve("PklProject").also { + it.writeString( + // language=pkl + """ + amends "pkl:Project" + + evaluatorSettings { + externalModuleReaders { + ["foo"] { + executable = "foo/bar/baz" + } + } + } + """ + .trimIndent() + ) + } + val project = Project.loadFromPath(pklProject, SecurityManagers.defaultManager, null) + assertThat(project.resolvedEvaluatorSettings.externalModuleReaders).hasSize(1) + assertThat(project.resolvedEvaluatorSettings.externalModuleReaders?.get("foo")!!.executable()) + .isEqualTo(projectDir.resolve("foo/bar/baz").toString()) + } + + @Test + fun `external readers -- executable is unmodified simple name`(@TempDir tempDir: Path) { + val projectDir = tempDir.resolve("project").also { it.createDirectories() } + val pklProject = + projectDir.resolve("PklProject").also { + it.writeString( + // language=pkl + """ + amends "pkl:Project" + + evaluatorSettings { + externalModuleReaders { + ["foo"] { + executable = "my-command" + } + } + } + """ + .trimIndent() + ) + } + + val project = Project.loadFromPath(pklProject, SecurityManagers.defaultManager, null) + assertThat(project.evaluatorSettings.externalModuleReaders).hasSize(1) + assertThat(project.evaluatorSettings.externalModuleReaders?.get("foo")!!.executable()) + .isEqualTo("my-command") + } } diff --git a/pkl-core/src/test/kotlin/org/pkl/core/util/PathResolverTest.kt b/pkl-core/src/test/kotlin/org/pkl/core/util/PathResolverTest.kt new file mode 100644 index 00000000..c1102a68 --- /dev/null +++ b/pkl-core/src/test/kotlin/org/pkl/core/util/PathResolverTest.kt @@ -0,0 +1,240 @@ +/* + * Copyright © 2026 Apple Inc. and the Pkl project authors. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.pkl.core.util + +import java.net.URI +import org.assertj.core.api.Assertions.assertThat +import org.junit.jupiter.api.Nested +import org.junit.jupiter.api.Test + +class PathResolverTest { + private val posix = PathResolvers.forPosix() + private val windows = PathResolvers.forWindows() + + @Nested + inner class PosixTests { + @Test + fun `simple relative path appended to file base`() { + assertThat(posix.resolvePath(URI("file:///home/user/base.pkl"), "sibling.pkl")) + .isEqualTo("/home/user/base.pkl/sibling.pkl") + } + + @Test + fun `relative path appended to directory base (trailing slash)`() { + assertThat(posix.resolvePath(URI("file:///home/user/dir/"), "file.pkl")) + .isEqualTo("/home/user/dir/file.pkl") + } + + @Test + fun `nested relative path`() { + assertThat(posix.resolvePath(URI("file:///home/user/base.pkl"), "sub/dir/file.pkl")) + .isEqualTo("/home/user/base.pkl/sub/dir/file.pkl") + } + + @Test + fun `absolute path overrides base`() { + assertThat(posix.resolvePath(URI("file:///home/user/base.pkl"), "/absolute/path.pkl")) + .isEqualTo("/absolute/path.pkl") + } + + @Test + fun `absolute path containing dot is normalized`() { + assertThat(posix.resolvePath(URI("file:///home/user/base.pkl"), "/foo/./bar.pkl")) + .isEqualTo("/foo/bar.pkl") + } + + @Test + fun `absolute path containing double-dot is normalized`() { + assertThat(posix.resolvePath(URI("file:///home/user/base.pkl"), "/foo/../bar.pkl")) + .isEqualTo("/bar.pkl") + } + + @Test + fun `single dot in relative path is elided`() { + assertThat(posix.resolvePath(URI("file:///home/user/base.pkl"), "./sibling.pkl")) + .isEqualTo("/home/user/base.pkl/sibling.pkl") + } + + @Test + fun `double-dot in relative path goes up one segment`() { + assertThat(posix.resolvePath(URI("file:///home/user/base.pkl"), "../sibling.pkl")) + .isEqualTo("/home/user/sibling.pkl") + } + + @Test + fun `two double-dots in relative path go up two segments`() { + assertThat(posix.resolvePath(URI("file:///home/user/a/b.pkl"), "../../c.pkl")) + .isEqualTo("/home/user/c.pkl") + } + + @Test + fun `mixed relative path with dot-dot`() { + assertThat(posix.resolvePath(URI("file:///home/user/base.pkl"), "sub/dir/../../other.pkl")) + .isEqualTo("/home/user/base.pkl/other.pkl") + } + + @Test + fun `double-dot beyond root clamps to root`() { + assertThat(posix.resolvePath(URI("file:///file.pkl"), "../../root.pkl")) + .isEqualTo("/root.pkl") + } + + @Test + fun `root base with relative path`() { + assertThat(posix.resolvePath(URI("file:///"), "file.pkl")).isEqualTo("/file.pkl") + } + + @Test + fun `URI with percent-encoded path is decoded`() { + // URI.getPath() decodes percent-encoding + assertThat(posix.resolvePath(URI("file:///home/user%20name/base.pkl"), "file.pkl")) + .isEqualTo("/home/user name/base.pkl/file.pkl") + } + } + + @Nested + inner class WindowsTests { + @Test + fun `drive letter URI with simple relative path`() { + assertThat(windows.resolvePath(URI("file:///C:/Users/user/base.pkl"), "relative.pkl")) + .isEqualTo("""C:\Users\user\base.pkl\relative.pkl""") + } + + @Test + fun `drive letter URI with nested relative path`() { + assertThat(windows.resolvePath(URI("file:///C:/Users/user/base.pkl"), "sub\\dir\\file.pkl")) + .isEqualTo("""C:\Users\user\base.pkl\sub\dir\file.pkl""") + } + + @Test + fun `drive letter URI with forward-slash relative path is normalised to backslash`() { + assertThat(windows.resolvePath(URI("file:///C:/Users/user/base.pkl"), "sub/dir/file.pkl")) + .isEqualTo("""C:\Users\user\base.pkl\sub\dir\file.pkl""") + } + + @Test + fun `drive letter URI with directory base (trailing backslash)`() { + assertThat(windows.resolvePath(URI("file:///C:/Users/dir/"), "file.pkl")) + .isEqualTo("""C:\Users\dir\file.pkl""") + } + + @Test + fun `backslash dot in relative path is elided`() { + assertThat(windows.resolvePath(URI("file:///C:/Users/user/base.pkl"), """..\sibling.pkl""")) + .isEqualTo("""C:\Users\user\sibling.pkl""") + } + + @Test + fun `forward-slash dot-dot in relative path is normalised`() { + assertThat(windows.resolvePath(URI("file:///C:/Users/user/base.pkl"), "../sibling.pkl")) + .isEqualTo("""C:\Users\user\sibling.pkl""") + } + + @Test + fun `backslash single-dot in relative path is elided`() { + assertThat(windows.resolvePath(URI("file:///C:/Users/user/base.pkl"), """.\\sibling.pkl""")) + .isEqualTo("""C:\Users\user\base.pkl\sibling.pkl""") + } + + @Test + fun `two double-dots go up two segments`() { + // join: C:\Users\user\a\b.pkl\..\..\c.pkl -> C:\Users\user\c.pkl + assertThat(windows.resolvePath(URI("file:///C:/Users/user/a/b.pkl"), "..\\..\\c.pkl")) + .isEqualTo("""C:\Users\user\c.pkl""") + } + + @Test + fun `double-dot beyond drive root clamps to root`() { + assertThat(windows.resolvePath(URI("file:///C:/base.pkl"), "..\\..\\out.pkl")) + .isEqualTo("""C:\out.pkl""") + } + + @Test + fun `absolute path on same drive overrides base`() { + assertThat(windows.resolvePath(URI("file:///C:/Users/base.pkl"), """C:\other\path.pkl""")) + .isEqualTo("""C:\other\path.pkl""") + } + + @Test + fun `absolute path on different drive overrides base`() { + assertThat(windows.resolvePath(URI("file:///C:/Users/base.pkl"), """D:\other.pkl""")) + .isEqualTo("""D:\other.pkl""") + } + + @Test + fun `absolute path with forward slashes is accepted`() { + assertThat(windows.resolvePath(URI("file:///C:/Users/base.pkl"), "D:/other.pkl")) + .isEqualTo("""D:\other.pkl""") + } + + @Test + fun `root-relative backslash path takes drive root from base`() { + // \root.pkl is root-relative; drive letter is inherited from base + assertThat(windows.resolvePath(URI("file:///C:/Users/base.pkl"), """\root.pkl""")) + .isEqualTo("""C:\root.pkl""") + } + + @Test + fun `root-relative forward-slash path takes drive root from base`() { + assertThat(windows.resolvePath(URI("file:///C:/Users/base.pkl"), "/root.pkl")) + .isEqualTo("""C:\root.pkl""") + } + + @Test + fun `UNC URI with simple relative path`() { + assertThat(windows.resolvePath(URI("file://server/share/base.pkl"), "relative.pkl")) + .isEqualTo("""\\server\share\base.pkl\relative.pkl""") + } + + @Test + fun `UNC URI with double-dot goes up within share`() { + assertThat(windows.resolvePath(URI("file://server/share/dir/base.pkl"), """..\sibling.pkl""")) + .isEqualTo("""\\server\share\dir\sibling.pkl""") + } + + @Test + fun `UNC URI double-dot beyond share root clamps to share root`() { + assertThat(windows.resolvePath(URI("file://server/share/base.pkl"), "..\\..\\.\\out.pkl")) + .isEqualTo("""\\server\share\out.pkl""") + } + + @Test + fun `UNC URI with absolute UNC path overrides base`() { + assertThat( + windows.resolvePath(URI("file://server/share/base.pkl"), """\\other\share\file.pkl""") + ) + .isEqualTo("""\\other\share\file.pkl""") + } + + @Test + fun `absolute path containing dot is normalized`() { + assertThat(windows.resolvePath(URI("file:///C:/Users/base.pkl"), """C:\foo\.\bar.pkl""")) + .isEqualTo("""C:\foo\bar.pkl""") + } + + @Test + fun `absolute path containing double-dot is normalized`() { + assertThat(windows.resolvePath(URI("file:///C:/Users/base.pkl"), """C:\foo\..\bar.pkl""")) + .isEqualTo("""C:\bar.pkl""") + } + + @Test + fun `file URI without drive letter`() { + assertThat(windows.resolvePath(URI("file:///path/to/foo"), "bar")) + .isEqualTo("""\path\to\foo\bar""") + } + } +} diff --git a/pkl-gradle/src/main/java/org/pkl/gradle/PklPlugin.java b/pkl-gradle/src/main/java/org/pkl/gradle/PklPlugin.java index a65cacbb..fb3291d4 100644 --- a/pkl-gradle/src/main/java/org/pkl/gradle/PklPlugin.java +++ b/pkl-gradle/src/main/java/org/pkl/gradle/PklPlugin.java @@ -326,6 +326,14 @@ public class PklPlugin implements Plugin { spec.getTestPort().convention(-1); spec.getHttpNoProxy().convention(List.of()); + + var gradleProjectDir = + project.provider( + () -> project.getLayout().getProjectDirectory().getAsFile().getAbsolutePath()); + spec.getExternalModuleReaders() + .configureEach(reader -> reader.getWorkingDir().convention(gradleProjectDir)); + spec.getExternalResourceReaders() + .configureEach(reader -> reader.getWorkingDir().convention(gradleProjectDir)); } private void configureCodeGenSpec(Project project, CodeGenSpec spec) { diff --git a/pkl-gradle/src/main/java/org/pkl/gradle/spec/ExternalReaderSpec.java b/pkl-gradle/src/main/java/org/pkl/gradle/spec/ExternalReaderSpec.java index 0d856fb8..f3a844f6 100644 --- a/pkl-gradle/src/main/java/org/pkl/gradle/spec/ExternalReaderSpec.java +++ b/pkl-gradle/src/main/java/org/pkl/gradle/spec/ExternalReaderSpec.java @@ -23,6 +23,7 @@ import org.gradle.api.Named; import org.gradle.api.provider.ListProperty; import org.gradle.api.provider.Property; import org.gradle.api.tasks.Input; +import org.gradle.api.tasks.Optional; import org.pkl.core.evaluatorSettings.PklEvaluatorSettings.ExternalReader; public abstract class ExternalReaderSpec implements Named { @@ -45,8 +46,13 @@ public abstract class ExternalReaderSpec implements Named { @Input public abstract ListProperty getArguments(); + @Input + @Optional + public abstract Property getWorkingDir(); + public ExternalReader toExternalReader() { - return new ExternalReader(getExecutable().get(), getArguments().get()); + return new ExternalReader( + getExecutable().get(), getArguments().get(), getWorkingDir().getOrNull()); } public static Map toExternalReaderMap( diff --git a/pkl-server/src/main/kotlin/org/pkl/server/Server.kt b/pkl-server/src/main/kotlin/org/pkl/server/Server.kt index 8ec95b4b..6a6cc698 100644 --- a/pkl-server/src/main/kotlin/org/pkl/server/Server.kt +++ b/pkl-server/src/main/kotlin/org/pkl/server/Server.kt @@ -304,6 +304,8 @@ class Server(private val transport: MessageTransport) : AutoCloseable { externalReaderProcesses .computeIfAbsent(evaluatorId) { ConcurrentHashMap() } .computeIfAbsent(spec) { - ExternalReaderProcess.of(PklEvaluatorSettings.ExternalReader(it.executable, it.arguments)) + ExternalReaderProcess.of( + PklEvaluatorSettings.ExternalReader(it.executable, it.arguments, it.workingDir) + ) } } diff --git a/pkl-server/src/main/kotlin/org/pkl/server/ServerMessagePackDecoder.kt b/pkl-server/src/main/kotlin/org/pkl/server/ServerMessagePackDecoder.kt index 0039cf04..5c4fc31e 100644 --- a/pkl-server/src/main/kotlin/org/pkl/server/ServerMessagePackDecoder.kt +++ b/pkl-server/src/main/kotlin/org/pkl/server/ServerMessagePackDecoder.kt @@ -149,5 +149,9 @@ class ServerMessagePackDecoder(unpacker: MessageUnpacker) : BaseMessagePackDecod } private fun unpackExternalReader(map: Map): ExternalReader = - ExternalReader(unpackString(map, "executable"), unpackStringListOrNull(map, "arguments")) + ExternalReader( + unpackString(map, "executable"), + unpackStringListOrNull(map, "arguments"), + unpackStringOrNull(map, "workingDir"), + ) } diff --git a/pkl-server/src/main/kotlin/org/pkl/server/ServerMessagePackEncoder.kt b/pkl-server/src/main/kotlin/org/pkl/server/ServerMessagePackEncoder.kt index e19f6220..e3dac1fe 100644 --- a/pkl-server/src/main/kotlin/org/pkl/server/ServerMessagePackEncoder.kt +++ b/pkl-server/src/main/kotlin/org/pkl/server/ServerMessagePackEncoder.kt @@ -99,9 +99,10 @@ class ServerMessagePackEncoder(packer: MessagePacker) : BaseMessagePackEncoder(p } private fun packExternalReader(spec: ExternalReader) { - packMapHeader(1, spec.arguments) + packMapHeader(1, spec.arguments, spec.workingDir) packKeyValue("executable", spec.executable) spec.arguments?.let { packKeyValue("arguments", it) } + spec.workingDir?.let { packKeyValue("workingDir", it) } } override fun encodeMessage(msg: Message) { diff --git a/pkl-server/src/main/kotlin/org/pkl/server/ServerMessages.kt b/pkl-server/src/main/kotlin/org/pkl/server/ServerMessages.kt index 02cbf959..9b978da6 100644 --- a/pkl-server/src/main/kotlin/org/pkl/server/ServerMessages.kt +++ b/pkl-server/src/main/kotlin/org/pkl/server/ServerMessages.kt @@ -49,7 +49,11 @@ data class CreateEvaluatorRequest( override fun requestId(): Long = requestId } -data class ExternalReader(val executable: String, val arguments: List?) +data class ExternalReader( + val executable: String, + val arguments: List?, + val workingDir: String?, +) data class Proxy(val address: URI?, val noProxy: List?) diff --git a/pkl-server/src/test/kotlin/org/pkl/server/ServerMessagePackCodecTest.kt b/pkl-server/src/test/kotlin/org/pkl/server/ServerMessagePackCodecTest.kt index c3c18092..0d4443a0 100644 --- a/pkl-server/src/test/kotlin/org/pkl/server/ServerMessagePackCodecTest.kt +++ b/pkl-server/src/test/kotlin/org/pkl/server/ServerMessagePackCodecTest.kt @@ -54,7 +54,7 @@ class ServerMessagePackCodecTest { val resourceReader2 = Messages.ResourceReaderSpec("resourceReader2", true, false) val moduleReader1 = Messages.ModuleReaderSpec("moduleReader1", true, true, true) val moduleReader2 = Messages.ModuleReaderSpec("moduleReader2", true, false, false) - val externalReader = ExternalReader("external-cmd", listOf("arg1", "arg2")) + val externalReader = ExternalReader("external-cmd", listOf("arg1", "arg2"), "/foo/bar") roundtrip( CreateEvaluatorRequest( requestId = 123, diff --git a/stdlib/EvaluatorSettings.pkl b/stdlib/EvaluatorSettings.pkl index 33c608e2..736c1dbf 100644 --- a/stdlib/EvaluatorSettings.pkl +++ b/stdlib/EvaluatorSettings.pkl @@ -19,6 +19,11 @@ @Since { version = "0.26.0" } module pkl.EvaluatorSettings +import "pkl:EvaluatorSettings" +import "pkl:platform" +// used by doc comments +import "pkl:Project" + /// The external properties available to Pkl, read using the `prop:` scheme. externalProperties: Mapping? @@ -118,6 +123,89 @@ externalResourceReaders: Mapping? @Since { version = "0.30.0" } traceMode: ("compact" | "pretty")? +/// These evaluator settings, whose settings are resolved against [enclosingUri] using OS rules for +/// [os]. +@Since { version = "0.32.0" } +function resolveForOs(enclosingUri: Uri, os: platform.OperatingSystem): EvaluatorSettings = (module) { + local forWindows = os.name == "Windows" + + when (module.modulePath != null) { + modulePath = new { + for (path in module.modulePath!!) { + resolvePath(enclosingUri, path, forWindows) + } + } + } + + when (module.moduleCacheDir != null) { + moduleCacheDir = resolvePath(enclosingUri, module.moduleCacheDir!!, forWindows) + } + + when (module.rootDir != null) { + rootDir = resolvePath(enclosingUri, module.rootDir!!, forWindows) + } + + when (module.externalResourceReaders != null) { + externalResourceReaders { + [[true]] { + executable = resolveExecutable(enclosingUri, super.executable, forWindows) + workingDir = resolvePath(enclosingUri, super.workingDir ?? "./", forWindows) + } + } + } + + when (module.externalModuleReaders != null) { + externalModuleReaders { + for (readerName, reader in module.externalModuleReaders!!) { + [readerName] { + executable = resolveExecutable(enclosingUri, reader.executable, forWindows) + workingDir = + if (reader.workingDir == null) + resolvePath(enclosingUri, "./", forWindows) + else + resolvePath(enclosingUri, reader.workingDir!!, forWindows) + } + } + } + } +} + +/// These evaluator settings, whose settings are resolved to URIs against [enclosingUri]. +/// +/// The following settings are resolved: +/// +/// * [modulePath] +/// * [rootDir] +/// * [moduleCacheDir] +/// * [ExternalReader.executable] +/// * [ExternalReader.workingDir] +/// +/// Returns file paths based on the host OS filesystem. +/// +/// If [ExternalReader.workingDir] is `null`, it resolves to the `./` path off of +/// [enclosingUri]. +/// +/// On POSIX-based systems (like macOS and linux), this resolves to paths such as `/path/to/dir`. +/// +/// On Windows, this resolves to drive letter paths such as `C:\path\to\dir`, or UNC paths such as +/// `\\network\share\path\to\dir`. +@Since { version = "0.32.0" } +function resolve(enclosingUri: Uri): EvaluatorSettings = + resolveForOs( + enclosingUri, + platform.current.operatingSystem, + ) + +local function resolveExecutable(base: String, path: String, forWindows: Boolean) = + let (hasSep = if (forWindows) path.contains(Regex(#"[/\\]"#)) else path.contains("/")) + if (hasSep) resolvePath(base, path, forWindows) else path + +external local function resolvePath( + baseUri: String(startsWith("file:/")), + path: String, + forWindows: Boolean, +): String + local const hostnameRegex = Regex(#"https?://([^/?#]*)"#) local const hasNonEmptyHostname = (it: String) -> @@ -268,16 +356,25 @@ class ExternalReader { /// The external reader executable. /// /// Will be spawned with the same environment variables and working directory as the Pkl process. - /// Executable is resolved according to the operating system's process spawning rules. - /// On macOS, Linux, and Windows platforms, this may be: /// - /// * An absolute path - /// * A relative path (to the current working directory) - /// * The name of the executable, to be resolved against the `PATH` environment variable + /// This may be: + /// + /// * An absolute or relative path. + /// * The name of the executable, to be resolved against the `PATH` environment variable. + /// + /// When declared inside a [Project], relative paths are resolved against + /// [Project.projectFileUri]. executable: String /// Additional command line arguments passed to the external reader process. arguments: Listing? + + /// The working directory used to spawn [executable]. + /// + /// When declared inside a [Project], relative paths are resolved against + /// [Project.projectFileUri]. + @Since { version = "0.32.0" } + workingDir: String? } local typealias ReservedHttpHeaderName = diff --git a/stdlib/Project.pkl b/stdlib/Project.pkl index 571a5e50..8f68ab1f 100644 --- a/stdlib/Project.pkl +++ b/stdlib/Project.pkl @@ -191,23 +191,32 @@ local isFileBasedProject = projectFileUri.startsWith("file:") /// Evaluator settings do not get published as part of a package. /// It is not possible for a package dependency to influence the evaluator settings of a project. /// -/// The following values can only be set if this is a file-based project. +/// The following values can only be set if this is a file-based project: /// /// - [modulePath][EvaluatorSettings.modulePath] /// - [rootDir][EvaluatorSettings.rootDir] /// - [moduleCacheDir][EvaluatorSettings.moduleCacheDir] -/// -/// For each of these, relative paths are resolved against the project's enclosing directory. +/// - [externalModuleReaders][EvaluatorSettings.externalModuleReaders] +/// - [externalResourceReaders][EvaluatorSettings.externalResourceReaders] evaluatorSettings: EvaluatorSettingsModule( (modulePath != null).implies(isFileBasedProject), (rootDir != null).implies(isFileBasedProject), (moduleCacheDir != null).implies(isFileBasedProject), + (externalModuleReaders != null).implies(isFileBasedProject), + (externalResourceReaders != null).implies(isFileBasedProject), ) +/// The [evaluatorSettings] resolved against the project dir. +@Since { version = "0.32.0" } +fixed resolvedEvaluatorSettings: EvaluatorSettingsModule = evaluatorSettings.resolve(projectFileUri) + /// The URI of the PklProject file. /// -/// This value is used to resolve relative paths when importing another local project as a -/// dependency. +/// This value is used to: +/// +/// * Resolve relative paths when importing another local project as a +/// dependency. +/// * Resolve relative paths declared within [evaluatorSettings]. projectFileUri: String = reflect.Module(module).uri /// Instantiates a project definition within the enclosing module. @@ -229,7 +238,8 @@ local const hasVersion = (it: Uri) -> if (versionSep == -1) false else - let (version = it.drop(versionSep + 1)) semver.parseOrNull(version) != null + let (version = it.drop(versionSep + 1)) + semver.parseOrNull(version) != null typealias PackageUri = Uri(startsWith("package:"), hasVersion)