diff --git a/.circleci/config.pkl b/.circleci/config.pkl index ae3cfd16..5446405c 100644 --- a/.circleci/config.pkl +++ b/.circleci/config.pkl @@ -14,7 +14,7 @@ // limitations under the License. //===----------------------------------------------------------------------===// // File gets rendered to .circleci/config.yml via git hook. -amends "package://pkg.pkl-lang.org/pkl-project-commons/pkl.impl.circleci@1.1.0#/PklCI.pkl" +amends "package://pkg.pkl-lang.org/pkl-project-commons/pkl.impl.circleci@1.1.1#/PklCI.pkl" import "jobs/BuildNativeJob.pkl" import "jobs/GradleCheckJob.pkl" @@ -96,6 +96,11 @@ local buildNativeJobs: Mapping = new { musl = true isRelease = _dist == "release" } + ["pkl-cli-windows-amd64-\(_dist)"] { + arch = "amd64" + os = "windows" + isRelease = _dist == "release" + } } } @@ -108,6 +113,11 @@ local gradleCheckJobs: Mapping = new { javaVersion = "21.0" isRelease = false } + ["gradle-check-jdk17-windows"] { + javaVersion = "17.0" + isRelease = false + os = "windows" + } } jobs { diff --git a/.circleci/config.yml b/.circleci/config.yml index 0ddc93f9..021ae3cb 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -12,18 +12,12 @@ jobs: - run: command: |- export PATH=~/staticdeps/bin:$PATH - ./gradlew --info --stacktrace -DreleaseBuild=true pkl-cli:macExecutableAmd64 pkl-core:testMacExecutableAmd64 + ./gradlew --info --stacktrace -DtestReportsDir=${HOME}/test-results -DreleaseBuild=true pkl-cli:macExecutableAmd64 pkl-core:testMacExecutableAmd64 name: gradle buildNative - persist_to_workspace: root: '.' paths: - pkl-cli/build/executable/ - - run: - command: |- - mkdir ~/test-results/ - find . -type f -regex ".*/build/test-results/.*xml" -exec cp {} ~/test-results/ \; - name: Gather test results - when: always - store_test_results: path: ~/test-results environment: @@ -94,18 +88,12 @@ jobs: - run: command: |- export PATH=~/staticdeps/bin:$PATH - ./gradlew --info --stacktrace -DreleaseBuild=true pkl-cli:linuxExecutableAmd64 pkl-core:testLinuxExecutableAmd64 + ./gradlew --info --stacktrace -DtestReportsDir=${HOME}/test-results -DreleaseBuild=true pkl-cli:linuxExecutableAmd64 pkl-core:testLinuxExecutableAmd64 name: gradle buildNative - persist_to_workspace: root: '.' paths: - pkl-cli/build/executable/ - - run: - command: |- - mkdir ~/test-results/ - find . -type f -regex ".*/build/test-results/.*xml" -exec cp {} ~/test-results/ \; - name: Gather test results - when: always - store_test_results: path: ~/test-results environment: @@ -120,18 +108,12 @@ jobs: - run: command: |- export PATH=~/staticdeps/bin:$PATH - ./gradlew --info --stacktrace -DreleaseBuild=true pkl-cli:macExecutableAarch64 pkl-core:testMacExecutableAarch64 + ./gradlew --info --stacktrace -DtestReportsDir=${HOME}/test-results -DreleaseBuild=true pkl-cli:macExecutableAarch64 pkl-core:testMacExecutableAarch64 name: gradle buildNative - persist_to_workspace: root: '.' paths: - pkl-cli/build/executable/ - - run: - command: |- - mkdir ~/test-results/ - find . -type f -regex ".*/build/test-results/.*xml" -exec cp {} ~/test-results/ \; - name: Gather test results - when: always - store_test_results: path: ~/test-results environment: @@ -186,18 +168,12 @@ jobs: - run: command: |- export PATH=~/staticdeps/bin:$PATH - ./gradlew --info --stacktrace -DreleaseBuild=true pkl-cli:linuxExecutableAarch64 pkl-core:testLinuxExecutableAarch64 + ./gradlew --info --stacktrace -DtestReportsDir=${HOME}/test-results -DreleaseBuild=true pkl-cli:linuxExecutableAarch64 pkl-core:testLinuxExecutableAarch64 name: gradle buildNative - persist_to_workspace: root: '.' paths: - pkl-cli/build/executable/ - - run: - command: |- - mkdir ~/test-results/ - find . -type f -regex ".*/build/test-results/.*xml" -exec cp {} ~/test-results/ \; - name: Gather test results - when: always - store_test_results: path: ~/test-results environment: @@ -269,18 +245,12 @@ jobs: - run: command: |- export PATH=~/staticdeps/bin:$PATH - ./gradlew --info --stacktrace -DreleaseBuild=true pkl-cli:alpineExecutableAmd64 pkl-core:testAlpineExecutableAmd64 + ./gradlew --info --stacktrace -DtestReportsDir=${HOME}/test-results -DreleaseBuild=true pkl-cli:alpineExecutableAmd64 pkl-core:testAlpineExecutableAmd64 name: gradle buildNative - persist_to_workspace: root: '.' paths: - pkl-cli/build/executable/ - - run: - command: |- - mkdir ~/test-results/ - find . -type f -regex ".*/build/test-results/.*xml" -exec cp {} ~/test-results/ \; - name: Gather test results - when: always - store_test_results: path: ~/test-results environment: @@ -289,6 +259,26 @@ jobs: resource_class: xlarge docker: - image: oraclelinux:8-slim + pkl-cli-windows-amd64-release: + steps: + - checkout + - run: + command: |- + export PATH=~/staticdeps/bin:$PATH + ./gradlew --info --stacktrace -DtestReportsDir=${HOME}/test-results -DreleaseBuild=true pkl-cli:windowsExecutableAmd64 pkl-core:testWindowsExecutableAmd64 + name: gradle buildNative + shell: bash.exe + - persist_to_workspace: + root: '.' + paths: + - pkl-cli/build/executable/ + - store_test_results: + path: ~/test-results + environment: + LANG: en_US.UTF-8 + resource_class: windows.large + machine: + image: windows-server-2022-gui:current pkl-cli-macOS-amd64-snapshot: steps: - checkout @@ -298,18 +288,12 @@ jobs: - run: command: |- export PATH=~/staticdeps/bin:$PATH - ./gradlew --info --stacktrace pkl-cli:macExecutableAmd64 pkl-core:testMacExecutableAmd64 + ./gradlew --info --stacktrace -DtestReportsDir=${HOME}/test-results pkl-cli:macExecutableAmd64 pkl-core:testMacExecutableAmd64 name: gradle buildNative - persist_to_workspace: root: '.' paths: - pkl-cli/build/executable/ - - run: - command: |- - mkdir ~/test-results/ - find . -type f -regex ".*/build/test-results/.*xml" -exec cp {} ~/test-results/ \; - name: Gather test results - when: always - store_test_results: path: ~/test-results environment: @@ -380,18 +364,12 @@ jobs: - run: command: |- export PATH=~/staticdeps/bin:$PATH - ./gradlew --info --stacktrace pkl-cli:linuxExecutableAmd64 pkl-core:testLinuxExecutableAmd64 + ./gradlew --info --stacktrace -DtestReportsDir=${HOME}/test-results pkl-cli:linuxExecutableAmd64 pkl-core:testLinuxExecutableAmd64 name: gradle buildNative - persist_to_workspace: root: '.' paths: - pkl-cli/build/executable/ - - run: - command: |- - mkdir ~/test-results/ - find . -type f -regex ".*/build/test-results/.*xml" -exec cp {} ~/test-results/ \; - name: Gather test results - when: always - store_test_results: path: ~/test-results environment: @@ -406,18 +384,12 @@ jobs: - run: command: |- export PATH=~/staticdeps/bin:$PATH - ./gradlew --info --stacktrace pkl-cli:macExecutableAarch64 pkl-core:testMacExecutableAarch64 + ./gradlew --info --stacktrace -DtestReportsDir=${HOME}/test-results pkl-cli:macExecutableAarch64 pkl-core:testMacExecutableAarch64 name: gradle buildNative - persist_to_workspace: root: '.' paths: - pkl-cli/build/executable/ - - run: - command: |- - mkdir ~/test-results/ - find . -type f -regex ".*/build/test-results/.*xml" -exec cp {} ~/test-results/ \; - name: Gather test results - when: always - store_test_results: path: ~/test-results environment: @@ -472,18 +444,12 @@ jobs: - run: command: |- export PATH=~/staticdeps/bin:$PATH - ./gradlew --info --stacktrace pkl-cli:linuxExecutableAarch64 pkl-core:testLinuxExecutableAarch64 + ./gradlew --info --stacktrace -DtestReportsDir=${HOME}/test-results pkl-cli:linuxExecutableAarch64 pkl-core:testLinuxExecutableAarch64 name: gradle buildNative - persist_to_workspace: root: '.' paths: - pkl-cli/build/executable/ - - run: - command: |- - mkdir ~/test-results/ - find . -type f -regex ".*/build/test-results/.*xml" -exec cp {} ~/test-results/ \; - name: Gather test results - when: always - store_test_results: path: ~/test-results environment: @@ -555,18 +521,12 @@ jobs: - run: command: |- export PATH=~/staticdeps/bin:$PATH - ./gradlew --info --stacktrace pkl-cli:alpineExecutableAmd64 pkl-core:testAlpineExecutableAmd64 + ./gradlew --info --stacktrace -DtestReportsDir=${HOME}/test-results pkl-cli:alpineExecutableAmd64 pkl-core:testAlpineExecutableAmd64 name: gradle buildNative - persist_to_workspace: root: '.' paths: - pkl-cli/build/executable/ - - run: - command: |- - mkdir ~/test-results/ - find . -type f -regex ".*/build/test-results/.*xml" -exec cp {} ~/test-results/ \; - name: Gather test results - when: always - store_test_results: path: ~/test-results environment: @@ -575,18 +535,32 @@ jobs: resource_class: xlarge docker: - image: oraclelinux:8-slim + pkl-cli-windows-amd64-snapshot: + steps: + - checkout + - run: + command: |- + export PATH=~/staticdeps/bin:$PATH + ./gradlew --info --stacktrace -DtestReportsDir=${HOME}/test-results pkl-cli:windowsExecutableAmd64 pkl-core:testWindowsExecutableAmd64 + name: gradle buildNative + shell: bash.exe + - persist_to_workspace: + root: '.' + paths: + - pkl-cli/build/executable/ + - store_test_results: + path: ~/test-results + environment: + LANG: en_US.UTF-8 + resource_class: windows.large + machine: + image: windows-server-2022-gui:current gradle-check-jdk17: steps: - checkout - run: - command: ./gradlew --info --stacktrace check + command: ./gradlew --info --stacktrace -DtestReportsDir=${HOME}/test-results check name: gradle check - - run: - command: |- - mkdir ~/test-results/ - find . -type f -regex ".*/build/test-results/.*xml" -exec cp {} ~/test-results/ \; - name: Gather test results - when: always - store_test_results: path: ~/test-results environment: @@ -597,32 +571,33 @@ jobs: steps: - checkout - run: - command: ./gradlew --info --stacktrace check + command: ./gradlew --info --stacktrace -DtestReportsDir=${HOME}/test-results check name: gradle check - - run: - command: |- - mkdir ~/test-results/ - find . -type f -regex ".*/build/test-results/.*xml" -exec cp {} ~/test-results/ \; - name: Gather test results - when: always - store_test_results: path: ~/test-results environment: LANG: en_US.UTF-8 docker: - image: cimg/openjdk:21.0 + gradle-check-jdk17-windows: + steps: + - checkout + - run: + command: ./gradlew --info --stacktrace -DtestReportsDir=${HOME}/test-results check + name: gradle check + - store_test_results: + path: ~/test-results + environment: + LANG: en_US.UTF-8 + resource_class: windows.large + machine: + image: windows-server-2022-gui:current bench: steps: - checkout - run: - command: ./gradlew --info --stacktrace bench:jmh + command: ./gradlew --info --stacktrace -DtestReportsDir=${HOME}/test-results bench:jmh name: bench:jmh - - run: - command: |- - mkdir ~/test-results/ - find . -type f -regex ".*/build/test-results/.*xml" -exec cp {} ~/test-results/ \; - name: Gather test results - when: always - store_test_results: path: ~/test-results environment: @@ -634,16 +609,10 @@ jobs: - checkout - run: command: |- - ./gradlew --info --stacktrace :pkl-gradle:build \ + ./gradlew --info --stacktrace -DtestReportsDir=${HOME}/test-results :pkl-gradle:build \ :pkl-gradle:compatibilityTestReleases \ :pkl-gradle:compatibilityTestCandidate name: gradle compatibility - - run: - command: |- - mkdir ~/test-results/ - find . -type f -regex ".*/build/test-results/.*xml" -exec cp {} ~/test-results/ \; - name: Gather test results - when: always - store_test_results: path: ~/test-results environment: @@ -656,17 +625,11 @@ jobs: - attach_workspace: at: '.' - run: - command: ./gradlew --info --stacktrace publishToSonatype + command: ./gradlew --info --stacktrace -DtestReportsDir=${HOME}/test-results publishToSonatype - persist_to_workspace: root: '.' paths: - pkl-cli/build/executable/ - - run: - command: |- - mkdir ~/test-results/ - find . -type f -regex ".*/build/test-results/.*xml" -exec cp {} ~/test-results/ \; - name: Gather test results - when: always - store_test_results: path: ~/test-results environment: @@ -679,17 +642,11 @@ jobs: - attach_workspace: at: '.' - run: - command: ./gradlew --info --stacktrace -DreleaseBuild=true publishToSonatype closeAndReleaseSonatypeStagingRepository + command: ./gradlew --info --stacktrace -DtestReportsDir=${HOME}/test-results -DreleaseBuild=true publishToSonatype closeAndReleaseSonatypeStagingRepository - persist_to_workspace: root: '.' paths: - pkl-cli/build/executable/ - - run: - command: |- - mkdir ~/test-results/ - find . -type f -regex ".*/build/test-results/.*xml" -exec cp {} ~/test-results/ \; - name: Gather test results - when: always - store_test_results: path: ~/test-results environment: @@ -753,6 +710,9 @@ workflows: - gradle-check-jdk21: requires: - hold + - gradle-check-jdk17-windows: + requires: + - hold when: matches: value: << pipeline.git.branch >> @@ -761,6 +721,7 @@ workflows: jobs: - gradle-check-jdk17 - gradle-check-jdk21 + - gradle-check-jdk17-windows - bench - gradle-compatibility - pkl-cli-macOS-amd64-snapshot @@ -768,10 +729,12 @@ workflows: - pkl-cli-macOS-aarch64-snapshot - pkl-cli-linux-aarch64-snapshot - pkl-cli-linux-alpine-amd64-snapshot + - pkl-cli-windows-amd64-snapshot - deploy-snapshot: requires: - gradle-check-jdk17 - gradle-check-jdk21 + - gradle-check-jdk17-windows - bench - gradle-compatibility - pkl-cli-macOS-amd64-snapshot @@ -779,6 +742,7 @@ workflows: - pkl-cli-macOS-aarch64-snapshot - pkl-cli-linux-aarch64-snapshot - pkl-cli-linux-alpine-amd64-snapshot + - pkl-cli-windows-amd64-snapshot context: pkl-maven-release - trigger-docsite-build: requires: @@ -803,6 +767,12 @@ workflows: ignore: /.*/ tags: only: /^v?\d+\.\d+\.\d+$/ + - gradle-check-jdk17-windows: + filters: + branches: + ignore: /.*/ + tags: + only: /^v?\d+\.\d+\.\d+$/ - bench: filters: branches: @@ -845,10 +815,17 @@ workflows: ignore: /.*/ tags: only: /^v?\d+\.\d+\.\d+$/ + - pkl-cli-windows-amd64-release: + filters: + branches: + ignore: /.*/ + tags: + only: /^v?\d+\.\d+\.\d+$/ - github-release: requires: - gradle-check-jdk17 - gradle-check-jdk21 + - gradle-check-jdk17-windows - bench - gradle-compatibility - pkl-cli-macOS-amd64-release @@ -856,6 +833,7 @@ workflows: - pkl-cli-macOS-aarch64-release - pkl-cli-linux-aarch64-release - pkl-cli-linux-alpine-amd64-release + - pkl-cli-windows-amd64-release context: pkl-github-release filters: branches: @@ -885,6 +863,7 @@ workflows: jobs: - gradle-check-jdk17 - gradle-check-jdk21 + - gradle-check-jdk17-windows - bench - gradle-compatibility - pkl-cli-macOS-amd64-release @@ -892,6 +871,7 @@ workflows: - pkl-cli-macOS-aarch64-release - pkl-cli-linux-aarch64-release - pkl-cli-linux-alpine-amd64-release + - pkl-cli-windows-amd64-release when: matches: value: << pipeline.git.branch >> diff --git a/.circleci/jobs/BuildNativeJob.pkl b/.circleci/jobs/BuildNativeJob.pkl index 8ce143d5..139dd346 100644 --- a/.circleci/jobs/BuildNativeJob.pkl +++ b/.circleci/jobs/BuildNativeJob.pkl @@ -16,12 +16,9 @@ /// Builds the native `pkl` CLI extends "GradleJob.pkl" -import "package://pkg.pkl-lang.org/pkl-pantry/com.circleci.v2@1.1.0#/Config.pkl" +import "package://pkg.pkl-lang.org/pkl-pantry/com.circleci.v2@1.1.2#/Config.pkl" import "package://pkg.pkl-lang.org/pkl-pantry/pkl.experimental.uri@1.0.0#/URI.pkl" -/// The OS to run on -os: "macOS"|"linux" - /// The architecture to use arch: "amd64"|"aarch64" @@ -119,10 +116,14 @@ steps { new Config.RunStep { name = "gradle buildNative" local _os = - if (os == "macOS") "mac" + if (module.os == "macOS") "mac" else if (musl) "alpine" + else if (module.os == "windows") "windows" else "linux" local jobName = "\(_os)Executable\(arch.capitalize())" + when (module.os == "windows") { + shell = "bash.exe" + } command = #""" export PATH=~/staticdeps/bin:$PATH ./gradlew \#(module.gradleArgs) pkl-cli:\#(jobName) pkl-core:test\#(jobName.capitalize()) @@ -142,7 +143,8 @@ job { xcode = "15.3.0" } resource_class = "macos.m1.large.gen1" - } else { + } + when (os == "linux") { docker { new { image = if (arch == "aarch64") "arm64v8/oraclelinux:8-slim" else "oraclelinux:8-slim" @@ -153,4 +155,10 @@ job { } resource_class = if (arch == "aarch64") "arm.xlarge" else "xlarge" } + when (os == "windows") { + machine { + image = "windows-server-2022-gui:current" + } + resource_class = "windows.large" + } } diff --git a/.circleci/jobs/DeployJob.pkl b/.circleci/jobs/DeployJob.pkl index 6ad2761d..a46faea7 100644 --- a/.circleci/jobs/DeployJob.pkl +++ b/.circleci/jobs/DeployJob.pkl @@ -15,7 +15,7 @@ //===----------------------------------------------------------------------===// extends "GradleJob.pkl" -import "package://pkg.pkl-lang.org/pkl-pantry/com.circleci.v2@1.1.0#/Config.pkl" +import "package://pkg.pkl-lang.org/pkl-pantry/com.circleci.v2@1.1.2#/Config.pkl" local self = this @@ -27,6 +27,8 @@ job { } } +os = "linux" + steps { new Config.AttachWorkspaceStep { at = "." } new Config.RunStep { diff --git a/.circleci/jobs/GradleCheckJob.pkl b/.circleci/jobs/GradleCheckJob.pkl index 1e4937ea..39ab7beb 100644 --- a/.circleci/jobs/GradleCheckJob.pkl +++ b/.circleci/jobs/GradleCheckJob.pkl @@ -15,9 +15,11 @@ //===----------------------------------------------------------------------===// extends "GradleJob.pkl" -import "package://pkg.pkl-lang.org/pkl-pantry/com.circleci.v2@1.1.0#/Config.pkl" +import "package://pkg.pkl-lang.org/pkl-pantry/com.circleci.v2@1.1.2#/Config.pkl" -javaVersion: "11.0"|"17.0"|"21.0" +javaVersion: "17.0"|"21.0" + +os = "linux" steps { new Config.RunStep { @@ -27,9 +29,17 @@ steps { } job { - docker { - new { - image = "cimg/openjdk:\(javaVersion)" + when (os == "linux") { + docker { + new { + image = "cimg/openjdk:\(javaVersion)" + } } } + when (os == "windows") { + machine { + image = "windows-server-2022-gui:current" + } + resource_class = "windows.large" + } } diff --git a/.circleci/jobs/GradleJob.pkl b/.circleci/jobs/GradleJob.pkl index 298093c9..bb9c7fb1 100644 --- a/.circleci/jobs/GradleJob.pkl +++ b/.circleci/jobs/GradleJob.pkl @@ -15,14 +15,18 @@ //===----------------------------------------------------------------------===// abstract module GradleJob -import "package://pkg.pkl-lang.org/pkl-pantry/com.circleci.v2@1.1.0#/Config.pkl" +import "package://pkg.pkl-lang.org/pkl-pantry/com.circleci.v2@1.1.2#/Config.pkl" /// Whether this is a release build or not. isRelease: Boolean = false +/// The OS to run on +os: "macOS"|"linux"|"windows" + fixed gradleArgs = new Listing { "--info" "--stacktrace" + "-DtestReportsDir=${HOME}/test-results" when (isRelease) { "-DreleaseBuild=true" } @@ -37,15 +41,6 @@ job: Config.Job = new { steps { "checkout" ...module.steps - new Config.RunStep { - // find all test results and write them to the home dir - name = "Gather test results" - command = """ - mkdir ~/test-results/ - find . -type f -regex ".*/build/test-results/.*xml" -exec cp {} ~/test-results/ \\; - """ - `when` = "always" - } new Config.StoreTestResults { path = "~/test-results" } diff --git a/.circleci/jobs/SimpleGradleJob.pkl b/.circleci/jobs/SimpleGradleJob.pkl index c34f4dce..ef2526b9 100644 --- a/.circleci/jobs/SimpleGradleJob.pkl +++ b/.circleci/jobs/SimpleGradleJob.pkl @@ -15,12 +15,14 @@ //===----------------------------------------------------------------------===// extends "GradleJob.pkl" -import "package://pkg.pkl-lang.org/pkl-pantry/com.circleci.v2@1.1.0#/Config.pkl" +import "package://pkg.pkl-lang.org/pkl-pantry/com.circleci.v2@1.1.2#/Config.pkl" name: String = command command: String +os = "linux" + steps { new Config.RunStep { name = module.name diff --git a/.gitattributes b/.gitattributes index 896495e3..39935713 100644 --- a/.gitattributes +++ b/.gitattributes @@ -4,3 +4,4 @@ /docs/** linguist-documentation *.pkl linguist-language=Groovy +* text eol=lf diff --git a/DEVELOPMENT.adoc b/DEVELOPMENT.adoc index a129fa4a..0f42d127 100644 --- a/DEVELOPMENT.adoc +++ b/DEVELOPMENT.adoc @@ -40,6 +40,7 @@ pkl-cli/build/executable/jpkl # run Java executable pkl-cli/build/executable/pkl-macos-amd64 # run Mac executable pkl-cli/build/executable/pkl-linux-amd64 # run Linux executable pkl-cli/build/executable/pkl-alpine-linux-amd64 # run Alpine Linux executable +pkl-cli/build/executable/pkl-windows-amd64.exe # run Windows executable ---- == Update Gradle @@ -68,6 +69,13 @@ based on version information from https://search.maven.org, https://plugins.grad * ANTLR code generation is performed by task `:pkl-core:generateGrammarSource` ** Output dir is `generated/antlr/` +== Remote JVM Debugging + +To enable remote JVM debugging when running Gradle tasks (e.g. test), add the flag `-Djvmdebug=true`. +This will listen on port 5005. + +Example: `./gradlew test -Djvmdebug=true` + == Resources For automated build setup examples see our https://github.com/apple/pkl/blob/main/.circleci/[CircleCI] jobs like our https://github.com/apple/pkl/blob/main/.circleci/jobs/BuildNativeJob.pkl[BuildNativeJob.pkl], where we build Pkl automatically. diff --git a/buildSrc/src/main/kotlin/BuildInfo.kt b/buildSrc/src/main/kotlin/BuildInfo.kt index d293d340..0264375d 100644 --- a/buildSrc/src/main/kotlin/BuildInfo.kt +++ b/buildSrc/src/main/kotlin/BuildInfo.kt @@ -26,6 +26,7 @@ open class BuildInfo(project: Project) { when { os.isMacOsX -> "macos" os.isLinux -> "linux" + os.isWindows -> "windows" else -> throw RuntimeException("${os.familyName} is not supported.") } } @@ -36,7 +37,8 @@ open class BuildInfo(project: Project) { val downloadUrl: String by lazy { val jdkMajor = graalVmJdkVersion.takeWhile { it != '.' } - "https://download.oracle.com/graalvm/$jdkMajor/archive/$baseName.tar.gz" + val extension = if (os.isWindows) "zip" else "tar.gz" + "https://download.oracle.com/graalvm/$jdkMajor/archive/$baseName.$extension" } val installDir: File by lazy { @@ -85,9 +87,14 @@ open class BuildInfo(project: Project) { val commitId: String by lazy { // only run command once per build invocation if (project === project.rootProject) { - Runtime.getRuntime() - .exec(arrayOf("git", "rev-parse", "--short", "HEAD"), arrayOf(), project.rootDir) - .inputStream.reader().readText().trim() + val process = ProcessBuilder() + .command("git", "rev-parse", "--short", "HEAD") + .directory(project.rootDir) + .start() + process.waitFor().also { exitCode -> + if (exitCode == -1) throw RuntimeException(process.errorStream.reader().readText()) + } + process.inputStream.reader().readText().trim() } else { project.rootProject.extensions.getByType(BuildInfo::class.java).commitId } diff --git a/buildSrc/src/main/kotlin/ExecutableJar.kt b/buildSrc/src/main/kotlin/ExecutableJar.kt index 70b79706..ed63a284 100644 --- a/buildSrc/src/main/kotlin/ExecutableJar.kt +++ b/buildSrc/src/main/kotlin/ExecutableJar.kt @@ -1,11 +1,11 @@ import org.gradle.api.DefaultTask import org.gradle.api.file.RegularFileProperty import org.gradle.api.provider.ListProperty +import org.gradle.api.provider.Property import org.gradle.api.tasks.Input import org.gradle.api.tasks.InputFile import org.gradle.api.tasks.OutputFile import org.gradle.api.tasks.TaskAction -import org.gradle.kotlin.dsl.listProperty /** * Builds a self-contained Pkl CLI Jar that is directly executable on *nix @@ -15,27 +15,25 @@ import org.gradle.kotlin.dsl.listProperty * * https://skife.org/java/unix/2011/06/20/really_executable_jars.html */ -open class ExecutableJar : DefaultTask() { +abstract class ExecutableJar : DefaultTask() { @get:InputFile - val inJar: RegularFileProperty = project.objects.fileProperty() + abstract val inJar: RegularFileProperty @get:OutputFile - val outJar: RegularFileProperty = project.objects.fileProperty() + abstract val outJar: RegularFileProperty @get:Input - val jvmArgs: ListProperty = project.objects.listProperty() + abstract val jvmArgs: ListProperty @TaskAction fun buildJar() { val inFile = inJar.get().asFile val outFile = outJar.get().asFile val escapedJvmArgs = jvmArgs.get().joinToString(separator = " ") { "\"$it\"" } - val startScript = """ #!/bin/sh exec java $escapedJvmArgs -jar $0 "$@" - """.trim().trimMargin() + "\n\n\n" - + """.trimIndent() + "\n\n\n" outFile.outputStream().use { outStream -> startScript.byteInputStream().use { it.copyTo(outStream) } inFile.inputStream().use { it.copyTo(outStream) } diff --git a/buildSrc/src/main/kotlin/pklAllProjects.gradle.kts b/buildSrc/src/main/kotlin/pklAllProjects.gradle.kts index 6ef0256e..b65b51ab 100644 --- a/buildSrc/src/main/kotlin/pklAllProjects.gradle.kts +++ b/buildSrc/src/main/kotlin/pklAllProjects.gradle.kts @@ -93,3 +93,28 @@ val updateDependencyLocks by tasks.registering { } val allDependencies by tasks.registering(DependencyReportTask::class) + +tasks.withType(Test::class).configureEach { + System.getProperty("testReportsDir")?.let { reportsDir -> + reports.junitXml.outputLocation.set(file(reportsDir).resolve(project.name).resolve(name)) + } + debugOptions { + enabled = System.getProperty("jvmdebug")?.toBoolean() ?: false + @Suppress("UnstableApiUsage") + host = "*" + port = 5005 + suspend = true + server = true + } +} + +tasks.withType(JavaExec::class).configureEach { + debugOptions { + enabled = System.getProperty("jvmdebug")?.toBoolean() ?: false + @Suppress("UnstableApiUsage") + host = "*" + port = 5005 + suspend = true + server = true + } +} diff --git a/buildSrc/src/main/kotlin/pklGraalVm.gradle.kts b/buildSrc/src/main/kotlin/pklGraalVm.gradle.kts index 175c83f2..eaf5a632 100644 --- a/buildSrc/src/main/kotlin/pklGraalVm.gradle.kts +++ b/buildSrc/src/main/kotlin/pklGraalVm.gradle.kts @@ -2,6 +2,7 @@ import java.nio.file.* import java.util.UUID import de.undercouch.gradle.tasks.download.Download import de.undercouch.gradle.tasks.download.Verify +import kotlin.io.path.createDirectories plugins { id("de.undercouch.download") @@ -9,7 +10,10 @@ plugins { val buildInfo = project.extensions.getByType() -val BuildInfo.GraalVm.downloadFile get() = file(homeDir).resolve("${baseName}.tar.gz") +val BuildInfo.GraalVm.downloadFile get(): File { + val extension = if (buildInfo.os.isWindows) "zip" else "tar.gz" + return file(homeDir).resolve("${baseName}.$extension") +} // tries to minimize chance of corruption by download-to-temp-file-and-move val downloadGraalVmAarch64 by tasks.registering(Download::class) { @@ -72,11 +76,10 @@ fun Task.configureInstallGraalVm(graalVm: BuildInfo.GraalVm) { } doLast { - val distroDir = "${graalVm.homeDir}/${UUID.randomUUID()}" + val distroDir = Paths.get(graalVm.homeDir, UUID.randomUUID().toString()) try { - mkdir(distroDir) - + distroDir.createDirectories() println("Extracting ${graalVm.downloadFile} into $distroDir") // faster and more reliable than Gradle's `copy { from tarTree() }` exec { @@ -85,17 +88,18 @@ fun Task.configureInstallGraalVm(graalVm: BuildInfo.GraalVm) { args("--strip-components=1", "-xzf", graalVm.downloadFile) } - val distroBinDir = if (buildInfo.os.isMacOsX) "$distroDir/Contents/Home/bin" else "$distroDir/bin" + val distroBinDir = if (buildInfo.os.isMacOsX) distroDir.resolve("Contents/Home/bin") else distroDir.resolve("bin") println("Installing native-image into $distroDir") exec { - executable = "$distroBinDir/gu" + val executableName = if (buildInfo.os.isWindows) "gu.cmd" else "gu" + executable = distroBinDir.resolve(executableName).toString() args("install", "--no-progress", "native-image") } println("Creating symlink ${graalVm.installDir} for $distroDir") - val tempLink = Paths.get("${graalVm.homeDir}/${UUID.randomUUID()}") - Files.createSymbolicLink(tempLink, Paths.get(distroDir)) + val tempLink = Paths.get(graalVm.homeDir, UUID.randomUUID().toString()) + Files.createSymbolicLink(tempLink, distroDir) try { Files.move(tempLink, graalVm.installDir.toPath(), StandardCopyOption.ATOMIC_MOVE) } catch (e: Exception) { diff --git a/docs/modules/language-reference/pages/index.adoc b/docs/modules/language-reference/pages/index.adoc index 8e1e38c7..b409b4bf 100644 --- a/docs/modules/language-reference/pages/index.adoc +++ b/docs/modules/language-reference/pages/index.adoc @@ -2140,9 +2140,18 @@ For example, a module with URI `modulepath:/animals/birds/pigeon.pkl` can import `modulepath:/animals/birds/parrot.pkl` with `import "parrot.pkl"` or `import "/animals/birds/parrot.pkl"`. -NOTE: When importing a relative folder or file that starts with `@`, the import string must be prefixed with `./`. -Otherwise, this syntax will be interpreted as dependency notation. +[NOTE] +.Paths on Windows +==== +Relative paths use the `/` character as the directory separator on all platforms, including Windows. +Paths that contain drive letters (e.g. `C:`) must be declared as an absolute file URI, for example: `import "file:///C:/path/to/my/module.pkl"`. Otherwise, they are interpreted as a URI scheme. +==== + +NOTE: When importing a relative directory or file that starts with `@`, the import string must be prefixed with `./`. +Otherwise, this syntax will be interpreted as xref:dependency-notation[dependency notation]. + +[#dependency-notation] ==== Dependency notation URIs Example: `+@birds/bird.pkl+` diff --git a/docs/modules/pkl-cli/pages/index.adoc b/docs/modules/pkl-cli/pages/index.adoc index 02b36513..5f4e1001 100644 --- a/docs/modules/pkl-cli/pages/index.adoc +++ b/docs/modules/pkl-cli/pages/index.adoc @@ -8,6 +8,7 @@ include::ROOT:partial$component-attributes.adoc[] :uri-pkl-linux-amd64-download: {uri-sonatype-snapshot-download}&a=pkl-cli-linux-amd64&e=bin :uri-pkl-linux-aarch64-download: {uri-sonatype-snapshot-download}&a=pkl-cli-linux-aarch64&e=bin :uri-pkl-alpine-download: {uri-sonatype-snapshot-download}&a=pkl-cli-alpine-linux-amd64&e=bin +:uri-pkl-windows-download: {uri-sonatype-snapshot-download}&a=pkl-cli-windows-amd64&e=exe :uri-pkl-java-download: {uri-sonatype-snapshot-download}&a=pkl-cli-java&e=jar ifdef::is-release-version[] @@ -16,6 +17,7 @@ ifdef::is-release-version[] :uri-pkl-linux-amd64-download: {github-releases}/pkl-linux-amd64 :uri-pkl-linux-aarch64-download: {github-releases}/pkl-linux-aarch64 :uri-pkl-alpine-download: {github-releases}/pkl-alpine-linux-amd64 +:uri-pkl-windows-download: {github-releases}/pkl-windows-amd64.exe :uri-pkl-java-download: {uri-maven-repo}/org/pkl-lang/pkl-cli-java/{pkl-artifact-version}/pkl-cli-java-{pkl-artifact-version}.jar endif::[] @@ -37,9 +39,10 @@ The CLI comes in multiple flavors: * Native Linux executable for amd64 * Native Linux executable for aarch64 * Native Alpine Linux executable for amd64 (cross-compiled and tested on Oracle Linux 8) -* Java executable (tested with Java 17/21 on macOS and Oracle Linux, may work on other platforms) +* Native Windows executable for amd64 (tested on Windows Server 2022) +* Java executable (tested with Java 17/21 on macOS and Oracle Linux) -On macOS and Linux, we recommend using the native executables. +On macOS, Linux, and Windows, we recommend using the native executables. They are self-contained, start up instantly, and run complex Pkl code much faster than the Java executable. .What is the Difference Between the Linux and Alpine Linux Executables? @@ -59,7 +62,7 @@ Except where noted otherwise, the rest of this page discusses the native executa [[homebrew]] === Homebrew -Release versions can be installed with {uri-homebrew}[Homebrew]. +On macOS and Linux, release versions can be installed with {uri-homebrew}[Homebrew]. ifdef::is-release-version[] To install Pkl, run: @@ -111,7 +114,7 @@ chmod +x pkl This should print something similar to: -[source,shell] +[source] [subs="+attributes"] ---- Pkl {pkl-version} (macOS, native) @@ -145,7 +148,7 @@ chmod +x pkl This should print something similar to: -[source,shell] +[source] [subs="+attributes"] ---- Pkl {pkl-version} (Linux, native) @@ -167,7 +170,7 @@ chmod +x pkl This should print something similar to: -[source,shell] +[source] [subs="+attributes"] ---- Pkl {pkl-version} (Linux, native) @@ -175,6 +178,23 @@ Pkl {pkl-version} (Linux, native) NOTE: We currently do not support the aarch64 architecture for Alpine Linux. +=== Windows Executable + +[source,PowerShell] +[subs="+attributes"] +---- +Invoke-WebRequest '{uri-pkl-windows-download}' -OutFile pkl.exe +.\pkl --version +---- + +[source] +[subs="+attributes"] +---- +Pkl {pkl-version} (Windows 10.0, native) +---- + +NOTE: We currently do not support the aarch64 architecture for Windows. + === Java Executable [source,shell] @@ -193,27 +213,8 @@ This should print something similar to: Pkl {pkl-version} (macOS 14.2, Java 17.0.10) ---- -=== Windows support -Pkl does not currently support running natively on Windows. Pkl has been reported to work on the https://learn.microsoft.com/en-us/windows/wsl/install[Windows Subsystem for Linux] and on a https://www.oracle.com/java/technologies/downloads/#jdk21-windows[Java Runtime] using https://search.maven.org/remote_content?g=org.pkl-lang&a=pkl-cli-java&v=LATEST[`jpkl` (the Java executable version of Pkl)]. - -The following is from successful uses of `jpkl` on Windows: -[source,shell] -[subs="+attributes"] ----- -> java -jar pkl-cli-java.jar --version -Pkl 0.26.0-dev+21e0e14 (Windows 10 10.0, Java 21.0.2) ----- - -Note: You must use forward slashes in all paths for Windows with absolute paths prefixed with file:///. The following examples are valid: -[source,shell] -[subs="+attributes"] ----- -> java -jar pkl-cli-java.jar eval file:///C:/Code/pkl/test.pkl -> java -jar pkl-cli-java.jar eval ../Code/pkl/test.pkl -> java -jar pkl-cli-java.jar eval pkl/test.pkl ----- -https://github.com/apple/pkl/issues/20[GitHub Issue #20] is used to track progress on support for the Windows platform. - +NOTE: The Java executable does not work as an executable file on Windows. +However, it will work as a jar, for example, with `java -jar jpkl`. [[usage]] == Usage diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 92e5c831..e0d2e7a6 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -15,6 +15,7 @@ graalVmSha256-macos-x64 = "14f4bd6417809905f86e786c779d0fc2feb840d7dac35ae3503eb graalVmSha256-macos-aarch64 = "e944c5ce5da56e683fc8f1a57191b46d9cb702930b1688bda064fcf467d876b8" graalVmSha256-linux-x64 = "112dc9b92d81a946f1b5b334646151b790785c813e76fcf13527a319003d7e2c" graalVmSha256-linux-aarch64 = "c95ac550d070f06666cf8c1023a098380dd565be00866473caf6ff1b7cdf680c" +graalVmSha256-windows-x64 = "1ab2291e71f54d73e3e57b7fccbf184cabcba37e16ca9d1cf42d08474a7c02f0" ideaExtPlugin = "1.1" javaPoet = "1.+" javaxInject = "1" diff --git a/pkl-cli/pkl-cli.gradle.kts b/pkl-cli/pkl-cli.gradle.kts index c9006431..692abd3b 100644 --- a/pkl-cli/pkl-cli.gradle.kts +++ b/pkl-cli/pkl-cli.gradle.kts @@ -33,6 +33,7 @@ val stagedMacAarch64Executable: Configuration by configurations.creating val stagedLinuxAmd64Executable: Configuration by configurations.creating val stagedLinuxAarch64Executable: Configuration by configurations.creating val stagedAlpineLinuxAmd64Executable: Configuration by configurations.creating +val stagedWindowsAmd64Executable: Configuration by configurations.creating dependencies { compileOnly(libs.svm) @@ -63,6 +64,7 @@ dependencies { stagedLinuxAmd64Executable(executableDir("pkl-linux-amd64")) stagedLinuxAarch64Executable(executableDir("pkl-linux-aarch64")) stagedAlpineLinuxAmd64Executable(executableDir("pkl-alpine-linux-amd64")) + stagedWindowsAmd64Executable(executableDir("pkl-windows-amd64.exe")) } tasks.jar { @@ -121,12 +123,17 @@ val testStartJavaExecutable by tasks.registering(Exec::class) { dependsOn(javaExecutable) val outputFile = layout.buildDirectory.file("testStartJavaExecutable") // dummy output to satisfy up-to-date check outputs.file(outputFile) - - executable = javaExecutable.get().outputs.files.singleFile.toString() - args("--version") - + + if (buildInfo.os.isWindows) { + executable = "java" + args("-jar", javaExecutable.get().outputs.files.singleFile.toString(), "--version") + } else { + executable = javaExecutable.get().outputs.files.singleFile.toString() + args("--version") + } + doFirst { outputFile.get().asFile.delete() } - + doLast { outputFile.get().asFile.writeText("OK") } } @@ -141,12 +148,13 @@ fun Exec.configureExecutable( ) { inputs.files(sourceSets.main.map { it.output }).withPropertyName("mainSourceSets").withPathSensitivity(PathSensitivity.RELATIVE) inputs.files(configurations.runtimeClasspath).withPropertyName("runtimeClasspath").withNormalizer(ClasspathNormalizer::class) - inputs.files(file(graalVm.baseDir).resolve("bin/native-image")).withPropertyName("graalVmNativeImage").withPathSensitivity(PathSensitivity.ABSOLUTE) + val nativeImageCommandName = if (buildInfo.os.isWindows) "native-image.cmd" else "native-image" + inputs.files(file(graalVm.baseDir).resolve("bin/$nativeImageCommandName")).withPropertyName("graalVmNativeImage").withPathSensitivity(PathSensitivity.ABSOLUTE) outputs.file(outputFile) outputs.cacheIf { true } workingDir(outputFile.map { it.asFile.parentFile }) - executable = "${graalVm.baseDir}/bin/native-image" + executable = "${graalVm.baseDir}/bin/$nativeImageCommandName" // JARs to exclude from the class path for the native-image build. val exclusions = listOf(libs.truffleApi, libs.graalSdk).map { it.get().module.name } @@ -276,6 +284,15 @@ val alpineExecutableAmd64: TaskProvider by tasks.registering(Exec::class) ) } +val windowsExecutableAmd64: TaskProvider by tasks.registering(Exec::class) { + dependsOn(":installGraalVmAmd64") + configureExecutable( + buildInfo.graalVmAmd64, + layout.buildDirectory.file("executable/pkl-windows-amd64"), + listOf("-Dfile.encoding=UTF-8") + ) +} + tasks.assembleNative { when { buildInfo.os.isMacOsX -> { @@ -284,6 +301,9 @@ tasks.assembleNative { dependsOn(macExecutableAarch64) } } + buildInfo.os.isWindows -> { + dependsOn(windowsExecutableAmd64) + } buildInfo.os.isLinux && buildInfo.arch == "aarch64" -> { dependsOn(linuxExecutableAarch64) } @@ -393,6 +413,20 @@ publishing { description.set("Native Pkl CLI executable for linux/amd64 and statically linked to musl.") } } + + create("windowsExecutableAmd64") { + artifactId = "pkl-cli-windows-amd64" + artifact(stagedWindowsAmd64Executable.singleFile) { + classifier = null + extension = "exe" + builtBy(stagedWindowsAmd64Executable) + } + pom { + name.set("pkl-cli-windows-amd64") + url.set("https://github.com/apple/pkl/tree/main/pkl-cli") + description.set("Native Pkl CLI executable for windows/amd64.") + } + } } } @@ -403,5 +437,6 @@ signing { sign(publishing.publications["macExecutableAarch64"]) sign(publishing.publications["macExecutableAmd64"]) sign(publishing.publications["alpineLinuxExecutableAmd64"]) + sign(publishing.publications["windowsExecutableAmd64"]) } //endregion diff --git a/pkl-cli/src/main/kotlin/org/pkl/cli/CliEvaluator.kt b/pkl-cli/src/main/kotlin/org/pkl/cli/CliEvaluator.kt index bcb7841d..96d02a3b 100644 --- a/pkl-cli/src/main/kotlin/org/pkl/cli/CliEvaluator.kt +++ b/pkl-cli/src/main/kotlin/org/pkl/cli/CliEvaluator.kt @@ -111,7 +111,9 @@ constructor( return moduleUris.associateWith { uri -> val moduleDir: String? = - IoUtils.toPath(uri)?.let { workingDir.relativize(it.parent).toString().ifEmpty { "." } } + IoUtils.toPath(uri)?.let { + IoUtils.relativize(it.parent, workingDir).toString().ifEmpty { "." } + } val moduleKey = try { moduleResolver.resolve(uri) @@ -158,7 +160,7 @@ constructor( } else { if (output.isNotEmpty()) { outputFile.writeString( - options.moduleOutputSeparator + IoUtils.getLineSeparator(), + options.moduleOutputSeparator + '\n', Charsets.UTF_8, StandardOpenOption.WRITE, StandardOpenOption.APPEND @@ -192,6 +194,14 @@ constructor( if (uri == VmUtils.REPL_TEXT_URI) ModuleSource.create(uri, reader.readText()) else ModuleSource.uri(uri) + private fun checkPathSpec(pathSpec: String) { + val illegal = pathSpec.indexOfFirst { IoUtils.isReservedFilenameChar(it) && it != '/' } + if (illegal == -1) { + return + } + throw CliException("Path spec `$pathSpec` contains illegal character `${pathSpec[illegal]}`.") + } + /** * Renders each module's `output.files`, writing each entry as a file into the specified output * directory. @@ -207,6 +217,7 @@ constructor( val moduleSource = toModuleSource(moduleUri, consoleReader) val output = evaluator.evaluateOutputFiles(moduleSource) for ((pathSpec, fileOutput) in output) { + checkPathSpec(pathSpec) val resolvedPath = outputDir.resolve(pathSpec).normalize() val realPath = if (resolvedPath.exists()) resolvedPath.toRealPath() else resolvedPath if (!realPath.startsWith(outputDir)) { @@ -228,7 +239,10 @@ constructor( writtenFiles[realPath] = OutputFile(pathSpec, moduleUri) realPath.createParentDirectories() realPath.writeString(fileOutput.text) - consoleWriter.write(currentWorkingDir.relativize(resolvedPath).toString() + "\n") + consoleWriter.write( + IoUtils.relativize(resolvedPath, currentWorkingDir).toString() + + IoUtils.getLineSeparator() + ) consoleWriter.flush() } } diff --git a/pkl-cli/src/main/kotlin/org/pkl/cli/CliPackageDownloader.kt b/pkl-cli/src/main/kotlin/org/pkl/cli/CliPackageDownloader.kt index e8042d08..ac77234d 100644 --- a/pkl-cli/src/main/kotlin/org/pkl/cli/CliPackageDownloader.kt +++ b/pkl-cli/src/main/kotlin/org/pkl/cli/CliPackageDownloader.kt @@ -42,7 +42,11 @@ class CliPackageDownloader( } when (errors.size) { 0 -> return - 1 -> throw CliException(errors.values.single().message!!) + 1 -> + throw CliException( + errors.values.single().message + ?: ("An unexpected error occurred: " + errors.values.single()) + ) else -> throw CliException( buildString { diff --git a/pkl-cli/src/main/kotlin/org/pkl/cli/commands/TestCommand.kt b/pkl-cli/src/main/kotlin/org/pkl/cli/commands/TestCommand.kt index cd0ae557..0e5f66d5 100644 --- a/pkl-cli/src/main/kotlin/org/pkl/cli/commands/TestCommand.kt +++ b/pkl-cli/src/main/kotlin/org/pkl/cli/commands/TestCommand.kt @@ -22,6 +22,7 @@ import com.github.ajalt.clikt.parameters.groups.provideDelegate import java.net.URI import org.pkl.cli.CliTestRunner import org.pkl.commons.cli.commands.BaseCommand +import org.pkl.commons.cli.commands.BaseOptions import org.pkl.commons.cli.commands.ProjectOptions import org.pkl.commons.cli.commands.TestOptions @@ -29,7 +30,7 @@ class TestCommand(helpLink: String) : BaseCommand(name = "test", help = "Run tests within the given module(s)", helpLink = helpLink) { val modules: List by argument(name = "", help = "Module paths or URIs to evaluate.") - .convert { parseModuleName(it) } + .convert { BaseOptions.parseModuleName(it) } .multiple() private val projectOptions by ProjectOptions() diff --git a/pkl-cli/src/test/kotlin/org/pkl/cli/CliEvaluatorTest.kt b/pkl-cli/src/test/kotlin/org/pkl/cli/CliEvaluatorTest.kt index 2ca85e72..44c9aa12 100644 --- a/pkl-cli/src/test/kotlin/org/pkl/cli/CliEvaluatorTest.kt +++ b/pkl-cli/src/test/kotlin/org/pkl/cli/CliEvaluatorTest.kt @@ -28,6 +28,9 @@ import org.junit.jupiter.api.AfterAll import org.junit.jupiter.api.AfterEach import org.junit.jupiter.api.Test import org.junit.jupiter.api.assertThrows +import org.junit.jupiter.api.condition.DisabledOnOs +import org.junit.jupiter.api.condition.EnabledOnOs +import org.junit.jupiter.api.condition.OS import org.junit.jupiter.api.io.TempDir import org.junit.jupiter.params.ParameterizedTest import org.junit.jupiter.params.provider.EnumSource @@ -424,7 +427,10 @@ result = someLib.x checkOutputFile(outputFiles[0], "result.pcf", contents) } + // Can't reliably create symlinks on Windows. + // Might get errors like "A required privilege is not held by the client". @Test + @DisabledOnOs(OS.WINDOWS) fun `moduleDir is relative to workingDir even through symlinks`() { val contents = "foo = 42" val realWorkingDir = tempDir.resolve("workingDir").createDirectories() @@ -978,6 +984,56 @@ result = someLib.x .hasMessageContaining("resolve to the same file path") } + @Test + @EnabledOnOs(OS.WINDOWS) + fun `multiple-file output throws when using invalid Windows characters`() { + val moduleUri = + writePklFile( + "test.pkl", + """ + output { + files { + ["foo:bar"] { text = "bar" } + } + } + """ + .trimIndent() + ) + + val options = + CliEvaluatorOptions( + CliBaseOptions(sourceModules = listOf(moduleUri), workingDir = tempDir), + multipleFileOutputPath = ".output" + ) + assertThatCode { evalToConsole(options) } + .hasMessageContaining("Path spec `foo:bar` contains illegal character `:`.") + } + + @Test + @EnabledOnOs(OS.WINDOWS) + fun `multiple-file output - cannot use backslash as dir separator on Windows`() { + val moduleUri = + writePklFile( + "test.pkl", + """ + output { + files { + ["foo\\bar"] { text = "bar" } + } + } + """ + .trimIndent() + ) + + val options = + CliEvaluatorOptions( + CliBaseOptions(sourceModules = listOf(moduleUri), workingDir = tempDir), + multipleFileOutputPath = ".output" + ) + assertThatCode { evalToConsole(options) } + .hasMessageContaining("Path spec `foo\\bar` contains illegal character `\\`.") + } + @Test fun `evaluate output expression`() { val moduleUri = diff --git a/pkl-cli/src/test/kotlin/org/pkl/cli/CliMainTest.kt b/pkl-cli/src/test/kotlin/org/pkl/cli/CliMainTest.kt index f07dca5c..bb567724 100644 --- a/pkl-cli/src/test/kotlin/org/pkl/cli/CliMainTest.kt +++ b/pkl-cli/src/test/kotlin/org/pkl/cli/CliMainTest.kt @@ -24,6 +24,8 @@ import org.assertj.core.api.Assertions.assertThat import org.assertj.core.api.AssertionsForClassTypes.assertThatCode import org.junit.jupiter.api.Test import org.junit.jupiter.api.assertThrows +import org.junit.jupiter.api.condition.DisabledOnOs +import org.junit.jupiter.api.condition.OS import org.junit.jupiter.api.io.TempDir import org.pkl.cli.commands.EvalCommand import org.pkl.cli.commands.RootCommand @@ -54,7 +56,10 @@ class CliMainTest { assertThatCode { cmd.parse(arrayOf("eval")) }.hasMessage("""Missing argument """"") } + // Can't reliably create symlinks on Windows. + // Might get errors like "A required privilege is not held by the client". @Test + @DisabledOnOs(OS.WINDOWS) fun `output to symlinked directory works`(@TempDir tempDir: Path) { val code = """ diff --git a/pkl-cli/src/test/kotlin/org/pkl/cli/CliProjectPackagerTest.kt b/pkl-cli/src/test/kotlin/org/pkl/cli/CliProjectPackagerTest.kt index 5d25c225..38fea778 100644 --- a/pkl-cli/src/test/kotlin/org/pkl/cli/CliProjectPackagerTest.kt +++ b/pkl-cli/src/test/kotlin/org/pkl/cli/CliProjectPackagerTest.kt @@ -15,6 +15,7 @@ */ package org.pkl.cli +import java.io.File import java.io.StringWriter import java.net.URI import java.nio.file.FileSystems @@ -27,6 +28,8 @@ import org.assertj.core.api.Assertions.assertThatCode import org.junit.jupiter.api.AfterAll import org.junit.jupiter.api.Test import org.junit.jupiter.api.assertThrows +import org.junit.jupiter.api.condition.DisabledOnOs +import org.junit.jupiter.api.condition.OS import org.junit.jupiter.api.io.TempDir import org.pkl.commons.cli.CliBaseOptions import org.pkl.commons.cli.CliException @@ -35,6 +38,7 @@ import org.pkl.commons.readString import org.pkl.commons.test.FileTestUtils import org.pkl.commons.test.PackageServer import org.pkl.commons.writeString +import org.pkl.core.util.IoUtils class CliProjectPackagerTest { companion object { @@ -690,7 +694,11 @@ class CliProjectPackagerTest { ) } + // Absolute path imports on Windows must use an absolute URI (e.g. file:///C:/Foo/Bar), because + // they must contain drive letters, which conflict with schemes. + // We skip validation for absolute URIs, so effectively we skip this check on Windows. @Test + @DisabledOnOs(OS.WINDOWS) fun `import path verification -- absolute import from root dir`(@TempDir tempDir: Path) { tempDir.writeFile( "main.pkl", @@ -738,6 +746,7 @@ class CliProjectPackagerTest { } @Test + @DisabledOnOs(OS.WINDOWS) fun `import path verification -- absolute read from root dir`(@TempDir tempDir: Path) { tempDir.writeFile( "main.pkl", @@ -858,17 +867,18 @@ class CliProjectPackagerTest { consoleWriter = out ) .run() + val sep = File.separatorChar assertThat(out.toString()) - .isEqualTo( + .isEqualToNormalizingNewlines( """ - .out/project1@1.0.0/project1@1.0.0.zip - .out/project1@1.0.0/project1@1.0.0.zip.sha256 - .out/project1@1.0.0/project1@1.0.0 - .out/project1@1.0.0/project1@1.0.0.sha256 - .out/project2@2.0.0/project2@2.0.0.zip - .out/project2@2.0.0/project2@2.0.0.zip.sha256 - .out/project2@2.0.0/project2@2.0.0 - .out/project2@2.0.0/project2@2.0.0.sha256 + .out${sep}project1@1.0.0${sep}project1@1.0.0.zip + .out${sep}project1@1.0.0${sep}project1@1.0.0.zip.sha256 + .out${sep}project1@1.0.0${sep}project1@1.0.0 + .out${sep}project1@1.0.0${sep}project1@1.0.0.sha256 + .out${sep}project2@2.0.0${sep}project2@2.0.0.zip + .out${sep}project2@2.0.0${sep}project2@2.0.0.zip.sha256 + .out${sep}project2@2.0.0${sep}project2@2.0.0 + .out${sep}project2@2.0.0${sep}project2@2.0.0.sha256 """ .trimIndent() @@ -956,13 +966,14 @@ class CliProjectPackagerTest { consoleWriter = out ) .run() + val sep = File.separatorChar assertThat(out.toString()) - .isEqualTo( + .isEqualToNormalizingNewlines( """ - .out/mangos@1.0.0/mangos@1.0.0.zip - .out/mangos@1.0.0/mangos@1.0.0.zip.sha256 - .out/mangos@1.0.0/mangos@1.0.0 - .out/mangos@1.0.0/mangos@1.0.0.sha256 + .out${sep}mangos@1.0.0${sep}mangos@1.0.0.zip + .out${sep}mangos@1.0.0${sep}mangos@1.0.0.zip.sha256 + .out${sep}mangos@1.0.0${sep}mangos@1.0.0 + .out${sep}mangos@1.0.0${sep}mangos@1.0.0.sha256 """ .trimIndent() @@ -971,7 +982,7 @@ class CliProjectPackagerTest { private fun Path.zipFilePaths(): List { return FileSystems.newFileSystem(URI("jar:${toUri()}"), emptyMap()).use { fs -> - Files.walk(fs.getPath("/")).map { it.toString() }.collect(Collectors.toList()) + Files.walk(fs.getPath("/")).map(IoUtils::toNormalizedPathString).collect(Collectors.toList()) } } } diff --git a/pkl-cli/src/test/kotlin/org/pkl/cli/CliProjectResolverTest.kt b/pkl-cli/src/test/kotlin/org/pkl/cli/CliProjectResolverTest.kt index e5e45b3d..4b4f654a 100644 --- a/pkl-cli/src/test/kotlin/org/pkl/cli/CliProjectResolverTest.kt +++ b/pkl-cli/src/test/kotlin/org/pkl/cli/CliProjectResolverTest.kt @@ -15,6 +15,7 @@ */ package org.pkl.cli +import java.io.File import java.io.StringWriter import java.nio.file.Path import org.assertj.core.api.Assertions.assertThat @@ -26,6 +27,7 @@ import org.pkl.commons.cli.CliBaseOptions import org.pkl.commons.cli.CliException import org.pkl.commons.test.FileTestUtils import org.pkl.commons.test.PackageServer +import org.pkl.core.util.IoUtils class CliProjectResolverTest { companion object { @@ -354,7 +356,7 @@ class CliProjectResolverTest { ) assertThat(errOut.toString()) .isEqualTo( - "WARN: local dependency `package://localhost:0/fruit@1.0.0` was overridden to remote dependency `package://localhost:0/fruit@1.0.5`.\n" + "WARN: local dependency `package://localhost:0/fruit@1.0.0` was overridden to remote dependency `package://localhost:0/fruit@1.0.5`.${IoUtils.getLineSeparator()}" ) } @@ -401,11 +403,12 @@ class CliProjectResolverTest { errWriter = errOut ) .run() + val sep = File.separatorChar assertThat(consoleOut.toString()) - .isEqualTo( + .isEqualToNormalizingNewlines( """ - $tempDir/project1/PklProject.deps.json - $tempDir/project2/PklProject.deps.json + $tempDir${sep}project1${sep}PklProject.deps.json + $tempDir${sep}project2${sep}PklProject.deps.json """ .trimIndent() diff --git a/pkl-commons-cli/src/main/kotlin/org/pkl/commons/cli/commands/BaseCommand.kt b/pkl-commons-cli/src/main/kotlin/org/pkl/commons/cli/commands/BaseCommand.kt index f5ae90b3..71527552 100644 --- a/pkl-commons-cli/src/main/kotlin/org/pkl/commons/cli/commands/BaseCommand.kt +++ b/pkl-commons-cli/src/main/kotlin/org/pkl/commons/cli/commands/BaseCommand.kt @@ -17,11 +17,6 @@ package org.pkl.commons.cli.commands import com.github.ajalt.clikt.core.CliktCommand import com.github.ajalt.clikt.parameters.groups.provideDelegate -import java.net.URI -import java.net.URISyntaxException -import org.pkl.commons.cli.CliException -import org.pkl.core.runtime.VmUtils -import org.pkl.core.util.IoUtils abstract class BaseCommand(name: String, helpLink: String, help: String = "") : CliktCommand( @@ -30,30 +25,4 @@ abstract class BaseCommand(name: String, helpLink: String, help: String = "") : epilog = "For more information, visit $helpLink", ) { val baseOptions by BaseOptions() - - /** - * Parses [moduleName] into a URI. If scheme is not present, we expect that this is a file path - * and encode any possibly invalid characters. If a scheme is present, we expect that this is a - * valid URI. - */ - protected fun parseModuleName(moduleName: String): URI = - when (moduleName) { - "-" -> VmUtils.REPL_TEXT_URI - else -> - try { - IoUtils.toUri(moduleName) - } catch (e: URISyntaxException) { - val message = buildString { - append("Module URI `$moduleName` has invalid syntax (${e.reason}).") - if (e.index > -1) { - append("\n\n") - append(moduleName) - append("\n") - append(" ".repeat(e.index)) - append("^") - } - } - throw CliException(message) - } - } } diff --git a/pkl-commons-cli/src/main/kotlin/org/pkl/commons/cli/commands/BaseOptions.kt b/pkl-commons-cli/src/main/kotlin/org/pkl/commons/cli/commands/BaseOptions.kt index de65e545..038d76d7 100644 --- a/pkl-commons-cli/src/main/kotlin/org/pkl/commons/cli/commands/BaseOptions.kt +++ b/pkl-commons-cli/src/main/kotlin/org/pkl/commons/cli/commands/BaseOptions.kt @@ -22,14 +22,50 @@ import com.github.ajalt.clikt.parameters.types.long import com.github.ajalt.clikt.parameters.types.path import java.io.File import java.net.URI +import java.net.URISyntaxException import java.nio.file.Path import java.time.Duration import java.util.regex.Pattern import org.pkl.commons.cli.CliBaseOptions +import org.pkl.commons.cli.CliException +import org.pkl.core.runtime.VmUtils import org.pkl.core.util.IoUtils @Suppress("MemberVisibilityCanBePrivate") class BaseOptions : OptionGroup() { + companion object { + /** + * Parses [moduleName] into a URI. If scheme is not present, we expect that this is a file path + * and encode any possibly invalid characters, and also normalize directory separators. If a + * scheme is present, we expect that this is a valid URI. + */ + fun parseModuleName(moduleName: String): URI = + when (moduleName) { + "-" -> VmUtils.REPL_TEXT_URI + else -> + // Don't use `IoUtils.toUri` here becaus we need to normalize `\` paths to `/` on Windows. + try { + if (IoUtils.isUriLike(moduleName)) URI(moduleName) + // Can't just use URI constructor, because URI(null, null, "C:/foo/bar", null) turns + // into `URI("C", null, "/foo/bar", null)`. + else if (IoUtils.isWindowsAbsolutePath(moduleName)) Path.of(moduleName).toUri() + else URI(null, null, IoUtils.toNormalizedPathString(Path.of(moduleName)), null) + } catch (e: URISyntaxException) { + val message = buildString { + append("Module URI `$moduleName` has invalid syntax (${e.reason}).") + if (e.index > -1) { + append("\n\n") + append(moduleName) + append("\n") + append(" ".repeat(e.index)) + append("^") + } + } + throw CliException(message) + } + } + } + private val defaults = CliBaseOptions() private val output = @@ -114,7 +150,7 @@ class BaseOptions : OptionGroup() { val settings: URI? by option(names = arrayOf("--settings"), help = "Pkl settings module to use.").single().convert { - IoUtils.toUri(it) + parseModuleName(it) } val timeout: Duration? by diff --git a/pkl-commons-cli/src/main/kotlin/org/pkl/commons/cli/commands/ModulesCommand.kt b/pkl-commons-cli/src/main/kotlin/org/pkl/commons/cli/commands/ModulesCommand.kt index c19fe58c..b2c45c05 100644 --- a/pkl-commons-cli/src/main/kotlin/org/pkl/commons/cli/commands/ModulesCommand.kt +++ b/pkl-commons-cli/src/main/kotlin/org/pkl/commons/cli/commands/ModulesCommand.kt @@ -29,7 +29,7 @@ abstract class ModulesCommand(name: String, helpLink: String, help: String = "") ) { open val modules: List by argument(name = "", help = "Module paths or URIs to evaluate.") - .convert { parseModuleName(it) } + .convert { BaseOptions.parseModuleName(it) } .multiple(required = true) protected val projectOptions by ProjectOptions() diff --git a/pkl-commons-test/pkl-commons-test.gradle.kts b/pkl-commons-test/pkl-commons-test.gradle.kts index 80a5394b..fc3e40b0 100644 --- a/pkl-commons-test/pkl-commons-test.gradle.kts +++ b/pkl-commons-test/pkl-commons-test.gradle.kts @@ -58,6 +58,11 @@ for (packageDir in file("src/main/files/packages").listFiles()!!) { } doLast { val outputFile = destinationDir.get().asFile.resolve("${packageDir.name}.json") + if (buildInfo.os.isWindows) { + val contents = outputFile.readText() + // workaround for https://github.com/gradle/gradle/issues/1151 + outputFile.writeText(contents.replace("\r\n", "\n")) + } shasumFile.get().asFile.writeText(outputFile.computeChecksum()) } } diff --git a/pkl-commons-test/src/main/kotlin/org/pkl/commons/test/InputOutputTestEngine.kt b/pkl-commons-test/src/main/kotlin/org/pkl/commons/test/InputOutputTestEngine.kt index 9366a500..4c7a864d 100644 --- a/pkl-commons-test/src/main/kotlin/org/pkl/commons/test/InputOutputTestEngine.kt +++ b/pkl-commons-test/src/main/kotlin/org/pkl/commons/test/InputOutputTestEngine.kt @@ -31,6 +31,7 @@ import org.junit.platform.engine.support.hierarchical.EngineExecutionContext import org.junit.platform.engine.support.hierarchical.HierarchicalTestEngine import org.junit.platform.engine.support.hierarchical.Node import org.junit.platform.engine.support.hierarchical.Node.DynamicTestExecutor +import org.pkl.commons.toNormalizedPathString abstract class InputOutputTestEngine : HierarchicalTestEngine() { @@ -106,7 +107,7 @@ abstract class InputOutputTestEngine : ): TestDescriptor { dirNode.inputDir.useDirectoryEntries { children -> for (child in children) { - val testPath = child.toString() + val testPath = child.toNormalizedPathString() val testName = child.fileName.toString() if (child.isRegularFile()) { if ( diff --git a/pkl-commons/src/main/kotlin/org/pkl/commons/Paths.kt b/pkl-commons/src/main/kotlin/org/pkl/commons/Paths.kt index f2c0723a..8f95c922 100644 --- a/pkl-commons/src/main/kotlin/org/pkl/commons/Paths.kt +++ b/pkl-commons/src/main/kotlin/org/pkl/commons/Paths.kt @@ -71,3 +71,13 @@ fun Path.deleteRecursively() { walk().use { paths -> paths.sorted(Comparator.reverseOrder()).forEach { it.deleteIfExists() } } } } + +private val isWindows by lazy { System.getProperty("os.name").contains("Windows") } + +/** Copy implementation from IoUtils.toNormalizedPathString */ +fun Path.toNormalizedPathString(): String { + if (isWindows) { + return toString().replace("\\", "/") + } + return toString() +} diff --git a/pkl-commons/src/main/kotlin/org/pkl/commons/Strings.kt b/pkl-commons/src/main/kotlin/org/pkl/commons/Strings.kt index cfe684d6..9adc4b0c 100644 --- a/pkl-commons/src/main/kotlin/org/pkl/commons/Strings.kt +++ b/pkl-commons/src/main/kotlin/org/pkl/commons/Strings.kt @@ -15,15 +15,24 @@ */ package org.pkl.commons +import java.io.File import java.net.URI import java.nio.file.Path +import java.util.regex.Pattern fun String.toPath(): Path = Path.of(this) +private val uriLike = Pattern.compile("\\w+:[^\\\\].*") + +private val windowsPathLike = Pattern.compile("\\w:\\\\.*") + /** Copy of org.pkl.core.util.IoUtils.toUri */ -fun String.toUri(): URI = - if (contains(":")) { - URI(this) - } else { - URI(null, null, this, null) +fun String.toUri(): URI { + if (uriLike.matcher(this).matches()) { + return URI(this) } + if (windowsPathLike.matcher(this).matches()) { + return File(this).toURI() + } + return URI(null, null, this, null) +} diff --git a/pkl-core/pkl-core.gradle.kts b/pkl-core/pkl-core.gradle.kts index 826cebe8..12b6425b 100644 --- a/pkl-core/pkl-core.gradle.kts +++ b/pkl-core/pkl-core.gradle.kts @@ -267,6 +267,21 @@ val testAlpineExecutableAmd64 by tasks.registering(Test::class) { } } +val testWindowsExecutableAmd64 by tasks.registering(Test::class) { + dependsOn(":pkl-cli:windowsExecutableAmd64") + + inputs.dir("src/test/files/LanguageSnippetTests/input") + inputs.dir("src/test/files/LanguageSnippetTests/input-helper") + inputs.dir("src/test/files/LanguageSnippetTests/output") + + testClassesDirs = files(tasks.test.get().testClassesDirs) + classpath = tasks.test.get().classpath + + useJUnitPlatform { + includeEngines("WindowsLanguageSnippetTestsEngine") + } +} + tasks.testNative { when { buildInfo.os.isMacOsX -> { @@ -284,6 +299,9 @@ tasks.testNative { dependsOn(testAlpineExecutableAmd64) } } + buildInfo.os.isWindows -> { + dependsOn(testWindowsExecutableAmd64) + } } } diff --git a/pkl-core/src/main/java/org/pkl/core/Platform.java b/pkl-core/src/main/java/org/pkl/core/Platform.java index 38bd0dae..30eb3309 100644 --- a/pkl-core/src/main/java/org/pkl/core/Platform.java +++ b/pkl-core/src/main/java/org/pkl/core/Platform.java @@ -31,6 +31,7 @@ public final class Platform { var pklVersion = Release.current().version().toString(); var osName = System.getProperty("os.name"); if (osName.equals("Mac OS X")) osName = "macOS"; + if (osName.contains("Windows")) osName = "Windows"; var osVersion = System.getProperty("os.version"); var architecture = System.getProperty("os.arch"); diff --git a/pkl-core/src/main/java/org/pkl/core/Release.java b/pkl-core/src/main/java/org/pkl/core/Release.java index ede7c53f..4a479ac1 100644 --- a/pkl-core/src/main/java/org/pkl/core/Release.java +++ b/pkl-core/src/main/java/org/pkl/core/Release.java @@ -49,6 +49,7 @@ public final class Release { var commitId = properties.getProperty("commitId"); var osName = System.getProperty("os.name"); if (osName.equals("Mac OS X")) osName = "macOS"; + if (osName.contains("Windows")) osName = "Windows"; var osVersion = System.getProperty("os.version"); var os = osName + " " + osVersion; var flavor = TruffleOptions.AOT ? "native" : "Java " + System.getProperty("java.version"); diff --git a/pkl-core/src/main/java/org/pkl/core/ast/builder/AstBuilder.java b/pkl-core/src/main/java/org/pkl/core/ast/builder/AstBuilder.java index c4d15a7e..bf44cf44 100644 --- a/pkl-core/src/main/java/org/pkl/core/ast/builder/AstBuilder.java +++ b/pkl-core/src/main/java/org/pkl/core/ast/builder/AstBuilder.java @@ -1794,10 +1794,17 @@ public final class AstBuilder extends AbstractAstBuilder { try { resolvedUri = IoUtils.resolve(context.getSecurityManager(), moduleKey, parsedUri); } catch (FileNotFoundException e) { - throw exceptionBuilder() - .evalError("cannotFindModule", importUri) - .withSourceSection(createSourceSection(importUriCtx)) - .build(); + + var exceptionBuilder = + exceptionBuilder() + .evalError("cannotFindModule", importUri) + .withSourceSection(createSourceSection(importUriCtx)); + var path = parsedUri.getPath(); + if (path != null && path.contains("\\")) { + exceptionBuilder.withHint( + "To resolve modules in nested directories, use `/` as the directory separator."); + } + throw exceptionBuilder.build(); } catch (URISyntaxException e) { throw exceptionBuilder() .evalError("invalidModuleUri", importUri) diff --git a/pkl-core/src/main/java/org/pkl/core/module/ModuleKeyFactories.java b/pkl-core/src/main/java/org/pkl/core/module/ModuleKeyFactories.java index d02823b1..f26816ef 100644 --- a/pkl-core/src/main/java/org/pkl/core/module/ModuleKeyFactories.java +++ b/pkl-core/src/main/java/org/pkl/core/module/ModuleKeyFactories.java @@ -18,8 +18,8 @@ package org.pkl.core.module; import java.net.URI; import java.net.URISyntaxException; import java.nio.file.FileSystemNotFoundException; -import java.nio.file.Path; import java.nio.file.Paths; +import java.nio.file.spi.FileSystemProvider; import java.util.ArrayList; import java.util.Collections; import java.util.List; @@ -141,15 +141,20 @@ public final class ModuleKeyFactories { private static class File implements ModuleKeyFactory { @Override public Optional create(URI uri) { - Path path; - try { - path = Path.of(uri); - } catch (FileSystemNotFoundException | IllegalArgumentException e) { - // none of the installed file system providers can handle this URI + // skip loading providers if the scheme is `file`. + if (uri.getScheme().equalsIgnoreCase("file")) { + return Optional.of(ModuleKeys.file(uri)); + } + // don't handle jar-file URIs (these are handled by GenericUrl). + if (uri.getScheme().equalsIgnoreCase("jar")) { return Optional.empty(); } - - return Optional.of(ModuleKeys.file(uri, path)); + for (FileSystemProvider provider : FileSystemProvider.installedProviders()) { + if (provider.getScheme().equalsIgnoreCase(uri.getScheme())) { + return Optional.of(ModuleKeys.file(uri)); + } + } + return Optional.empty(); } } diff --git a/pkl-core/src/main/java/org/pkl/core/module/ModuleKeys.java b/pkl-core/src/main/java/org/pkl/core/module/ModuleKeys.java index 48f0ddbd..60f6cb8c 100644 --- a/pkl-core/src/main/java/org/pkl/core/module/ModuleKeys.java +++ b/pkl-core/src/main/java/org/pkl/core/module/ModuleKeys.java @@ -16,9 +16,11 @@ package org.pkl.core.module; import com.oracle.truffle.api.CompilerDirectives.TruffleBoundary; +import java.io.File; import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStream; +import java.net.JarURLConnection; import java.net.URI; import java.net.URISyntaxException; import java.net.http.HttpRequest; @@ -88,8 +90,8 @@ public final class ModuleKeys { } /** Creates a module key for a {@code file:} module. */ - public static ModuleKey file(URI uri, Path path) { - return new File(uri, path); + public static ModuleKey file(URI uri) { + return new File(uri); } /** @@ -290,12 +292,10 @@ public final class ModuleKeys { private static class File extends DependencyAwareModuleKey { final URI uri; - final Path path; - File(URI uri, Path path) { + File(URI uri) { super(uri); this.uri = uri; - this.path = path; } @Override @@ -316,7 +316,13 @@ public final class ModuleKeys { public ResolvedModuleKey resolve(SecurityManager securityManager) throws IOException, SecurityManagerException { securityManager.checkResolveModule(uri); - var realPath = path.toRealPath(); + // Disallow paths that contain `\\` characters if on Windows + // (require `/` as the path separator on all OSes) + var uriPath = uri.getPath(); + if (java.io.File.separatorChar == '\\' && uriPath != null && uriPath.contains("\\")) { + throw new FileNotFoundException(); + } + var realPath = Path.of(uri).toRealPath(); var resolvedUri = realPath.toUri(); securityManager.checkResolveModule(resolvedUri); return ResolvedModuleKeys.file(this, resolvedUri, realPath); @@ -325,7 +331,7 @@ public final class ModuleKeys { @Override protected Map getDependencies() { var projectDepsManager = VmContext.get(null).getProjectDependenciesManager(); - if (projectDepsManager == null || !projectDepsManager.hasPath(path)) { + if (projectDepsManager == null || !projectDepsManager.hasPath(Path.of(uri))) { throw new PackageLoadError("cannotResolveDependencyNoProject"); } return projectDepsManager.getDependencies(); @@ -519,6 +525,12 @@ public final class ModuleKeys { var url = IoUtils.toUrl(uri); var conn = url.openConnection(); conn.connect(); + if (conn instanceof JarURLConnection && IoUtils.isWindows()) { + // On Windows, opening a JarURLConnection prevents the jar file from being deleted, unless + // cacheing is disabled. + // See https://bugs.openjdk.org/browse/JDK-8239054 + conn.setUseCaches(false); + } try (InputStream stream = conn.getInputStream()) { URI redirected; try { diff --git a/pkl-core/src/main/java/org/pkl/core/module/ModulePathResolver.java b/pkl-core/src/main/java/org/pkl/core/module/ModulePathResolver.java index 47a0eb77..d2265c5d 100644 --- a/pkl-core/src/main/java/org/pkl/core/module/ModulePathResolver.java +++ b/pkl-core/src/main/java/org/pkl/core/module/ModulePathResolver.java @@ -30,6 +30,7 @@ import java.util.Map; import javax.annotation.concurrent.GuardedBy; import org.pkl.core.module.PathElement.TreePathElement; import org.pkl.core.runtime.FileSystemManager; +import org.pkl.core.util.IoUtils; import org.pkl.core.util.LateInit; /** @@ -152,8 +153,8 @@ public final class ModulePathResolver implements AutoCloseable { // in case of duplicate path, first entry wins (cf. class loader) stream.forEach( (path) -> { - var relativized = basePath.relativize(path); - fileCache.putIfAbsent(relativized.toString(), path); + var relativized = IoUtils.relativize(path, basePath); + fileCache.putIfAbsent(IoUtils.toNormalizedPathString(relativized), path); var element = cachedPathElementRoot; for (var i = 0; i < relativized.getNameCount(); i++) { var name = relativized.getName(i).toString(); diff --git a/pkl-core/src/main/java/org/pkl/core/module/ProjectDependenciesManager.java b/pkl-core/src/main/java/org/pkl/core/module/ProjectDependenciesManager.java index b07ee29d..633ee277 100644 --- a/pkl-core/src/main/java/org/pkl/core/module/ProjectDependenciesManager.java +++ b/pkl-core/src/main/java/org/pkl/core/module/ProjectDependenciesManager.java @@ -207,13 +207,15 @@ public final class ProjectDependenciesManager { if (projectDeps == null) { var depsPath = getProjectDepsFile(); if (!Files.exists(depsPath)) { - throw new VmExceptionBuilder().evalError("missingProjectDepsJson", projectDir).build(); + throw new VmExceptionBuilder() + .evalError("missingProjectDepsJson", projectDir.toUri()) + .build(); } try { projectDeps = ProjectDeps.parse(depsPath); } catch (IOException | URISyntaxException | JsonParseException e) { throw new VmExceptionBuilder() - .evalError("invalidProjectDepsJson", depsPath, e.getMessage()) + .evalError("invalidProjectDepsJson", depsPath.toUri(), e.getMessage()) .withCause(e) .build(); } diff --git a/pkl-core/src/main/java/org/pkl/core/module/ResolvedModuleKeys.java b/pkl-core/src/main/java/org/pkl/core/module/ResolvedModuleKeys.java index 647bd9c3..8fcaaa96 100644 --- a/pkl-core/src/main/java/org/pkl/core/module/ResolvedModuleKeys.java +++ b/pkl-core/src/main/java/org/pkl/core/module/ResolvedModuleKeys.java @@ -15,10 +15,12 @@ */ package org.pkl.core.module; +import java.io.File; import java.io.IOException; import java.net.URI; import java.net.URL; import java.nio.charset.StandardCharsets; +import java.nio.file.AccessDeniedException; import java.nio.file.Files; import java.nio.file.Path; import org.pkl.core.util.IoUtils; @@ -75,7 +77,16 @@ public final class ResolvedModuleKeys { @Override public String loadSource() throws IOException { - return Files.readString(path, StandardCharsets.UTF_8); + try { + return Files.readString(path, StandardCharsets.UTF_8); + } catch (AccessDeniedException e) { + // Windows throws `AccessDeniedException` when reading directories. + // Sync error between different OSes. + if (Files.isDirectory(path)) { + throw new IOException("Is a directory"); + } + throw e; + } } } diff --git a/pkl-core/src/main/java/org/pkl/core/packages/Dependency.java b/pkl-core/src/main/java/org/pkl/core/packages/Dependency.java index 2fe41448..991462a9 100644 --- a/pkl-core/src/main/java/org/pkl/core/packages/Dependency.java +++ b/pkl-core/src/main/java/org/pkl/core/packages/Dependency.java @@ -50,7 +50,7 @@ public abstract class Dependency { public Path resolveAssetPath(Path projectDir, PackageAssetUri packageAssetUri) { // drop 1 to remove leading `/` - var assetPath = packageAssetUri.getAssetPath().toString().substring(1); + var assetPath = packageAssetUri.getAssetPath().substring(1); return projectDir.resolve(path).resolve(assetPath); } diff --git a/pkl-core/src/main/java/org/pkl/core/packages/PackageAssetUri.java b/pkl-core/src/main/java/org/pkl/core/packages/PackageAssetUri.java index 6e52146b..ed6ce589 100644 --- a/pkl-core/src/main/java/org/pkl/core/packages/PackageAssetUri.java +++ b/pkl-core/src/main/java/org/pkl/core/packages/PackageAssetUri.java @@ -20,6 +20,7 @@ import java.net.URISyntaxException; import java.nio.file.Path; import org.pkl.core.Version; import org.pkl.core.util.ErrorMessages; +import org.pkl.core.util.IoUtils; /** * The canonical URI of an asset within a package, i.e., a package URI with a fragment path. For @@ -28,7 +29,7 @@ import org.pkl.core.util.ErrorMessages; public final class PackageAssetUri { private final URI uri; private final PackageUri packageUri; - private final Path assetPath; + private final String assetPath; public static PackageAssetUri create(URI uri) { try { @@ -41,7 +42,7 @@ public final class PackageAssetUri { public PackageAssetUri(PackageUri packageUri, String assetPath) { this.uri = packageUri.getUri().resolve("#" + assetPath); this.packageUri = packageUri; - this.assetPath = Path.of(assetPath); + this.assetPath = assetPath; } public PackageAssetUri(String uri) throws URISyntaxException { @@ -60,7 +61,7 @@ public final class PackageAssetUri { throw new URISyntaxException( uri.toString(), ErrorMessages.create("cannotHaveRelativeFragment", fragment, uri)); } - this.assetPath = Path.of(fragment); + this.assetPath = fragment; } public URI getUri() { @@ -71,7 +72,7 @@ public final class PackageAssetUri { return packageUri; } - public Path getAssetPath() { + public String getAssetPath() { return assetPath; } @@ -102,6 +103,7 @@ public final class PackageAssetUri { } public PackageAssetUri resolve(String path) { - return new PackageAssetUri(packageUri, assetPath.resolve(path).toString()); + var resolvedPath = IoUtils.toNormalizedPathString(Path.of(assetPath).resolve(path)); + return new PackageAssetUri(packageUri, resolvedPath); } } diff --git a/pkl-core/src/main/java/org/pkl/core/packages/PackageResolvers.java b/pkl-core/src/main/java/org/pkl/core/packages/PackageResolvers.java index dbc3fd49..fbf3d085 100644 --- a/pkl-core/src/main/java/org/pkl/core/packages/PackageResolvers.java +++ b/pkl-core/src/main/java/org/pkl/core/packages/PackageResolvers.java @@ -335,8 +335,7 @@ final class PackageResolvers { var entries = cachedEntries.get(packageUri); // need to normalize here but not in `doListElments` nor `doHasElement` because // `TreePathElement.getElement` does normalization already. - var path = uri.getAssetPath().normalize().toString(); - assert path.startsWith("/"); + var path = IoUtils.toNormalizedPathString(Path.of(uri.getAssetPath()).normalize()); return entries.get(path).array(); } @@ -496,7 +495,9 @@ final class PackageResolvers { downloadMetadata(packageUri, requestUri, tmpPath, checksums); Files.createDirectories(cachePath.getParent()); Files.move(tmpPath, cachePath, StandardCopyOption.ATOMIC_MOVE); - Files.setPosixFilePermissions(cachePath, FILE_PERMISSIONS); + if (!IoUtils.isWindows()) { + Files.setPosixFilePermissions(cachePath, FILE_PERMISSIONS); + } return cachePath; } finally { Files.deleteIfExists(tmpPath); @@ -545,7 +546,9 @@ final class PackageResolvers { verifyPackageZipBytes(packageUri, dependencyMetadata, checksumBytes); Files.createDirectories(cachePath.getParent()); Files.move(tmpPath, cachePath, StandardCopyOption.ATOMIC_MOVE); - Files.setPosixFilePermissions(cachePath, FILE_PERMISSIONS); + if (!IoUtils.isWindows()) { + Files.setPosixFilePermissions(cachePath, FILE_PERMISSIONS); + } return cachePath; } finally { Files.deleteIfExists(tmpPath); diff --git a/pkl-core/src/main/java/org/pkl/core/project/ProjectDependenciesResolver.java b/pkl-core/src/main/java/org/pkl/core/project/ProjectDependenciesResolver.java index d37f3c7f..f528f84b 100644 --- a/pkl-core/src/main/java/org/pkl/core/project/ProjectDependenciesResolver.java +++ b/pkl-core/src/main/java/org/pkl/core/project/ProjectDependenciesResolver.java @@ -33,6 +33,7 @@ import org.pkl.core.packages.PackageUri; import org.pkl.core.util.EconomicMaps; import org.pkl.core.util.EconomicSets; import org.pkl.core.util.ErrorMessages; +import org.pkl.core.util.IoUtils; import org.pkl.core.util.Nullable; /** @@ -78,7 +79,7 @@ public final class ProjectDependenciesResolver { private void log(String message) { try { - logWriter.write(message + "\n"); + logWriter.write(message + IoUtils.getLineSeparator()); } catch (IOException e) { throw new UncheckedIOException(e); } @@ -130,7 +131,7 @@ public final class ProjectDependenciesResolver { var packageUri = declaredDependencies.getMyPackageUri(); assert packageUri != null; var projectDir = Path.of(declaredDependencies.getProjectFileUri()).getParent(); - var relativePath = this.project.getProjectDir().relativize(projectDir); + var relativePath = IoUtils.relativize(projectDir, this.project.getProjectDir()); var localDependency = new LocalDependency(packageUri.toProjectPackageUri(), relativePath); updateDependency(localDependency); buildResolvedDependencies(declaredDependencies); diff --git a/pkl-core/src/main/java/org/pkl/core/project/ProjectDeps.java b/pkl-core/src/main/java/org/pkl/core/project/ProjectDeps.java index d5b61b59..be18cac8 100644 --- a/pkl-core/src/main/java/org/pkl/core/project/ProjectDeps.java +++ b/pkl-core/src/main/java/org/pkl/core/project/ProjectDeps.java @@ -35,6 +35,7 @@ import org.pkl.core.packages.PackageLoadError; import org.pkl.core.packages.PackageUtils; import org.pkl.core.runtime.VmExceptionBuilder; import org.pkl.core.util.EconomicMaps; +import org.pkl.core.util.IoUtils; import org.pkl.core.util.Nullable; import org.pkl.core.util.json.Json; import org.pkl.core.util.json.Json.FormatException; @@ -196,7 +197,7 @@ public final class ProjectDeps { jsonWriter.beginObject(); jsonWriter.name("type").value("local"); jsonWriter.name("uri").value(localDependency.getPackageUri().toString()); - jsonWriter.name("path").value(localDependency.getPath().toString()); + jsonWriter.name("path").value(IoUtils.toNormalizedPathString(localDependency.getPath())); jsonWriter.endObject(); } diff --git a/pkl-core/src/main/java/org/pkl/core/project/ProjectPackager.java b/pkl-core/src/main/java/org/pkl/core/project/ProjectPackager.java index f1bffab3..28492eae 100644 --- a/pkl-core/src/main/java/org/pkl/core/project/ProjectPackager.java +++ b/pkl-core/src/main/java/org/pkl/core/project/ProjectPackager.java @@ -20,6 +20,8 @@ import java.io.IOException; import java.io.OutputStream; import java.io.UncheckedIOException; import java.io.Writer; +import java.net.URI; +import java.net.URISyntaxException; import java.nio.file.Files; import java.nio.file.Path; import java.security.DigestOutputStream; @@ -119,13 +121,18 @@ public final class ProjectPackager { this.outputWriter = outputWriter; } + private void writeLine(String line) throws IOException { + outputWriter.write(line); + outputWriter.write(IoUtils.getLineSeparator()); + } + public void createPackages() throws IOException { for (var project : projects) { var packageResult = doPackage(project); - outputWriter.write(workingDir.relativize(packageResult.getMetadataFile()) + "\n"); - outputWriter.write(workingDir.relativize(packageResult.getMetadataChecksumFile()) + "\n"); - outputWriter.write(workingDir.relativize(packageResult.getZipFile()) + "\n"); - outputWriter.write(workingDir.relativize(packageResult.getZipChecksumFile()) + "\n"); + writeLine(IoUtils.relativize(packageResult.getMetadataFile(), workingDir).toString()); + writeLine(IoUtils.relativize(packageResult.getMetadataChecksumFile(), workingDir).toString()); + writeLine(IoUtils.relativize(packageResult.getZipFile(), workingDir).toString()); + writeLine(IoUtils.relativize(packageResult.getZipChecksumFile(), workingDir).toString()); outputWriter.flush(); } } @@ -302,8 +309,8 @@ public final class ProjectPackager { } try (var zos = new ZipOutputStream(digestOutputStream)) { for (var file : files) { - var relativePath = project.getProjectDir().relativize(file); - var zipEntry = new ZipEntry(relativePath.toString()); + var relativePath = IoUtils.relativize(file, project.getProjectDir()); + var zipEntry = new ZipEntry(IoUtils.toNormalizedPathString(relativePath)); zipEntry.setTimeLocal(ZIP_ENTRY_MTIME); zos.putNextEntry(zipEntry); Files.copy(file, zos); @@ -342,8 +349,8 @@ public final class ProjectPackager { .filter(Files::isRegularFile) .filter( (it) -> { - var fileNameRelativeToProjectRoot = - project.getProjectDir().relativize(it).toString(); + var relativePath = IoUtils.relativize(it, project.getProjectDir()); + var fileNameRelativeToProjectRoot = IoUtils.toNormalizedPathString(relativePath); for (var pattern : excludePatterns) { if (pattern.matcher(it.getFileName().toString()).matches()) { return false; @@ -363,7 +370,7 @@ public final class ProjectPackager { } private boolean isAbsoluteImport(String importStr) { - return importStr.matches("\\w:.*") || importStr.startsWith("@"); + return importStr.matches("\\w+:.*") || importStr.startsWith("@"); } /** @@ -386,8 +393,17 @@ public final class ProjectPackager { if (isAbsoluteImport(importStr)) { continue; } - var importPath = Path.of(importStr); - if (importPath.isAbsolute() && !project.getProjectDir().toString().equals("/")) { + URI importUri; + try { + importUri = IoUtils.toUri(importStr); + } catch (URISyntaxException e) { + throw new VmExceptionBuilder() + .evalError("invalidModuleUri", importStr) + .withSourceSection(sourceSection) + .build() + .toPklException(stackFrameTransformer); + } + if (importStr.startsWith("/") && !project.getProjectDir().toString().equals("/")) { throw new VmExceptionBuilder() .evalError("invalidRelativeProjectImport", importStr) .withSourceSection(sourceSection) @@ -395,6 +411,7 @@ public final class ProjectPackager { .toPklException(stackFrameTransformer); } var currentPath = pklModulePath.getParent(); + var importPath = Path.of(importUri.getPath()); // It's not good enough to just check the normalized path to see whether it exists within the // root dir. // It's possible that the import path resolves to a path outside the project dir, @@ -416,7 +433,7 @@ public final class ProjectPackager { private @Nullable List> getImportsAndReads(Path pklModulePath) { try { - var moduleKey = ModuleKeys.file(pklModulePath.toUri(), pklModulePath); + var moduleKey = ModuleKeys.file(pklModulePath.toUri()); var resolvedModuleKey = ResolvedModuleKeys.file(moduleKey, moduleKey.getUri(), pklModulePath); return ImportsAndReadsParser.parse(moduleKey, resolvedModuleKey); } catch (IOException e) { diff --git a/pkl-core/src/main/java/org/pkl/core/runtime/ModuleCache.java b/pkl-core/src/main/java/org/pkl/core/runtime/ModuleCache.java index f01f350f..5558246b 100644 --- a/pkl-core/src/main/java/org/pkl/core/runtime/ModuleCache.java +++ b/pkl-core/src/main/java/org/pkl/core/runtime/ModuleCache.java @@ -195,10 +195,16 @@ public final class ModuleCache { } catch (SecurityManagerException | PackageLoadError e) { throw new VmExceptionBuilder().withOptionalLocation(importNode).withCause(e).build(); } catch (FileNotFoundException | NoSuchFileException e) { - throw new VmExceptionBuilder() - .withOptionalLocation(importNode) - .evalError("cannotFindModule", module.getUri()) - .build(); + var exceptionBuilder = + new VmExceptionBuilder() + .withOptionalLocation(importNode) + .evalError("cannotFindModule", module.getUri()); + var path = module.getUri().getPath(); + if (path != null && path.contains("\\")) { + exceptionBuilder.withHint( + "To resolve modules in nested directories, use `/` as the directory separator."); + } + throw exceptionBuilder.build(); } catch (IOException e) { throw new VmExceptionBuilder() .withOptionalLocation(importNode) diff --git a/pkl-core/src/main/java/org/pkl/core/util/IoUtils.java b/pkl-core/src/main/java/org/pkl/core/util/IoUtils.java index a5da3df2..02359057 100644 --- a/pkl-core/src/main/java/org/pkl/core/util/IoUtils.java +++ b/pkl-core/src/main/java/org/pkl/core/util/IoUtils.java @@ -35,6 +35,7 @@ import java.util.regex.Pattern; import java.util.zip.ZipEntry; import java.util.zip.ZipOutputStream; import org.pkl.core.PklBugException; +import org.pkl.core.Platform; import org.pkl.core.SecurityManager; import org.pkl.core.SecurityManagerException; import org.pkl.core.module.ModuleKey; @@ -43,7 +44,10 @@ import org.pkl.core.runtime.VmExceptionBuilder; public final class IoUtils { - private static final Pattern uriLike = Pattern.compile("\\w+:.*"); + // Don't match paths like `C:\`, which are drive letters on Windows. + private static final Pattern uriLike = Pattern.compile("\\w+:[^\\\\].*"); + + private static final Pattern windowsPathLike = Pattern.compile("\\w:\\\\.*"); private IoUtils() {} @@ -66,12 +70,20 @@ public final class IoUtils { return uriLike.matcher(str).matches(); } + public static boolean isWindowsAbsolutePath(String str) { + if (!isWindows()) return false; + return windowsPathLike.matcher(str).matches(); + } + /** * Converts the given string to a {@link URI}. This method MUST be used for constructing module * and resource URIs. Unlike {@code new URI(str)}, it correctly escapes paths of relative URIs. */ public static URI toUri(String str) throws URISyntaxException { - return isUriLike(str) ? new URI(str) : new URI(null, null, str, null); + if (isUriLike(str)) { + return new URI(str); + } + return new URI(null, null, str, null); } /** Like {@link #toUri(String)}, except without checked exceptions. */ @@ -150,7 +162,8 @@ public final class IoUtils { new SimpleFileVisitor<>() { public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { - zipStream.putNextEntry(new ZipEntry(sourceDir.relativize(file).toString())); + var relativePath = relativize(file, sourceDir); + zipStream.putNextEntry(new ZipEntry(toNormalizedPathString(relativePath))); Files.copy(file, zipStream); zipStream.closeEntry(); return FileVisitResult.CONTINUE; @@ -180,6 +193,10 @@ public final class IoUtils { return System.getProperty("line.separator"); } + public static Boolean isWindows() { + return Platform.current().operatingSystem().name().equals("Windows"); + } + public static String getName(String path) { var lastSep = Math.max(path.lastIndexOf('/'), path.lastIndexOf('\\')); return path.substring(lastSep + 1); @@ -362,7 +379,9 @@ public final class IoUtils { } } - // URI.relativize() won't construct relative paths containing ".." + // URI.relativize won't construct relative paths containing `..`. + // Can't use Path.relativize because certain URI characters will throw InvalidPathException + // on Windows. public static URI relativize(URI uri, URI base) { if (uri.isOpaque() || base.isOpaque() @@ -370,19 +389,60 @@ public final class IoUtils { || !Objects.equals(uri.getAuthority(), base.getAuthority())) { return uri; } - - var basePath = Path.of(base.getPath()); - if (!base.getRawPath().endsWith("/")) basePath = basePath.getParent(); - var resultPath = basePath.relativize(Path.of(uri.getPath())); - + var uriPath = uri.normalize().getPath(); + var basePath = base.normalize().getPath(); try { - return new URI( - null, null, null, -1, resultPath.toString(), uri.getQuery(), uri.getFragment()); + if (basePath.isEmpty()) { + return uri; + } + var uriParts = Arrays.asList(uriPath.split("/")); + var baseParts = Arrays.asList(basePath.split("/")); + if (!basePath.endsWith("/")) { + // strip the last path segment of the base uri, unless it ends in a slash. `/foo/bar.pkl` -> + // `/foo` + baseParts = baseParts.subList(0, baseParts.size() - 1); + } + if (uriParts.equals(baseParts)) { + return new URI(null, null, null, -1, "", uri.getQuery(), uri.getFragment()); + } + var start = 0; + while (start < Math.min(uriParts.size(), baseParts.size())) { + if (!uriParts.get(start).equals(baseParts.get(start))) { + break; + } + start++; + } + var uriPartsRemaining = uriParts.subList(start, uriParts.size()); + var basePartsRemainig = baseParts.subList(start, baseParts.size()); + if (basePartsRemainig.isEmpty()) { + return new URI( + null, + null, + null, + -1, + String.join("/", uriPartsRemaining), + uri.getQuery(), + uri.getFragment()); + } + var resultingPath = + "../".repeat(basePartsRemainig.size()) + String.join("/", uriPartsRemaining); + return new URI(null, null, null, -1, resultingPath, uri.getQuery(), uri.getFragment()); } catch (URISyntaxException e) { - throw new IllegalArgumentException(e); + // Impossible; started from a valid URI to begin with. + throw PklBugException.unreachableCode(); } } + // On Windows, `Path.relativize` will fail if the two paths have different roots. + public static Path relativize(Path path, Path base) { + if (isWindows()) { + if (path.isAbsolute() && base.isAbsolute() && !path.getRoot().equals(base.getRoot())) { + return path; + } + } + return base.relativize(path); + } + public static boolean isWhitespace(String str) { return str.codePoints().allMatch(Character::isWhitespace); } @@ -597,6 +657,63 @@ public final class IoUtils { return newUri; } + public static boolean isReservedFilenameChar(char character) { + if (isWindows()) { + return isReservedWindowsFilenameChar(character); + } + // posix; only NULL and `/` are reserved. + return character == 0 || character == '/'; + } + + /** Tells if this character cannot be used for filenames on Windows. */ + public static boolean isReservedWindowsFilenameChar(char character) { + return switch (character) { + case 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + '<', + '>', + ':', + '"', + '\\', + '/', + '|', + '?', + '*' -> + true; + default -> false; + }; + } + /** * Windows reserves characters {@code <>:"\|?*} in filenames. * @@ -608,19 +725,27 @@ public final class IoUtils { var sb = new StringBuilder(); for (var i = 0; i < path.length(); i++) { var character = path.charAt(i); - switch (character) { - case '<', '>', ':', '"', '\\', '|', '?', '*' -> { - sb.append('('); - sb.append(ByteArrayUtils.toHex(new byte[] {(byte) character})); - sb.append(")"); - } - case '(' -> sb.append("(("); - default -> sb.append(path.charAt(i)); + if (isReservedWindowsFilenameChar(character) && character != '/') { + sb.append('('); + sb.append(ByteArrayUtils.toHex(new byte[] {(byte) character})); + sb.append(")"); + } else if (character == '(') { + sb.append("(("); + } else { + sb.append(character); } } return sb.toString(); } + /** Returns a path string that uses unix-like path separators. */ + public static String toNormalizedPathString(Path path) { + if (isWindows()) { + return path.toString().replace("\\", "/"); + } + return path.toString(); + } + private static int getExclamationMarkIndex(String jarUri) { var index = jarUri.indexOf('!'); if (index == -1) { diff --git a/pkl-core/src/test/files/LanguageSnippetTests/input/api/releaseModule.pkl b/pkl-core/src/test/files/LanguageSnippetTests/input/api/releaseModule.pkl index 0ef1c390..8436992b 100644 --- a/pkl-core/src/test/files/LanguageSnippetTests/input/api/releaseModule.pkl +++ b/pkl-core/src/test/files/LanguageSnippetTests/input/api/releaseModule.pkl @@ -12,7 +12,7 @@ facts { } ["versionInfo"] { - current.versionInfo.contains("macOS") || current.versionInfo.contains("Linux") + current.versionInfo.contains("macOS") || current.versionInfo.contains("Linux") || current.versionInfo.contains("Windows") } ["commitId"] { diff --git a/pkl-core/src/test/files/LanguageSnippetTests/input/errors/invalidImportBackslashSep.pkl b/pkl-core/src/test/files/LanguageSnippetTests/input/errors/invalidImportBackslashSep.pkl new file mode 100644 index 00000000..82a3c4aa --- /dev/null +++ b/pkl-core/src/test/files/LanguageSnippetTests/input/errors/invalidImportBackslashSep.pkl @@ -0,0 +1,2 @@ +// In all OSes, the directory separator is forward slash. +res = import(#"..\basic\baseModule.pkl"#) diff --git a/pkl-core/src/test/files/LanguageSnippetTests/output/errors/invalidImportBackslashSep.err b/pkl-core/src/test/files/LanguageSnippetTests/output/errors/invalidImportBackslashSep.err new file mode 100644 index 00000000..e4d36b97 --- /dev/null +++ b/pkl-core/src/test/files/LanguageSnippetTests/output/errors/invalidImportBackslashSep.err @@ -0,0 +1,12 @@ +–– Pkl Error –– +Cannot find module `file:///$snippetsDir/input/errors/..%5Cbasic%5CbaseModule.pkl`. + +x | res = import(#"..\basic\baseModule.pkl"#) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +at invalidImportBackslashSep#res (file:///$snippetsDir/input/errors/invalidImportBackslashSep.pkl) + +To resolve modules in nested directories, use `/` as the directory separator. + +xxx | text = renderer.renderDocument(value) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +at pkl.base#Module.output.text (pkl:base) diff --git a/pkl-core/src/test/files/LanguageSnippetTests/output/projects/badProjectDeps1/bug.err b/pkl-core/src/test/files/LanguageSnippetTests/output/projects/badProjectDeps1/bug.err index fe03d702..2c21653e 100644 --- a/pkl-core/src/test/files/LanguageSnippetTests/output/projects/badProjectDeps1/bug.err +++ b/pkl-core/src/test/files/LanguageSnippetTests/output/projects/badProjectDeps1/bug.err @@ -1,5 +1,5 @@ –– Pkl Error –– -Cannot resolve dependency because file `/$snippetsDir/input/projects/badProjectDeps1/PklProject.deps.json` is malformed. +Cannot resolve dependency because file `file:///$snippetsDir/input/projects/badProjectDeps1/PklProject.deps.json` is malformed. Run `pkl project resolve` to re-create this file. x | import "@bird/Bird.pkl" diff --git a/pkl-core/src/test/files/LanguageSnippetTests/output/projects/badProjectDeps2/bug.err b/pkl-core/src/test/files/LanguageSnippetTests/output/projects/badProjectDeps2/bug.err index 3d43f741..d42b59b7 100644 --- a/pkl-core/src/test/files/LanguageSnippetTests/output/projects/badProjectDeps2/bug.err +++ b/pkl-core/src/test/files/LanguageSnippetTests/output/projects/badProjectDeps2/bug.err @@ -1,5 +1,5 @@ –– Pkl Error –– -Cannot resolve dependency because file `/$snippetsDir/input/projects/badProjectDeps2/PklProject.deps.json` is malformed. +Cannot resolve dependency because file `file:///$snippetsDir/input/projects/badProjectDeps2/PklProject.deps.json` is malformed. Run `pkl project resolve` to re-create this file. x | import "@bird/Bird.pkl" diff --git a/pkl-core/src/test/files/LanguageSnippetTests/output/projects/missingProjectDeps/bug.err b/pkl-core/src/test/files/LanguageSnippetTests/output/projects/missingProjectDeps/bug.err index f965f99b..679ecc06 100644 --- a/pkl-core/src/test/files/LanguageSnippetTests/output/projects/missingProjectDeps/bug.err +++ b/pkl-core/src/test/files/LanguageSnippetTests/output/projects/missingProjectDeps/bug.err @@ -1,5 +1,5 @@ –– Pkl Error –– -Cannot resolve dependency because file `PklProject.deps.json` is missing in project directory `/$snippetsDir/input/projects/missingProjectDeps`. +Cannot resolve dependency because file `PklProject.deps.json` is missing in project directory `file:///$snippetsDir/input/projects/missingProjectDeps/`. x | import "@birds/Bird.pkl" ^^^^^^^^^^^^^^^^^ diff --git a/pkl-core/src/test/kotlin/org/pkl/core/EvaluatorTest.kt b/pkl-core/src/test/kotlin/org/pkl/core/EvaluatorTest.kt index 0d6e1c4d..f0de9784 100644 --- a/pkl-core/src/test/kotlin/org/pkl/core/EvaluatorTest.kt +++ b/pkl-core/src/test/kotlin/org/pkl/core/EvaluatorTest.kt @@ -74,11 +74,12 @@ class EvaluatorTest { @Test fun `evaluate non-existing file`() { + val file = File("/non/existing") val e = assertThrows { - evaluator.evaluate(file(File("/non/existing"))) + evaluator.evaluate(file(file)) } assertThat(e) - .hasMessageContaining("Cannot find module `file:///non/existing`.") + .hasMessageContaining("Cannot find module `${file.toPath().toUri()}`.") } @Test @@ -92,13 +93,14 @@ class EvaluatorTest { @Test fun `evaluate non-existing path`() { + val path = "/non/existing".toPath() val e = assertThrows { - evaluator.evaluate(path("/non/existing".toPath())) + evaluator.evaluate(path(path)) } assertThat(e) - .hasMessageContaining("Cannot find module `file:///non/existing`.") + .hasMessageContaining("Cannot find module `${path.toUri()}`.") } - + @Test fun `evaluate zip file system path`(@TempDir tempDir: Path) { val zipFile = createModulesZip(tempDir) diff --git a/pkl-core/src/test/kotlin/org/pkl/core/LanguageSnippetTests.kt b/pkl-core/src/test/kotlin/org/pkl/core/LanguageSnippetTests.kt index 5e5ac8af..1d2c788e 100644 --- a/pkl-core/src/test/kotlin/org/pkl/core/LanguageSnippetTests.kt +++ b/pkl-core/src/test/kotlin/org/pkl/core/LanguageSnippetTests.kt @@ -13,3 +13,6 @@ class LinuxLanguageSnippetTests @Testable class AlpineLanguageSnippetTests + +@Testable +class WindowsLanguageSnippetTests diff --git a/pkl-core/src/test/kotlin/org/pkl/core/LanguageSnippetTestsEngine.kt b/pkl-core/src/test/kotlin/org/pkl/core/LanguageSnippetTestsEngine.kt index 516743b5..192bf0d7 100644 --- a/pkl-core/src/test/kotlin/org/pkl/core/LanguageSnippetTestsEngine.kt +++ b/pkl-core/src/test/kotlin/org/pkl/core/LanguageSnippetTestsEngine.kt @@ -51,7 +51,7 @@ abstract class AbstractLanguageSnippetTestsEngine : InputOutputTestEngine() { else parent?.getProjectDir() override fun expectedOutputFileFor(inputFile: Path): Path { - val relativePath = inputDir.relativize(inputFile).toString() + val relativePath = IoUtils.relativize(inputFile, inputDir).toString() val stdoutPath = if (relativePath.matches(hiddenExtensionRegex)) relativePath.dropLast(4) else relativePath.dropLast(3) + "pcf" @@ -62,12 +62,12 @@ abstract class AbstractLanguageSnippetTestsEngine : InputOutputTestEngine() { // disable SHA verification for packages IoUtils.setTestMode() } - + override fun afterAll() { packageServer.close() } - - protected fun String.stripFilePaths() = replace(snippetsDir.toString(), "/\$snippetsDir") + + protected fun String.stripFilePaths() = replace(snippetsDir.toUri().toString(), "file:///\$snippetsDir/") protected fun String.stripLineNumbers() = replace(lineNumberRegex) { result -> // replace line number with equivalent number of 'x' characters to keep formatting intact @@ -82,6 +82,11 @@ abstract class AbstractLanguageSnippetTestsEngine : InputOutputTestEngine() { protected fun String.stripStdlibLocationSha(): String = replace("https://github.com/apple/pkl/blob/${Release.current().commitId()}/stdlib/", "https://github.com/apple/pkl/blob/\$commitId/stdlib/") + + protected fun String.withUnixLineEndings(): String { + return if (System.lineSeparator() == "\r\n") replace("\r\n", "\n") + else this + } } class LanguageSnippetTestsEngine : AbstractLanguageSnippetTestsEngine() { @@ -143,7 +148,7 @@ class LanguageSnippetTestsEngine : AbstractLanguageSnippetTestsEngine() { .stripVersionCheckErrorMessage() } - val stderr = logWriter.toString() + val stderr = logWriter.toString().withUnixLineEndings() return (success && stderr.isBlank()) to (output + stderr).stripFilePaths().stripWebsite().stripStdlibLocationSha() } @@ -216,7 +221,7 @@ abstract class AbstractNativeLanguageSnippetTestsEngine : AbstractLanguageSnippe val process = builder.start() return try { val (out, err) = listOf(process.inputStream, process.errorStream) - .map { it.reader().readText() } + .map { it.reader().readText().withUnixLineEndings() } val success = process.waitFor() == 0 && err.isBlank() success to (out + err) .stripFilePaths() @@ -254,3 +259,8 @@ class AlpineLanguageSnippetTestsEngine : AbstractNativeLanguageSnippetTestsEngin override val pklExecutablePath: Path = rootProjectDir.resolve("pkl-cli/build/executable/pkl-alpine-linux-amd64") override val testClass: KClass<*> = AlpineLanguageSnippetTests::class } + +class WindowsLanguageSnippetTestsEngine : AbstractNativeLanguageSnippetTestsEngine() { + override val pklExecutablePath: Path = rootProjectDir.resolve("pkl-cli/build/executable/pkl-windows-amd64.exe") + override val testClass: KClass<*> = WindowsLanguageSnippetTests::class +} diff --git a/pkl-core/src/test/kotlin/org/pkl/core/SecurityManagersTest.kt b/pkl-core/src/test/kotlin/org/pkl/core/SecurityManagersTest.kt index 9f79614a..62615879 100644 --- a/pkl-core/src/test/kotlin/org/pkl/core/SecurityManagersTest.kt +++ b/pkl-core/src/test/kotlin/org/pkl/core/SecurityManagersTest.kt @@ -181,11 +181,11 @@ class SecurityManagersTest { rootDir ) - manager.checkResolveModule(URI("file:///foo/bar/baz.pkl")) - manager.checkReadResource(URI("file:///foo/bar/baz.pkl")) + manager.checkResolveModule(Path.of("/foo/bar/baz.pkl").toUri()) + manager.checkReadResource(Path.of("/foo/bar/baz.pkl").toUri()) - manager.checkResolveModule(URI("file:///foo/bar/qux/../baz.pkl")) - manager.checkReadResource(URI("file:///foo/bar/qux/../baz.pkl")) + manager.checkResolveModule(Path.of("/foo/bar/qux/../baz.pkl").toUri()) + manager.checkReadResource(Path.of("/foo/bar/qux/../baz.pkl").toUri()) } @Test @@ -233,17 +233,17 @@ class SecurityManagersTest { ) assertThrows { - manager.checkResolveModule(URI("file:///foo/baz.pkl")) + manager.checkResolveModule(Path.of("/foo/baz.pkl").toUri()) } assertThrows { - manager.checkReadResource(URI("file:///foo/baz.pkl")) + manager.checkReadResource(Path.of("/foo/baz.pkl").toUri()) } assertThrows { - manager.checkResolveModule(URI("file:///foo/bar/../baz.pkl")) + manager.checkResolveModule(Path.of("/foo/bar/../baz.pkl").toUri()) } assertThrows { - manager.checkReadResource(URI("file:///foo/bar/../baz.pkl")) + manager.checkReadResource(Path.of("/foo/bar/../baz.pkl").toUri()) } } } diff --git a/pkl-core/src/test/kotlin/org/pkl/core/module/ModuleKeysTest.kt b/pkl-core/src/test/kotlin/org/pkl/core/module/ModuleKeysTest.kt index 35297a4f..1056cd21 100644 --- a/pkl-core/src/test/kotlin/org/pkl/core/module/ModuleKeysTest.kt +++ b/pkl-core/src/test/kotlin/org/pkl/core/module/ModuleKeysTest.kt @@ -57,7 +57,7 @@ class ModuleKeysTest { file.writeString("age = 40") val uri = file.toUri() - val key = ModuleKeys.file(uri, file.toAbsolutePath()) + val key = ModuleKeys.file(uri) assertThat(key.uri).isEqualTo(uri) assertThat(key.isCached).isTrue diff --git a/pkl-core/src/test/kotlin/org/pkl/core/project/ProjectDependenciesResolverTest.kt b/pkl-core/src/test/kotlin/org/pkl/core/project/ProjectDependenciesResolverTest.kt index ff6d77a1..a5fb22b7 100644 --- a/pkl-core/src/test/kotlin/org/pkl/core/project/ProjectDependenciesResolverTest.kt +++ b/pkl-core/src/test/kotlin/org/pkl/core/project/ProjectDependenciesResolverTest.kt @@ -6,6 +6,7 @@ import org.junit.jupiter.api.Test import org.junit.jupiter.api.assertThrows import org.pkl.commons.test.FileTestUtils import org.pkl.commons.test.PackageServer +import org.pkl.commons.toPath import org.pkl.core.http.HttpClient import org.pkl.core.PklException import org.pkl.core.SecurityManagers @@ -34,7 +35,7 @@ class ProjectDependenciesResolverTest { @Test fun resolveDependencies() { - val project2Path = Path.of(javaClass.getResource("project2/PklProject")!!.path) + val project2Path = javaClass.getResource("project2/PklProject")!!.toURI().toPath() val project = Project.loadFromPath(project2Path) val packageResolver = PackageResolver.getInstance(SecurityManagers.defaultManager, httpClient, null) val deps = ProjectDependenciesResolver(project, packageResolver, System.out.writer()).resolve() @@ -72,7 +73,7 @@ class ProjectDependenciesResolverTest { @Test fun `fails if project declares a package with an incorrect checksum`() { - val projectPath = Path.of(javaClass.getResource("badProjectChecksum/PklProject")!!.path) + val projectPath = javaClass.getResource("badProjectChecksum/PklProject")!!.toURI().toPath() val project = Project.loadFromPath(projectPath) val packageResolver = PackageResolver.getInstance(SecurityManagers.defaultManager, httpClient, null) val e = assertThrows { diff --git a/pkl-core/src/test/kotlin/org/pkl/core/project/ProjectTest.kt b/pkl-core/src/test/kotlin/org/pkl/core/project/ProjectTest.kt index 450dc088..4fc5cf7d 100644 --- a/pkl-core/src/test/kotlin/org/pkl/core/project/ProjectTest.kt +++ b/pkl-core/src/test/kotlin/org/pkl/core/project/ProjectTest.kt @@ -137,7 +137,7 @@ class ProjectTest { @Test fun `evaluate project module -- invalid checksum`() { PackageServer().use { server -> - val projectDir = Path.of(javaClass.getResource("badProjectChecksum2/")!!.path) + val projectDir = Path.of(javaClass.getResource("badProjectChecksum2/")!!.toURI()) val project = Project.loadFromPath(projectDir.resolve("PklProject")) val httpClient = HttpClient.builder() .addCertificates(FileTestUtils.selfSignedCertificate) diff --git a/pkl-core/src/test/kotlin/org/pkl/core/util/IoUtilsTest.kt b/pkl-core/src/test/kotlin/org/pkl/core/util/IoUtilsTest.kt index 1e169df8..8f7034fb 100644 --- a/pkl-core/src/test/kotlin/org/pkl/core/util/IoUtilsTest.kt +++ b/pkl-core/src/test/kotlin/org/pkl/core/util/IoUtilsTest.kt @@ -117,70 +117,69 @@ class IoUtilsTest { @Test fun `relativize file URLs`() { - // perhaps URI("") would be a more precise result assertThat( IoUtils.relativize( - URI("file://foo/bar/baz.pkl"), - URI("file://foo/bar/baz.pkl") + URI("file:///foo/bar/baz.pkl"), + URI("file:///foo/bar/baz.pkl") ) ).isEqualTo(URI("baz.pkl")) assertThat( IoUtils.relativize( - URI("file://foo/bar/baz.pkl"), - URI("file://foo/bar/qux.pkl") + URI("file:///foo/bar/baz.pkl"), + URI("file:///foo/bar/qux.pkl") ) ).isEqualTo(URI("baz.pkl")) assertThat( IoUtils.relativize( - URI("file://foo/bar/baz.pkl"), - URI("file://foo/bar/") + URI("file:///foo/bar/baz.pkl"), + URI("file:///foo/bar/") ) ).isEqualTo(URI("baz.pkl")) assertThat( IoUtils.relativize( - URI("file://foo/bar/baz.pkl"), - URI("file://foo/bar") + URI("file:///foo/bar/baz.pkl"), + URI("file:///foo/bar") ) ).isEqualTo(URI("bar/baz.pkl")) // URI.relativize() returns an absolute URI here assertThat( IoUtils.relativize( - URI("file://foo/bar/baz.pkl"), - URI("file://foo/qux/") + URI("file:///foo/bar/baz.pkl"), + URI("file:///foo/qux/") ) ).isEqualTo(URI("../bar/baz.pkl")) assertThat( IoUtils.relativize( - URI("file://foo/bar/baz.pkl"), - URI("file://foo/qux/qux2/") + URI("file:///foo/bar/baz.pkl"), + URI("file:///foo/qux/qux2/") ) ).isEqualTo(URI("../../bar/baz.pkl")) assertThat( IoUtils.relativize( - URI("file://foo/bar/baz.pkl"), - URI("file://foo/qux/qux2") + URI("file:///foo/bar/baz.pkl"), + URI("file:///foo/qux/qux2") ) ).isEqualTo(URI("../bar/baz.pkl")) assertThat( IoUtils.relativize( - URI("file://foo/bar/baz.pkl"), - URI("file://qux/qux2/") + URI("file:///foo/bar/baz.pkl"), + URI("file:///qux/qux2/") ) - ).isEqualTo(URI("file://foo/bar/baz.pkl")) + ).isEqualTo(URI("../../foo/bar/baz.pkl")) assertThat( IoUtils.relativize( - URI("file://foo/bar/baz.pkl"), - URI("https://foo/bar/baz.pkl") + URI("file:///foo/bar/baz.pkl"), + URI("https:///foo/bar/baz.pkl") ) - ).isEqualTo(URI("file://foo/bar/baz.pkl")) + ).isEqualTo(URI("file:///foo/bar/baz.pkl")) } @Test @@ -343,7 +342,7 @@ class IoUtilsTest { val file3 = tempDir.resolve("base1/dir2/foo.pkl").createParentDirectories().createFile() val uri = file2.toUri() - val key = ModuleKeys.file(uri, file2) + val key = ModuleKeys.file(uri) assertThat(IoUtils.resolve(FakeSecurityManager, key, URI("..."))).isEqualTo(file1.toUri()) assertThat(IoUtils.resolve(FakeSecurityManager, key, URI(".../foo.pkl"))).isEqualTo(file1.toUri()) diff --git a/pkl-core/src/test/resources/META-INF/services/org.junit.platform.engine.TestEngine b/pkl-core/src/test/resources/META-INF/services/org.junit.platform.engine.TestEngine index 57e49043..9912afb4 100644 --- a/pkl-core/src/test/resources/META-INF/services/org.junit.platform.engine.TestEngine +++ b/pkl-core/src/test/resources/META-INF/services/org.junit.platform.engine.TestEngine @@ -4,3 +4,4 @@ org.pkl.core.MacAarch64LanguageSnippetTestsEngine org.pkl.core.LinuxAmd64LanguageSnippetTestsEngine org.pkl.core.LinuxAarch64LanguageSnippetTestsEngine org.pkl.core.AlpineLanguageSnippetTestsEngine +org.pkl.core.WindowsLanguageSnippetTestsEngine diff --git a/pkl-doc/src/main/kotlin/org/pkl/doc/CliDocGenerator.kt b/pkl-doc/src/main/kotlin/org/pkl/doc/CliDocGenerator.kt index 13b34ef0..e9421521 100644 --- a/pkl-doc/src/main/kotlin/org/pkl/doc/CliDocGenerator.kt +++ b/pkl-doc/src/main/kotlin/org/pkl/doc/CliDocGenerator.kt @@ -22,6 +22,7 @@ import kotlin.Pair import org.pkl.commons.cli.CliBaseOptions.Companion.getProjectFile import org.pkl.commons.cli.CliCommand import org.pkl.commons.cli.CliException +import org.pkl.commons.toPath import org.pkl.core.* import org.pkl.core.module.ModuleKeyFactories import org.pkl.core.packages.* @@ -136,7 +137,7 @@ class CliDocGenerator(private val options: CliDocGeneratorOptions) : CliCommand( val packageUris = mutableListOf() for (moduleUri in options.base.normalizedSourceModules) { if (moduleUri.scheme == "file") { - val dir = Path.of(moduleUri).parent + val dir = moduleUri.toPath().parent val projectFile = dir.getProjectFile(options.base.normalizedRootDir) if (projectFile != null) { pklProjectPaths.add(projectFile) @@ -229,13 +230,12 @@ class CliDocGenerator(private val options: CliDocGeneratorOptions) : CliCommand( DocPackageInfo.fromPkl(module).apply { evaluator.collectImportedModules(overviewImports) } - schemasByDocPackageInfoAndPath[docPackageInfo to Path.of(uri.path).parent] = - mutableSetOf() + schemasByDocPackageInfoAndPath[docPackageInfo to uri.toPath().parent] = mutableSetOf() } for (uri in regularModuleUris) { val entry = - schemasByDocPackageInfoAndPath.keys.find { uri.path.startsWith(it.second.toString()) } + schemasByDocPackageInfoAndPath.keys.find { uri.toPath().startsWith(it.second) } ?: throw CliException("Could not find a doc-package-info.pkl for module $uri") val schema = evaluator.evaluateSchema(ModuleSource.uri(uri)).apply { diff --git a/pkl-doc/src/main/kotlin/org/pkl/doc/DocGenerator.kt b/pkl-doc/src/main/kotlin/org/pkl/doc/DocGenerator.kt index 05bd25a7..436f66ad 100644 --- a/pkl-doc/src/main/kotlin/org/pkl/doc/DocGenerator.kt +++ b/pkl-doc/src/main/kotlin/org/pkl/doc/DocGenerator.kt @@ -26,6 +26,7 @@ import org.pkl.commons.deleteRecursively import org.pkl.core.ModuleSchema import org.pkl.core.PClassInfo import org.pkl.core.Version +import org.pkl.core.util.IoUtils /** * Entry point for the low-level Pkldoc API. @@ -126,7 +127,7 @@ class DocGenerator( val dest = basePath.resolve("current") if (dest.exists() && dest.isSameFileAs(src)) continue dest.deleteIfExists() - dest.createSymbolicLinkPointingTo(basePath.relativize(src)) + dest.createSymbolicLinkPointingTo(IoUtils.relativize(src, basePath)) } } } diff --git a/pkl-doc/src/main/kotlin/org/pkl/doc/Main.kt b/pkl-doc/src/main/kotlin/org/pkl/doc/Main.kt index 8575d29c..1bc69150 100644 --- a/pkl-doc/src/main/kotlin/org/pkl/doc/Main.kt +++ b/pkl-doc/src/main/kotlin/org/pkl/doc/Main.kt @@ -28,6 +28,7 @@ import java.net.URI import java.nio.file.Path import org.pkl.commons.cli.cliMain import org.pkl.commons.cli.commands.BaseCommand +import org.pkl.commons.cli.commands.BaseOptions.Companion.parseModuleName import org.pkl.commons.cli.commands.ProjectOptions import org.pkl.core.Release diff --git a/pkl-doc/src/test/kotlin/org/pkl/doc/CliDocGeneratorTest.kt b/pkl-doc/src/test/kotlin/org/pkl/doc/CliDocGeneratorTest.kt index 898cf445..290d7a0c 100644 --- a/pkl-doc/src/test/kotlin/org/pkl/doc/CliDocGeneratorTest.kt +++ b/pkl-doc/src/test/kotlin/org/pkl/doc/CliDocGeneratorTest.kt @@ -37,6 +37,7 @@ import org.pkl.commons.test.PackageServer import org.pkl.commons.test.listFilesRecursively import org.pkl.commons.toPath import org.pkl.core.Version +import org.pkl.core.util.IoUtils import org.pkl.doc.DocGenerator.Companion.current class CliDocGeneratorTest { @@ -92,11 +93,17 @@ class CliDocGeneratorTest { private val actualOutputFiles: List by lazy { actualOutputDir.listFilesRecursively() } private val expectedRelativeOutputFiles: List by lazy { - expectedOutputFiles.map { expectedOutputDir.relativize(it).toString() } + expectedOutputFiles.map { path -> + IoUtils.toNormalizedPathString(expectedOutputDir.relativize(path)).let { str -> + // Git will by default clone symlinks as shortcuts on Windows, and shortcuts have a + // `.lnk` extension. + if (IoUtils.isWindows() && str.endsWith(".lnk")) str.dropLast(4) else str + } + } } private val actualRelativeOutputFiles: List by lazy { - actualOutputFiles.map { actualOutputDir.relativize(it).toString() } + actualOutputFiles.map { IoUtils.toNormalizedPathString(actualOutputDir.relativize(it)) } } private val binaryFileExtensions = @@ -219,6 +226,11 @@ class CliDocGeneratorTest { .withFailMessage("Test bug: $actualFile should exist but does not.") .exists() + // symlinks on Git and Windows is rather finnicky; they create shortcuts by default unless + // a core Git option is set. Also, by default, symlinks require administrator privileges to run. + // We'll just test that the symlink got created but skip verifying that it points to the right + // place. + if (actualFile.isSymbolicLink() && IoUtils.isWindows()) return val expectedFile = expectedOutputDir.resolve(relativeFilePath) if (expectedFile.exists()) { when { diff --git a/pkl-doc/src/test/kotlin/org/pkl/doc/DocScopeTest.kt b/pkl-doc/src/test/kotlin/org/pkl/doc/DocScopeTest.kt index 8b028f03..6a26a564 100644 --- a/pkl-doc/src/test/kotlin/org/pkl/doc/DocScopeTest.kt +++ b/pkl-doc/src/test/kotlin/org/pkl/doc/DocScopeTest.kt @@ -16,6 +16,7 @@ package org.pkl.doc import java.net.URI +import java.nio.file.Path import org.assertj.core.api.Assertions.assertThat import org.junit.jupiter.api.AfterAll import org.junit.jupiter.api.Test @@ -221,6 +222,6 @@ class DocScopeTest { val scope = SiteScope(listOf(), mapOf(), { evaluator.evaluateSchema(uri(it)) }, outputDir) // used to return `/non/index.html` - assertThat(scope.url.path).isEqualTo("/non/existing/index.html") + assertThat(scope.url.toPath()).isEqualTo(Path.of("/non/existing/index.html").toAbsolutePath()) } } diff --git a/pkl-executor/pkl-executor.gradle.kts b/pkl-executor/pkl-executor.gradle.kts index abd5b507..a93d7958 100644 --- a/pkl-executor/pkl-executor.gradle.kts +++ b/pkl-executor/pkl-executor.gradle.kts @@ -1,4 +1,5 @@ import java.nio.file.Files +import java.nio.file.LinkOption plugins { pklAllProjects @@ -62,12 +63,23 @@ val prepareHistoricalDistributions by tasks.registering { val distributionDir = outputDir.get().asFile.toPath() .also(Files::createDirectories) for (file in pklHistoricalDistributions.files) { - val link = distributionDir.resolve(file.name) - if (!Files.isSymbolicLink(link)) { - if (Files.exists(link)) { - Files.delete(link) + val target = distributionDir.resolve(file.name) + // Create normal files on Windows, symlink on macOS/linux (need admin priveleges to create + // symlinks on Windows) + if (buildInfo.os.isWindows) { + if (!Files.isRegularFile(target, LinkOption.NOFOLLOW_LINKS)) { + if (Files.exists(target)) { + Files.delete(target) + } + Files.copy(file.toPath(), target) + } + } else { + if (!Files.isSymbolicLink(target)) { + if (Files.exists(target)) { + Files.delete(target) + } + Files.createSymbolicLink(target, file.toPath()) } - Files.createSymbolicLink(link, file.toPath()) } } } diff --git a/pkl-executor/src/main/java/org/pkl/executor/EmbeddedExecutor.java b/pkl-executor/src/main/java/org/pkl/executor/EmbeddedExecutor.java index 5dfbb963..3f1a4318 100644 --- a/pkl-executor/src/main/java/org/pkl/executor/EmbeddedExecutor.java +++ b/pkl-executor/src/main/java/org/pkl/executor/EmbeddedExecutor.java @@ -157,7 +157,15 @@ final class EmbeddedExecutor implements Executor { private static Path toDisplayPath(Path modulePath, ExecutorOptions options) { var rootDir = options.getRootDir(); - return rootDir == null ? modulePath : rootDir.relativize(modulePath); + return rootDir == null ? modulePath : relativize(modulePath, rootDir); + } + + // On Windows, `Path.relativize` will fail if the two paths have different roots. + private static Path relativize(Path path, Path base) { + if (path.isAbsolute() && base.isAbsolute() && !path.getRoot().equals(base.getRoot())) { + return path; + } + return base.relativize(path); } @Override diff --git a/pkl-executor/src/test/kotlin/org/pkl/executor/EmbeddedExecutorTest.kt b/pkl-executor/src/test/kotlin/org/pkl/executor/EmbeddedExecutorTest.kt index 97b85d0d..7a8d44c2 100644 --- a/pkl-executor/src/test/kotlin/org/pkl/executor/EmbeddedExecutorTest.kt +++ b/pkl-executor/src/test/kotlin/org/pkl/executor/EmbeddedExecutorTest.kt @@ -15,6 +15,7 @@ import org.pkl.commons.test.FilteringClassLoader import org.pkl.commons.test.PackageServer import org.pkl.commons.toPath import org.pkl.core.Release +import java.io.File import java.nio.file.Files import java.nio.file.Path import java.time.Duration @@ -197,9 +198,10 @@ class EmbeddedExecutorTest { Executors.embedded(listOf("/non/existing".toPath())) } + val sep = File.separatorChar assertThat(e.message) .contains("Cannot find Jar file") - .contains("/non/existing") + .contains("${sep}non${sep}existing") } @Test diff --git a/pkl-gradle/src/main/java/org/pkl/gradle/task/ModulesTask.java b/pkl-gradle/src/main/java/org/pkl/gradle/task/ModulesTask.java index 3acc52e8..8717a280 100644 --- a/pkl-gradle/src/main/java/org/pkl/gradle/task/ModulesTask.java +++ b/pkl-gradle/src/main/java/org/pkl/gradle/task/ModulesTask.java @@ -134,7 +134,7 @@ public abstract class ModulesTask extends BasePklTask { */ private URI parsedModuleNotationToUri(Object notation) { if (notation instanceof File file) { - return IoUtils.createUri(file.getPath()); + return IoUtils.createUri(IoUtils.toNormalizedPathString(file.toPath())); } else if (notation instanceof URI uri) { return uri; } diff --git a/pkl-gradle/src/test/kotlin/org/pkl/gradle/EvaluatorsTest.kt b/pkl-gradle/src/test/kotlin/org/pkl/gradle/EvaluatorsTest.kt index 56326abd..ccd58ca4 100644 --- a/pkl-gradle/src/test/kotlin/org/pkl/gradle/EvaluatorsTest.kt +++ b/pkl-gradle/src/test/kotlin/org/pkl/gradle/EvaluatorsTest.kt @@ -2,7 +2,9 @@ package org.pkl.gradle import org.assertj.core.api.Assertions import org.pkl.commons.readString +import org.pkl.commons.readString import org.pkl.commons.test.PackageServer +import org.pkl.commons.toNormalizedPathString import org.assertj.core.api.Assertions.assertThat import org.junit.jupiter.api.Test import org.junit.jupiter.api.io.TempDir @@ -199,8 +201,8 @@ class EvaluatorsTest : AbstractTest() { @Test fun `source module URIs`() { - val pklFile = writeFile( - "test.pkl", """ + writeFile( + "testDir/test.pkl", """ person { name = "Pigeon" age = 20 + 10 @@ -218,7 +220,7 @@ class EvaluatorsTest : AbstractTest() { evaluators { evalTest { sourceModules = [uri("modulepath:/test.pkl")] - modulePath.from "${pklFile.parent}" + modulePath.from layout.projectDirectory.dir("testDir") outputFile = layout.projectDirectory.file("test.pcf") settingsModule = "pkl:settings" } @@ -387,7 +389,7 @@ class EvaluatorsTest : AbstractTest() { @Test fun `explicitly set cache dir`(@TempDir tempDir: Path) { writeBuildFile("pcf", """ - moduleCacheDir = file("$tempDir") + moduleCacheDir = file("${tempDir.toUri()}") """.trimIndent()) writeFile( "test.pkl", diff --git a/pkl-gradle/src/test/kotlin/org/pkl/gradle/TestsTest.kt b/pkl-gradle/src/test/kotlin/org/pkl/gradle/TestsTest.kt index 46dc235b..769ec540 100644 --- a/pkl-gradle/src/test/kotlin/org/pkl/gradle/TestsTest.kt +++ b/pkl-gradle/src/test/kotlin/org/pkl/gradle/TestsTest.kt @@ -1,6 +1,7 @@ package org.pkl.gradle import org.assertj.core.api.Assertions.assertThat +import org.pkl.commons.toNormalizedPathString import org.junit.jupiter.api.Test import java.nio.file.Path import kotlin.io.path.readText @@ -46,8 +47,7 @@ class TestsTest : AbstractTest() { val output = runTask("evalTest", expectFailure = true).output.stripFilesAndLines() - assertThat(output.trimStart()).startsWith(""" - > Task :evalTest FAILED + assertThat(output).contains(""" module test (file:///file, line x) test ❌ Error: @@ -143,7 +143,7 @@ class TestsTest : AbstractTest() { val pklFile = writePklFile(contents = bigTest) writeFile("test.pkl-expected.pcf", bigTestExpected) - writeBuildFile("junitReportsDir = file('${pklFile.parent}/build')") + writeBuildFile("junitReportsDir = file('${pklFile.parent.toNormalizedPathString()}/build')") runTask("evalTest", expectFailure = true) diff --git a/pkl-server/src/test/kotlin/org/pkl/server/BinaryEvaluatorSnippetTests.kt b/pkl-server/src/test/kotlin/org/pkl/server/BinaryEvaluatorSnippetTests.kt index ba303dab..b530d888 100644 --- a/pkl-server/src/test/kotlin/org/pkl/server/BinaryEvaluatorSnippetTests.kt +++ b/pkl-server/src/test/kotlin/org/pkl/server/BinaryEvaluatorSnippetTests.kt @@ -58,7 +58,8 @@ class BinaryEvaluatorSnippetTestEngine : InputOutputTestEngine() { null ) - private fun String.stripFilePaths() = replace(snippetsDir.toString(), "/\$snippetsDir") + private fun String.stripFilePaths() = + replace(snippetsDir.toUri().toString(), "file:///\$snippetsDir/") override fun generateOutputFor(inputFile: Path): Pair { val bytes = evaluator.evaluate(ModuleSource.path(inputFile), null) diff --git a/stdlib/platform.pkl b/stdlib/platform.pkl index 088e8bbc..c7d5297e 100644 --- a/stdlib/platform.pkl +++ b/stdlib/platform.pkl @@ -66,7 +66,7 @@ class VirtualMachine { /// The operating system of a platform. class OperatingSystem { /// The name of this operating system. - name: String + name: "macOS"|"Linux"|"Windows"|String /// The version of this operating system. version: String