starred/pkl
1
0
Fork 0
mirror of https://github.com/apple/pkl.git synced 2026-05-14 02:49:56 +02:00
Code Issues 198 Packages Projects Releases 10 Wiki Activity

Add support for Windows (#492)

Browse Source 
This adds support for Windows.
The in-language path separator is still `/`, to ensure Pkl programs are cross-platform.

Log lines are written using CRLF endings on Windows.
Modules that are combined with `--module-output-separator` uses LF endings to ensure
consistent rendering across platforms.

`jpkl` does not work on Windows as a direct executable.
However, it can work with `java -jar jpkl`.

Additional details:

* Adjust git settings for Windows
* Add native executable for pkl cli
* Add jdk17 windows Gradle check in CI
* Adjust CI test reports to be staged within Gradle rather than by shell script.
* Fix: encode more characters that are not safe Windows paths
* Skip running tests involving symbolic links on Windows (these require administrator privileges to run).
* Introduce custom implementation of `IoUtils.relativize`
* Allow Gradle to initialize ExecutableJar `Property` values
* Add Gradle flag to enable remote JVM debugging

Co-authored-by: Philip K.F. Hölzenspies <holzensp@gmail.com>
This commit is contained in:
Daniel Chao
2024-05-28 15:56:20 -07:00
committed by GitHub
parent 5e4ccfd4e8
commit 8ec06e631f
76 changed files with 905 additions and 402 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
.circleci/config.pkl
View File

@@ -14,7 +14,7 @@
// limitations under the License.
//===----------------------------------------------------------------------===//
// File gets rendered to .circleci/config.yml via git hook.
amends "package://pkg.pkl-lang.org/pkl-project-commons/pkl.impl.circleci@1.1.0#/PklCI.pkl"
amends "package://pkg.pkl-lang.org/pkl-project-commons/pkl.impl.circleci@1.1.1#/PklCI.pkl"
import "jobs/BuildNativeJob.pkl"
import "jobs/GradleCheckJob.pkl"
@@ -96,6 +96,11 @@ local buildNativeJobs: Mapping<String, BuildNativeJob> = new {
musl = true
isRelease = _dist == "release"
}
["pkl-cli-windows-amd64-\(_dist)"] {
arch = "amd64"
os = "windows"
isRelease = _dist == "release"
}
}
}
@@ -108,6 +113,11 @@ local gradleCheckJobs: Mapping<String, GradleCheckJob> = new {
javaVersion = "21.0"
isRelease = false
}
["gradle-check-jdk17-windows"] {
javaVersion = "17.0"
isRelease = false
os = "windows"
}
}
jobs {

204
.circleci/config.yml
View File

@@ -12,18 +12,12 @@ jobs:
- run:
command: |-
export PATH=~/staticdeps/bin:$PATH
./gradlew --info --stacktrace -DreleaseBuild=true pkl-cli:macExecutableAmd64 pkl-core:testMacExecutableAmd64
./gradlew --info --stacktrace -DtestReportsDir=${HOME}/test-results -DreleaseBuild=true pkl-cli:macExecutableAmd64 pkl-core:testMacExecutableAmd64
name: gradle buildNative
- persist_to_workspace:
root: '.'
paths:
- pkl-cli/build/executable/
- run:
command: |-
mkdir ~/test-results/
find . -type f -regex ".*/build/test-results/.*xml" -exec cp {} ~/test-results/ \;
name: Gather test results
when: always
- store_test_results:
path: ~/test-results
environment:
@@ -94,18 +88,12 @@ jobs:
- run:
command: |-
export PATH=~/staticdeps/bin:$PATH
./gradlew --info --stacktrace -DreleaseBuild=true pkl-cli:linuxExecutableAmd64 pkl-core:testLinuxExecutableAmd64
./gradlew --info --stacktrace -DtestReportsDir=${HOME}/test-results -DreleaseBuild=true pkl-cli:linuxExecutableAmd64 pkl-core:testLinuxExecutableAmd64
name: gradle buildNative
- persist_to_workspace:
root: '.'
paths:
- pkl-cli/build/executable/
- run:
command: |-
mkdir ~/test-results/
find . -type f -regex ".*/build/test-results/.*xml" -exec cp {} ~/test-results/ \;
name: Gather test results
when: always
- store_test_results:
path: ~/test-results
environment:
@@ -120,18 +108,12 @@ jobs:
- run:
command: |-
export PATH=~/staticdeps/bin:$PATH
./gradlew --info --stacktrace -DreleaseBuild=true pkl-cli:macExecutableAarch64 pkl-core:testMacExecutableAarch64
./gradlew --info --stacktrace -DtestReportsDir=${HOME}/test-results -DreleaseBuild=true pkl-cli:macExecutableAarch64 pkl-core:testMacExecutableAarch64
name: gradle buildNative
- persist_to_workspace:
root: '.'
paths:
- pkl-cli/build/executable/
- run:
command: |-
mkdir ~/test-results/
find . -type f -regex ".*/build/test-results/.*xml" -exec cp {} ~/test-results/ \;
name: Gather test results
when: always
- store_test_results:
path: ~/test-results
environment:
@@ -186,18 +168,12 @@ jobs:
- run:
command: |-
export PATH=~/staticdeps/bin:$PATH
./gradlew --info --stacktrace -DreleaseBuild=true pkl-cli:linuxExecutableAarch64 pkl-core:testLinuxExecutableAarch64
./gradlew --info --stacktrace -DtestReportsDir=${HOME}/test-results -DreleaseBuild=true pkl-cli:linuxExecutableAarch64 pkl-core:testLinuxExecutableAarch64
name: gradle buildNative
- persist_to_workspace:
root: '.'
paths:
- pkl-cli/build/executable/
- run:
command: |-
mkdir ~/test-results/
find . -type f -regex ".*/build/test-results/.*xml" -exec cp {} ~/test-results/ \;
name: Gather test results
when: always
- store_test_results:
path: ~/test-results
environment:
@@ -269,18 +245,12 @@ jobs:
- run:
command: |-
export PATH=~/staticdeps/bin:$PATH
./gradlew --info --stacktrace -DreleaseBuild=true pkl-cli:alpineExecutableAmd64 pkl-core:testAlpineExecutableAmd64
./gradlew --info --stacktrace -DtestReportsDir=${HOME}/test-results -DreleaseBuild=true pkl-cli:alpineExecutableAmd64 pkl-core:testAlpineExecutableAmd64
name: gradle buildNative
- persist_to_workspace:
root: '.'
paths:
- pkl-cli/build/executable/
- run:
command: |-
mkdir ~/test-results/
find . -type f -regex ".*/build/test-results/.*xml" -exec cp {} ~/test-results/ \;
name: Gather test results
when: always
- store_test_results:
path: ~/test-results
environment:
@@ -289,6 +259,26 @@ jobs:
resource_class: xlarge
docker:
- image: oraclelinux:8-slim
pkl-cli-windows-amd64-release:
steps:
- checkout
- run:
command: |-
export PATH=~/staticdeps/bin:$PATH
./gradlew --info --stacktrace -DtestReportsDir=${HOME}/test-results -DreleaseBuild=true pkl-cli:windowsExecutableAmd64 pkl-core:testWindowsExecutableAmd64
name: gradle buildNative
shell: bash.exe
- persist_to_workspace:
root: '.'
paths:
- pkl-cli/build/executable/
- store_test_results:
path: ~/test-results
environment:
LANG: en_US.UTF-8
resource_class: windows.large
machine:
image: windows-server-2022-gui:current
pkl-cli-macOS-amd64-snapshot:
steps:
- checkout
@@ -298,18 +288,12 @@ jobs:
- run:
command: |-
export PATH=~/staticdeps/bin:$PATH
./gradlew --info --stacktrace pkl-cli:macExecutableAmd64 pkl-core:testMacExecutableAmd64
./gradlew --info --stacktrace -DtestReportsDir=${HOME}/test-results pkl-cli:macExecutableAmd64 pkl-core:testMacExecutableAmd64
name: gradle buildNative
- persist_to_workspace:
root: '.'
paths:
- pkl-cli/build/executable/
- run:
command: |-
mkdir ~/test-results/
find . -type f -regex ".*/build/test-results/.*xml" -exec cp {} ~/test-results/ \;
name: Gather test results
when: always
- store_test_results:
path: ~/test-results
environment:
@@ -380,18 +364,12 @@ jobs:
- run:
command: |-
export PATH=~/staticdeps/bin:$PATH
./gradlew --info --stacktrace pkl-cli:linuxExecutableAmd64 pkl-core:testLinuxExecutableAmd64
./gradlew --info --stacktrace -DtestReportsDir=${HOME}/test-results pkl-cli:linuxExecutableAmd64 pkl-core:testLinuxExecutableAmd64
name: gradle buildNative
- persist_to_workspace:
root: '.'
paths:
- pkl-cli/build/executable/
- run:
command: |-
mkdir ~/test-results/
find . -type f -regex ".*/build/test-results/.*xml" -exec cp {} ~/test-results/ \;
name: Gather test results
when: always
- store_test_results:
path: ~/test-results
environment:
@@ -406,18 +384,12 @@ jobs:
- run:
command: |-
export PATH=~/staticdeps/bin:$PATH
./gradlew --info --stacktrace pkl-cli:macExecutableAarch64 pkl-core:testMacExecutableAarch64
./gradlew --info --stacktrace -DtestReportsDir=${HOME}/test-results pkl-cli:macExecutableAarch64 pkl-core:testMacExecutableAarch64
name: gradle buildNative
- persist_to_workspace:
root: '.'
paths:
- pkl-cli/build/executable/
- run:
command: |-
mkdir ~/test-results/
find . -type f -regex ".*/build/test-results/.*xml" -exec cp {} ~/test-results/ \;
name: Gather test results
when: always
- store_test_results:
path: ~/test-results
environment:
@@ -472,18 +444,12 @@ jobs:
- run:
command: |-
export PATH=~/staticdeps/bin:$PATH
./gradlew --info --stacktrace pkl-cli:linuxExecutableAarch64 pkl-core:testLinuxExecutableAarch64
./gradlew --info --stacktrace -DtestReportsDir=${HOME}/test-results pkl-cli:linuxExecutableAarch64 pkl-core:testLinuxExecutableAarch64
name: gradle buildNative
- persist_to_workspace:
root: '.'
paths:
- pkl-cli/build/executable/
- run:
command: |-
mkdir ~/test-results/
find . -type f -regex ".*/build/test-results/.*xml" -exec cp {} ~/test-results/ \;
name: Gather test results
when: always
- store_test_results:
path: ~/test-results
environment:
@@ -555,18 +521,12 @@ jobs:
- run:
command: |-
export PATH=~/staticdeps/bin:$PATH
./gradlew --info --stacktrace pkl-cli:alpineExecutableAmd64 pkl-core:testAlpineExecutableAmd64
./gradlew --info --stacktrace -DtestReportsDir=${HOME}/test-results pkl-cli:alpineExecutableAmd64 pkl-core:testAlpineExecutableAmd64
name: gradle buildNative
- persist_to_workspace:
root: '.'
paths:
- pkl-cli/build/executable/
- run:
command: |-
mkdir ~/test-results/
find . -type f -regex ".*/build/test-results/.*xml" -exec cp {} ~/test-results/ \;
name: Gather test results
when: always
- store_test_results:
path: ~/test-results
environment:
@@ -575,18 +535,32 @@ jobs:
resource_class: xlarge
docker:
- image: oraclelinux:8-slim
pkl-cli-windows-amd64-snapshot:
steps:
- checkout
- run:
command: |-
export PATH=~/staticdeps/bin:$PATH
./gradlew --info --stacktrace -DtestReportsDir=${HOME}/test-results pkl-cli:windowsExecutableAmd64 pkl-core:testWindowsExecutableAmd64
name: gradle buildNative
shell: bash.exe
- persist_to_workspace:
root: '.'
paths:
- pkl-cli/build/executable/
- store_test_results:
path: ~/test-results
environment:
LANG: en_US.UTF-8
resource_class: windows.large
machine:
image: windows-server-2022-gui:current
gradle-check-jdk17:
steps:
- checkout
- run:
command: ./gradlew --info --stacktrace check
command: ./gradlew --info --stacktrace -DtestReportsDir=${HOME}/test-results check
name: gradle check
- run:
command: |-
mkdir ~/test-results/
find . -type f -regex ".*/build/test-results/.*xml" -exec cp {} ~/test-results/ \;
name: Gather test results
when: always
- store_test_results:
path: ~/test-results
environment:
@@ -597,32 +571,33 @@ jobs:
steps:
- checkout
- run:
command: ./gradlew --info --stacktrace check
command: ./gradlew --info --stacktrace -DtestReportsDir=${HOME}/test-results check
name: gradle check
- run:
command: |-
mkdir ~/test-results/
find . -type f -regex ".*/build/test-results/.*xml" -exec cp {} ~/test-results/ \;
name: Gather test results
when: always
- store_test_results:
path: ~/test-results
environment:
LANG: en_US.UTF-8
docker:
- image: cimg/openjdk:21.0
gradle-check-jdk17-windows:
steps:
- checkout
- run:
command: ./gradlew --info --stacktrace -DtestReportsDir=${HOME}/test-results check
name: gradle check
- store_test_results:
path: ~/test-results
environment:
LANG: en_US.UTF-8
resource_class: windows.large
machine:
image: windows-server-2022-gui:current
bench:
steps:
- checkout
- run:
command: ./gradlew --info --stacktrace bench:jmh
command: ./gradlew --info --stacktrace -DtestReportsDir=${HOME}/test-results bench:jmh
name: bench:jmh
- run:
command: |-
mkdir ~/test-results/
find . -type f -regex ".*/build/test-results/.*xml" -exec cp {} ~/test-results/ \;
name: Gather test results
when: always
- store_test_results:
path: ~/test-results
environment:
@@ -634,16 +609,10 @@ jobs:
- checkout
- run:
command: |-
./gradlew --info --stacktrace :pkl-gradle:build \
./gradlew --info --stacktrace -DtestReportsDir=${HOME}/test-results :pkl-gradle:build \
:pkl-gradle:compatibilityTestReleases \
:pkl-gradle:compatibilityTestCandidate
name: gradle compatibility
- run:
command: |-
mkdir ~/test-results/
find . -type f -regex ".*/build/test-results/.*xml" -exec cp {} ~/test-results/ \;
name: Gather test results
when: always
- store_test_results:
path: ~/test-results
environment:
@@ -656,17 +625,11 @@ jobs:
- attach_workspace:
at: '.'
- run:
command: ./gradlew --info --stacktrace publishToSonatype
command: ./gradlew --info --stacktrace -DtestReportsDir=${HOME}/test-results publishToSonatype
- persist_to_workspace:
root: '.'
paths:
- pkl-cli/build/executable/
- run:
command: |-
mkdir ~/test-results/
find . -type f -regex ".*/build/test-results/.*xml" -exec cp {} ~/test-results/ \;
name: Gather test results
when: always
- store_test_results:
path: ~/test-results
environment:
@@ -679,17 +642,11 @@ jobs:
- attach_workspace:
at: '.'
- run:
command: ./gradlew --info --stacktrace -DreleaseBuild=true publishToSonatype closeAndReleaseSonatypeStagingRepository
command: ./gradlew --info --stacktrace -DtestReportsDir=${HOME}/test-results -DreleaseBuild=true publishToSonatype closeAndReleaseSonatypeStagingRepository
- persist_to_workspace:
root: '.'
paths:
- pkl-cli/build/executable/
- run:
command: |-
mkdir ~/test-results/
find . -type f -regex ".*/build/test-results/.*xml" -exec cp {} ~/test-results/ \;
name: Gather test results
when: always
- store_test_results:
path: ~/test-results
environment:
@@ -753,6 +710,9 @@ workflows:
- gradle-check-jdk21:
requires:
- hold
- gradle-check-jdk17-windows:
requires:
- hold
when:
matches:
value: << pipeline.git.branch >>
@@ -761,6 +721,7 @@ workflows:
jobs:
- gradle-check-jdk17
- gradle-check-jdk21
- gradle-check-jdk17-windows
- bench
- gradle-compatibility
- pkl-cli-macOS-amd64-snapshot
@@ -768,10 +729,12 @@ workflows:
- pkl-cli-macOS-aarch64-snapshot
- pkl-cli-linux-aarch64-snapshot
- pkl-cli-linux-alpine-amd64-snapshot
- pkl-cli-windows-amd64-snapshot
- deploy-snapshot:
requires:
- gradle-check-jdk17
- gradle-check-jdk21
- gradle-check-jdk17-windows
- bench
- gradle-compatibility
- pkl-cli-macOS-amd64-snapshot
@@ -779,6 +742,7 @@ workflows:
- pkl-cli-macOS-aarch64-snapshot
- pkl-cli-linux-aarch64-snapshot
- pkl-cli-linux-alpine-amd64-snapshot
- pkl-cli-windows-amd64-snapshot
context: pkl-maven-release
- trigger-docsite-build:
requires:
@@ -803,6 +767,12 @@ workflows:
ignore: /.*/
tags:
only: /^v?\d+\.\d+\.\d+$/
- gradle-check-jdk17-windows:
filters:
branches:
ignore: /.*/
tags:
only: /^v?\d+\.\d+\.\d+$/
- bench:
filters:
branches:
@@ -845,10 +815,17 @@ workflows:
ignore: /.*/
tags:
only: /^v?\d+\.\d+\.\d+$/
- pkl-cli-windows-amd64-release:
filters:
branches:
ignore: /.*/
tags:
only: /^v?\d+\.\d+\.\d+$/
- github-release:
requires:
- gradle-check-jdk17
- gradle-check-jdk21
- gradle-check-jdk17-windows
- bench
- gradle-compatibility
- pkl-cli-macOS-amd64-release
@@ -856,6 +833,7 @@ workflows:
- pkl-cli-macOS-aarch64-release
- pkl-cli-linux-aarch64-release
- pkl-cli-linux-alpine-amd64-release
- pkl-cli-windows-amd64-release
context: pkl-github-release
filters:
branches:
@@ -885,6 +863,7 @@ workflows:
jobs:
- gradle-check-jdk17
- gradle-check-jdk21
- gradle-check-jdk17-windows
- bench
- gradle-compatibility
- pkl-cli-macOS-amd64-release
@@ -892,6 +871,7 @@ workflows:
- pkl-cli-macOS-aarch64-release
- pkl-cli-linux-aarch64-release
- pkl-cli-linux-alpine-amd64-release
- pkl-cli-windows-amd64-release
when:
matches:
value: << pipeline.git.branch >>

20
.circleci/jobs/BuildNativeJob.pkl
View File

@@ -16,12 +16,9 @@
/// Builds the native `pkl` CLI
extends "GradleJob.pkl"
import "package://pkg.pkl-lang.org/pkl-pantry/com.circleci.v2@1.1.0#/Config.pkl"
import "package://pkg.pkl-lang.org/pkl-pantry/com.circleci.v2@1.1.2#/Config.pkl"
import "package://pkg.pkl-lang.org/pkl-pantry/pkl.experimental.uri@1.0.0#/URI.pkl"
/// The OS to run on
os: "macOS"|"linux"
/// The architecture to use
arch: "amd64"|"aarch64"
@@ -119,10 +116,14 @@ steps {
new Config.RunStep {
name = "gradle buildNative"
local _os =
if (os == "macOS") "mac"
if (module.os == "macOS") "mac"
else if (musl) "alpine"
else if (module.os == "windows") "windows"
else "linux"
local jobName = "\(_os)Executable\(arch.capitalize())"
when (module.os == "windows") {
shell = "bash.exe"
}
command = #"""
export PATH=~/staticdeps/bin:$PATH
./gradlew \#(module.gradleArgs) pkl-cli:\#(jobName) pkl-core:test\#(jobName.capitalize())
@@ -142,7 +143,8 @@ job {
xcode = "15.3.0"
}
resource_class = "macos.m1.large.gen1"
} else {
}
when (os == "linux") {
docker {
new {
image = if (arch == "aarch64") "arm64v8/oraclelinux:8-slim" else "oraclelinux:8-slim"
@@ -153,4 +155,10 @@ job {
}
resource_class = if (arch == "aarch64") "arm.xlarge" else "xlarge"
}
when (os == "windows") {
machine {
image = "windows-server-2022-gui:current"
}
resource_class = "windows.large"
}
}

4
.circleci/jobs/DeployJob.pkl
View File

@@ -15,7 +15,7 @@
//===----------------------------------------------------------------------===//
extends "GradleJob.pkl"
import "package://pkg.pkl-lang.org/pkl-pantry/com.circleci.v2@1.1.0#/Config.pkl"
import "package://pkg.pkl-lang.org/pkl-pantry/com.circleci.v2@1.1.2#/Config.pkl"
local self = this
@@ -27,6 +27,8 @@ job {
}
}
os = "linux"
steps {
new Config.AttachWorkspaceStep { at = "." }
new Config.RunStep {

20
.circleci/jobs/GradleCheckJob.pkl
View File

@@ -15,9 +15,11 @@
//===----------------------------------------------------------------------===//
extends "GradleJob.pkl"
import "package://pkg.pkl-lang.org/pkl-pantry/com.circleci.v2@1.1.0#/Config.pkl"
import "package://pkg.pkl-lang.org/pkl-pantry/com.circleci.v2@1.1.2#/Config.pkl"
javaVersion: "11.0"|"17.0"|"21.0"
javaVersion: "17.0"|"21.0"
os = "linux"
steps {
new Config.RunStep {
@@ -27,9 +29,17 @@ steps {
}
job {
docker {
new {
image = "cimg/openjdk:\(javaVersion)"
when (os == "linux") {
docker {
new {
image = "cimg/openjdk:\(javaVersion)"
}
}
}
when (os == "windows") {
machine {
image = "windows-server-2022-gui:current"
}
resource_class = "windows.large"
}
}

15
.circleci/jobs/GradleJob.pkl
View File

@@ -15,14 +15,18 @@
//===----------------------------------------------------------------------===//
abstract module GradleJob
import "package://pkg.pkl-lang.org/pkl-pantry/com.circleci.v2@1.1.0#/Config.pkl"
import "package://pkg.pkl-lang.org/pkl-pantry/com.circleci.v2@1.1.2#/Config.pkl"
/// Whether this is a release build or not.
isRelease: Boolean = false
/// The OS to run on
os: "macOS"|"linux"|"windows"
fixed gradleArgs = new Listing {
"--info"
"--stacktrace"
"-DtestReportsDir=${HOME}/test-results"
when (isRelease) {
"-DreleaseBuild=true"
}
@@ -37,15 +41,6 @@ job: Config.Job = new {
steps {
"checkout"
...module.steps
new Config.RunStep {
// find all test results and write them to the home dir
name = "Gather test results"
command = """
mkdir ~/test-results/
find . -type f -regex ".*/build/test-results/.*xml" -exec cp {} ~/test-results/ \\;
"""
`when` = "always"
}
new Config.StoreTestResults {
path = "~/test-results"
}

4
.circleci/jobs/SimpleGradleJob.pkl
View File

@@ -15,12 +15,14 @@
//===----------------------------------------------------------------------===//
extends "GradleJob.pkl"
import "package://pkg.pkl-lang.org/pkl-pantry/com.circleci.v2@1.1.0#/Config.pkl"
import "package://pkg.pkl-lang.org/pkl-pantry/com.circleci.v2@1.1.2#/Config.pkl"
name: String = command
command: String
os = "linux"
steps {
new Config.RunStep {
name = module.name

1
.gitattributes vendored
View File

@@ -4,3 +4,4 @@
/docs/** linguist-documentation
*.pkl linguist-language=Groovy
* text eol=lf

8
DEVELOPMENT.adoc
View File

@@ -40,6 +40,7 @@ pkl-cli/build/executable/jpkl # run Java executable
pkl-cli/build/executable/pkl-macos-amd64 # run Mac executable
pkl-cli/build/executable/pkl-linux-amd64 # run Linux executable
pkl-cli/build/executable/pkl-alpine-linux-amd64 # run Alpine Linux executable
pkl-cli/build/executable/pkl-windows-amd64.exe # run Windows executable
----
== Update Gradle
@@ -68,6 +69,13 @@ based on version information from https://search.maven.org, https://plugins.grad
* ANTLR code generation is performed by task `:pkl-core:generateGrammarSource`
** Output dir is `generated/antlr/`
== Remote JVM Debugging
To enable remote JVM debugging when running Gradle tasks (e.g. test), add the flag `-Djvmdebug=true`.
This will listen on port 5005.
Example: `./gradlew test -Djvmdebug=true`
== Resources
For automated build setup examples see our https://github.com/apple/pkl/blob/main/.circleci/[CircleCI] jobs like our https://github.com/apple/pkl/blob/main/.circleci/jobs/BuildNativeJob.pkl[BuildNativeJob.pkl], where we build Pkl automatically.

15
buildSrc/src/main/kotlin/BuildInfo.kt
View File

@@ -26,6 +26,7 @@ open class BuildInfo(project: Project) {
when {
os.isMacOsX -> "macos"
os.isLinux -> "linux"
os.isWindows -> "windows"
else -> throw RuntimeException("${os.familyName} is not supported.")
}
}
@@ -36,7 +37,8 @@ open class BuildInfo(project: Project) {
val downloadUrl: String by lazy {
val jdkMajor = graalVmJdkVersion.takeWhile { it != '.' }
"https://download.oracle.com/graalvm/$jdkMajor/archive/$baseName.tar.gz"
val extension = if (os.isWindows) "zip" else "tar.gz"
"https://download.oracle.com/graalvm/$jdkMajor/archive/$baseName.$extension"
}
val installDir: File by lazy {
@@ -85,9 +87,14 @@ open class BuildInfo(project: Project) {
val commitId: String by lazy {
// only run command once per build invocation
if (project === project.rootProject) {
Runtime.getRuntime()
.exec(arrayOf("git", "rev-parse", "--short", "HEAD"), arrayOf(), project.rootDir)
.inputStream.reader().readText().trim()
val process = ProcessBuilder()
.command("git", "rev-parse", "--short", "HEAD")
.directory(project.rootDir)
.start()
process.waitFor().also { exitCode ->
if (exitCode == -1) throw RuntimeException(process.errorStream.reader().readText())
}
process.inputStream.reader().readText().trim()
} else {
project.rootProject.extensions.getByType(BuildInfo::class.java).commitId
}

14
buildSrc/src/main/kotlin/ExecutableJar.kt
View File

@@ -1,11 +1,11 @@
import org.gradle.api.DefaultTask
import org.gradle.api.file.RegularFileProperty
import org.gradle.api.provider.ListProperty
import org.gradle.api.provider.Property
import org.gradle.api.tasks.Input
import org.gradle.api.tasks.InputFile
import org.gradle.api.tasks.OutputFile
import org.gradle.api.tasks.TaskAction
import org.gradle.kotlin.dsl.listProperty
/**
* Builds a self-contained Pkl CLI Jar that is directly executable on *nix
@@ -15,27 +15,25 @@ import org.gradle.kotlin.dsl.listProperty
*
* https://skife.org/java/unix/2011/06/20/really_executable_jars.html
*/
open class ExecutableJar : DefaultTask() {
abstract class ExecutableJar : DefaultTask() {
@get:InputFile
val inJar: RegularFileProperty = project.objects.fileProperty()
abstract val inJar: RegularFileProperty
@get:OutputFile
val outJar: RegularFileProperty = project.objects.fileProperty()
abstract val outJar: RegularFileProperty
@get:Input
val jvmArgs: ListProperty<String> = project.objects.listProperty()
abstract val jvmArgs: ListProperty<String>
@TaskAction
fun buildJar() {
val inFile = inJar.get().asFile
val outFile = outJar.get().asFile
val escapedJvmArgs = jvmArgs.get().joinToString(separator = " ") { "\"$it\"" }
val startScript = """
#!/bin/sh
exec java $escapedJvmArgs -jar $0 "$@"
""".trim().trimMargin() + "\n\n\n"
""".trimIndent() + "\n\n\n"
outFile.outputStream().use { outStream ->
startScript.byteInputStream().use { it.copyTo(outStream) }
inFile.inputStream().use { it.copyTo(outStream) }

25
buildSrc/src/main/kotlin/pklAllProjects.gradle.kts
View File

@@ -93,3 +93,28 @@ val updateDependencyLocks by tasks.registering {
}
val allDependencies by tasks.registering(DependencyReportTask::class)
tasks.withType(Test::class).configureEach {
System.getProperty("testReportsDir")?.let { reportsDir ->
reports.junitXml.outputLocation.set(file(reportsDir).resolve(project.name).resolve(name))
}
debugOptions {
enabled = System.getProperty("jvmdebug")?.toBoolean() ?: false
@Suppress("UnstableApiUsage")
host = "*"
port = 5005
suspend = true
server = true
}
}
tasks.withType(JavaExec::class).configureEach {
debugOptions {
enabled = System.getProperty("jvmdebug")?.toBoolean() ?: false
@Suppress("UnstableApiUsage")
host = "*"
port = 5005
suspend = true
server = true
}
}

20
buildSrc/src/main/kotlin/pklGraalVm.gradle.kts
View File

@@ -2,6 +2,7 @@ import java.nio.file.*
import java.util.UUID
import de.undercouch.gradle.tasks.download.Download
import de.undercouch.gradle.tasks.download.Verify
import kotlin.io.path.createDirectories
plugins {
id("de.undercouch.download")
@@ -9,7 +10,10 @@ plugins {
val buildInfo = project.extensions.getByType<BuildInfo>()
val BuildInfo.GraalVm.downloadFile get() = file(homeDir).resolve("${baseName}.tar.gz")
val BuildInfo.GraalVm.downloadFile get(): File {
val extension = if (buildInfo.os.isWindows) "zip" else "tar.gz"
return file(homeDir).resolve("${baseName}.$extension")
}
// tries to minimize chance of corruption by download-to-temp-file-and-move
val downloadGraalVmAarch64 by tasks.registering(Download::class) {
@@ -72,11 +76,10 @@ fun Task.configureInstallGraalVm(graalVm: BuildInfo.GraalVm) {
}
doLast {
val distroDir = "${graalVm.homeDir}/${UUID.randomUUID()}"
val distroDir = Paths.get(graalVm.homeDir, UUID.randomUUID().toString())
try {
mkdir(distroDir)
distroDir.createDirectories()
println("Extracting ${graalVm.downloadFile} into $distroDir")
// faster and more reliable than Gradle's `copy { from tarTree() }`
exec {
@@ -85,17 +88,18 @@ fun Task.configureInstallGraalVm(graalVm: BuildInfo.GraalVm) {
args("--strip-components=1", "-xzf", graalVm.downloadFile)
}
val distroBinDir = if (buildInfo.os.isMacOsX) "$distroDir/Contents/Home/bin" else "$distroDir/bin"
val distroBinDir = if (buildInfo.os.isMacOsX) distroDir.resolve("Contents/Home/bin") else distroDir.resolve("bin")
println("Installing native-image into $distroDir")
exec {
executable = "$distroBinDir/gu"
val executableName = if (buildInfo.os.isWindows) "gu.cmd" else "gu"
executable = distroBinDir.resolve(executableName).toString()
args("install", "--no-progress", "native-image")
}
println("Creating symlink ${graalVm.installDir} for $distroDir")
val tempLink = Paths.get("${graalVm.homeDir}/${UUID.randomUUID()}")
Files.createSymbolicLink(tempLink, Paths.get(distroDir))
val tempLink = Paths.get(graalVm.homeDir, UUID.randomUUID().toString())
Files.createSymbolicLink(tempLink, distroDir)
try {
Files.move(tempLink, graalVm.installDir.toPath(), StandardCopyOption.ATOMIC_MOVE)
} catch (e: Exception) {

13
docs/modules/language-reference/pages/index.adoc
View File

@@ -2140,9 +2140,18 @@ For example, a module with URI `modulepath:/animals/birds/pigeon.pkl`
can import `modulepath:/animals/birds/parrot.pkl`
with `import "parrot.pkl"` or `import "/animals/birds/parrot.pkl"`.
NOTE: When importing a relative folder or file that starts with `@`, the import string must be prefixed with `./`.
Otherwise, this syntax will be interpreted as dependency notation.
[NOTE]
.Paths on Windows
====
Relative paths use the `/` character as the directory separator on all platforms, including Windows.
Paths that contain drive letters (e.g. `C:`) must be declared as an absolute file URI, for example: `import "file:///C:/path/to/my/module.pkl"`. Otherwise, they are interpreted as a URI scheme.
====
NOTE: When importing a relative directory or file that starts with `@`, the import string must be prefixed with `./`.
Otherwise, this syntax will be interpreted as xref:dependency-notation[dependency notation].
[#dependency-notation]
==== Dependency notation URIs
Example: `+@birds/bird.pkl+`

55
docs/modules/pkl-cli/pages/index.adoc
View File

@@ -8,6 +8,7 @@ include::ROOT:partial$component-attributes.adoc[]
:uri-pkl-linux-amd64-download: {uri-sonatype-snapshot-download}&a=pkl-cli-linux-amd64&e=bin
:uri-pkl-linux-aarch64-download: {uri-sonatype-snapshot-download}&a=pkl-cli-linux-aarch64&e=bin
:uri-pkl-alpine-download: {uri-sonatype-snapshot-download}&a=pkl-cli-alpine-linux-amd64&e=bin
:uri-pkl-windows-download: {uri-sonatype-snapshot-download}&a=pkl-cli-windows-amd64&e=exe
:uri-pkl-java-download: {uri-sonatype-snapshot-download}&a=pkl-cli-java&e=jar
ifdef::is-release-version[]
@@ -16,6 +17,7 @@ ifdef::is-release-version[]
:uri-pkl-linux-amd64-download: {github-releases}/pkl-linux-amd64
:uri-pkl-linux-aarch64-download: {github-releases}/pkl-linux-aarch64
:uri-pkl-alpine-download: {github-releases}/pkl-alpine-linux-amd64
:uri-pkl-windows-download: {github-releases}/pkl-windows-amd64.exe
:uri-pkl-java-download: {uri-maven-repo}/org/pkl-lang/pkl-cli-java/{pkl-artifact-version}/pkl-cli-java-{pkl-artifact-version}.jar
endif::[]
@@ -37,9 +39,10 @@ The CLI comes in multiple flavors:
* Native Linux executable for amd64
* Native Linux executable for aarch64
* Native Alpine Linux executable for amd64 (cross-compiled and tested on Oracle Linux 8)
* Java executable (tested with Java 17/21 on macOS and Oracle Linux, may work on other platforms)
* Native Windows executable for amd64 (tested on Windows Server 2022)
* Java executable (tested with Java 17/21 on macOS and Oracle Linux)
On macOS and Linux, we recommend using the native executables.
On macOS, Linux, and Windows, we recommend using the native executables.
They are self-contained, start up instantly, and run complex Pkl code much faster than the Java executable.
.What is the Difference Between the Linux and Alpine Linux Executables?
@@ -59,7 +62,7 @@ Except where noted otherwise, the rest of this page discusses the native executa
[[homebrew]]
=== Homebrew
Release versions can be installed with {uri-homebrew}[Homebrew].
On macOS and Linux, release versions can be installed with {uri-homebrew}[Homebrew].
ifdef::is-release-version[]
To install Pkl, run:
@@ -111,7 +114,7 @@ chmod +x pkl
This should print something similar to:
[source,shell]
[source]
[subs="+attributes"]
----
Pkl {pkl-version} (macOS, native)
@@ -145,7 +148,7 @@ chmod +x pkl
This should print something similar to:
[source,shell]
[source]
[subs="+attributes"]
----
Pkl {pkl-version} (Linux, native)
@@ -167,7 +170,7 @@ chmod +x pkl
This should print something similar to:
[source,shell]
[source]
[subs="+attributes"]
----
Pkl {pkl-version} (Linux, native)
@@ -175,6 +178,23 @@ Pkl {pkl-version} (Linux, native)
NOTE: We currently do not support the aarch64 architecture for Alpine Linux.
=== Windows Executable
[source,PowerShell]
[subs="+attributes"]
----
Invoke-WebRequest '{uri-pkl-windows-download}' -OutFile pkl.exe
.\pkl --version
----
[source]
[subs="+attributes"]
----
Pkl {pkl-version} (Windows 10.0, native)
----
NOTE: We currently do not support the aarch64 architecture for Windows.
=== Java Executable
[source,shell]
@@ -193,27 +213,8 @@ This should print something similar to:
Pkl {pkl-version} (macOS 14.2, Java 17.0.10)
----
=== Windows support
Pkl does not currently support running natively on Windows. Pkl has been reported to work on the https://learn.microsoft.com/en-us/windows/wsl/install[Windows Subsystem for Linux] and on a https://www.oracle.com/java/technologies/downloads/#jdk21-windows[Java Runtime] using https://search.maven.org/remote_content?g=org.pkl-lang&a=pkl-cli-java&v=LATEST[`jpkl` (the Java executable version of Pkl)].
The following is from successful uses of `jpkl` on Windows:
[source,shell]
[subs="+attributes"]
----
> java -jar pkl-cli-java.jar --version
Pkl 0.26.0-dev+21e0e14 (Windows 10 10.0, Java 21.0.2)
----
Note: You must use forward slashes in all paths for Windows with absolute paths prefixed with file:///. The following examples are valid:
[source,shell]
[subs="+attributes"]
----
> java -jar pkl-cli-java.jar eval file:///C:/Code/pkl/test.pkl
> java -jar pkl-cli-java.jar eval ../Code/pkl/test.pkl
> java -jar pkl-cli-java.jar eval pkl/test.pkl
----
https://github.com/apple/pkl/issues/20[GitHub Issue #20] is used to track progress on support for the Windows platform.
NOTE: The Java executable does not work as an executable file on Windows.
However, it will work as a jar, for example, with `java -jar jpkl`.
[[usage]]
== Usage

1
gradle/libs.versions.toml
View File

@@ -15,6 +15,7 @@ graalVmSha256-macos-x64 = "14f4bd6417809905f86e786c779d0fc2feb840d7dac35ae3503eb
graalVmSha256-macos-aarch64 = "e944c5ce5da56e683fc8f1a57191b46d9cb702930b1688bda064fcf467d876b8"
graalVmSha256-linux-x64 = "112dc9b92d81a946f1b5b334646151b790785c813e76fcf13527a319003d7e2c"
graalVmSha256-linux-aarch64 = "c95ac550d070f06666cf8c1023a098380dd565be00866473caf6ff1b7cdf680c"
graalVmSha256-windows-x64 = "1ab2291e71f54d73e3e57b7fccbf184cabcba37e16ca9d1cf42d08474a7c02f0"
ideaExtPlugin = "1.1"
javaPoet = "1.+"
javaxInject = "1"

49
pkl-cli/pkl-cli.gradle.kts
View File

@@ -33,6 +33,7 @@ val stagedMacAarch64Executable: Configuration by configurations.creating
val stagedLinuxAmd64Executable: Configuration by configurations.creating
val stagedLinuxAarch64Executable: Configuration by configurations.creating
val stagedAlpineLinuxAmd64Executable: Configuration by configurations.creating
val stagedWindowsAmd64Executable: Configuration by configurations.creating
dependencies {
compileOnly(libs.svm)
@@ -63,6 +64,7 @@ dependencies {
stagedLinuxAmd64Executable(executableDir("pkl-linux-amd64"))
stagedLinuxAarch64Executable(executableDir("pkl-linux-aarch64"))
stagedAlpineLinuxAmd64Executable(executableDir("pkl-alpine-linux-amd64"))
stagedWindowsAmd64Executable(executableDir("pkl-windows-amd64.exe"))
}
tasks.jar {
@@ -121,12 +123,17 @@ val testStartJavaExecutable by tasks.registering(Exec::class) {
dependsOn(javaExecutable)
val outputFile = layout.buildDirectory.file("testStartJavaExecutable") // dummy output to satisfy up-to-date check
outputs.file(outputFile)
executable = javaExecutable.get().outputs.files.singleFile.toString()
args("--version")
if (buildInfo.os.isWindows) {
executable = "java"
args("-jar", javaExecutable.get().outputs.files.singleFile.toString(), "--version")
} else {
executable = javaExecutable.get().outputs.files.singleFile.toString()
args("--version")
}
doFirst { outputFile.get().asFile.delete() }
doLast { outputFile.get().asFile.writeText("OK") }
}
@@ -141,12 +148,13 @@ fun Exec.configureExecutable(
) {
inputs.files(sourceSets.main.map { it.output }).withPropertyName("mainSourceSets").withPathSensitivity(PathSensitivity.RELATIVE)
inputs.files(configurations.runtimeClasspath).withPropertyName("runtimeClasspath").withNormalizer(ClasspathNormalizer::class)
inputs.files(file(graalVm.baseDir).resolve("bin/native-image")).withPropertyName("graalVmNativeImage").withPathSensitivity(PathSensitivity.ABSOLUTE)
val nativeImageCommandName = if (buildInfo.os.isWindows) "native-image.cmd" else "native-image"
inputs.files(file(graalVm.baseDir).resolve("bin/$nativeImageCommandName")).withPropertyName("graalVmNativeImage").withPathSensitivity(PathSensitivity.ABSOLUTE)
outputs.file(outputFile)
outputs.cacheIf { true }
workingDir(outputFile.map { it.asFile.parentFile })
executable = "${graalVm.baseDir}/bin/native-image"
executable = "${graalVm.baseDir}/bin/$nativeImageCommandName"
// JARs to exclude from the class path for the native-image build.
val exclusions = listOf(libs.truffleApi, libs.graalSdk).map { it.get().module.name }
@@ -276,6 +284,15 @@ val alpineExecutableAmd64: TaskProvider<Exec> by tasks.registering(Exec::class)
)
}
val windowsExecutableAmd64: TaskProvider<Exec> by tasks.registering(Exec::class) {
dependsOn(":installGraalVmAmd64")
configureExecutable(
buildInfo.graalVmAmd64,
layout.buildDirectory.file("executable/pkl-windows-amd64"),
listOf("-Dfile.encoding=UTF-8")
)
}
tasks.assembleNative {
when {
buildInfo.os.isMacOsX -> {
@@ -284,6 +301,9 @@ tasks.assembleNative {
dependsOn(macExecutableAarch64)
}
}
buildInfo.os.isWindows -> {
dependsOn(windowsExecutableAmd64)
}
buildInfo.os.isLinux && buildInfo.arch == "aarch64" -> {
dependsOn(linuxExecutableAarch64)
}
@@ -393,6 +413,20 @@ publishing {
description.set("Native Pkl CLI executable for linux/amd64 and statically linked to musl.")
}
}
create<MavenPublication>("windowsExecutableAmd64") {
artifactId = "pkl-cli-windows-amd64"
artifact(stagedWindowsAmd64Executable.singleFile) {
classifier = null
extension = "exe"
builtBy(stagedWindowsAmd64Executable)
}
pom {
name.set("pkl-cli-windows-amd64")
url.set("https://github.com/apple/pkl/tree/main/pkl-cli")
description.set("Native Pkl CLI executable for windows/amd64.")
}
}
}
}
@@ -403,5 +437,6 @@ signing {
sign(publishing.publications["macExecutableAarch64"])
sign(publishing.publications["macExecutableAmd64"])
sign(publishing.publications["alpineLinuxExecutableAmd64"])
sign(publishing.publications["windowsExecutableAmd64"])
}
//endregion

20
pkl-cli/src/main/kotlin/org/pkl/cli/CliEvaluator.kt
View File

@@ -111,7 +111,9 @@ constructor(
return moduleUris.associateWith { uri ->
val moduleDir: String? =
IoUtils.toPath(uri)?.let { workingDir.relativize(it.parent).toString().ifEmpty { "." } }
IoUtils.toPath(uri)?.let {
IoUtils.relativize(it.parent, workingDir).toString().ifEmpty { "." }
}
val moduleKey =
try {
moduleResolver.resolve(uri)
@@ -158,7 +160,7 @@ constructor(
} else {
if (output.isNotEmpty()) {
outputFile.writeString(
options.moduleOutputSeparator + IoUtils.getLineSeparator(),
options.moduleOutputSeparator + '\n',
Charsets.UTF_8,
StandardOpenOption.WRITE,
StandardOpenOption.APPEND
@@ -192,6 +194,14 @@ constructor(
if (uri == VmUtils.REPL_TEXT_URI) ModuleSource.create(uri, reader.readText())
else ModuleSource.uri(uri)
private fun checkPathSpec(pathSpec: String) {
val illegal = pathSpec.indexOfFirst { IoUtils.isReservedFilenameChar(it) && it != '/' }
if (illegal == -1) {
return
}
throw CliException("Path spec `$pathSpec` contains illegal character `${pathSpec[illegal]}`.")
}
/**
* Renders each module's `output.files`, writing each entry as a file into the specified output
* directory.
@@ -207,6 +217,7 @@ constructor(
val moduleSource = toModuleSource(moduleUri, consoleReader)
val output = evaluator.evaluateOutputFiles(moduleSource)
for ((pathSpec, fileOutput) in output) {
checkPathSpec(pathSpec)
val resolvedPath = outputDir.resolve(pathSpec).normalize()
val realPath = if (resolvedPath.exists()) resolvedPath.toRealPath() else resolvedPath
if (!realPath.startsWith(outputDir)) {
@@ -228,7 +239,10 @@ constructor(
writtenFiles[realPath] = OutputFile(pathSpec, moduleUri)
realPath.createParentDirectories()
realPath.writeString(fileOutput.text)
consoleWriter.write(currentWorkingDir.relativize(resolvedPath).toString() + "\n")
consoleWriter.write(
IoUtils.relativize(resolvedPath, currentWorkingDir).toString() +
IoUtils.getLineSeparator()
)
consoleWriter.flush()
}
}

6
pkl-cli/src/main/kotlin/org/pkl/cli/CliPackageDownloader.kt
View File

@@ -42,7 +42,11 @@ class CliPackageDownloader(
}
when (errors.size) {
0 -> return
1 -> throw CliException(errors.values.single().message!!)
1 ->
throw CliException(
errors.values.single().message
?: ("An unexpected error occurred: " + errors.values.single())
)
else ->
throw CliException(
buildString {

3
pkl-cli/src/main/kotlin/org/pkl/cli/commands/TestCommand.kt
View File

@@ -22,6 +22,7 @@ import com.github.ajalt.clikt.parameters.groups.provideDelegate
import java.net.URI
import org.pkl.cli.CliTestRunner
import org.pkl.commons.cli.commands.BaseCommand
import org.pkl.commons.cli.commands.BaseOptions
import org.pkl.commons.cli.commands.ProjectOptions
import org.pkl.commons.cli.commands.TestOptions
@@ -29,7 +30,7 @@ class TestCommand(helpLink: String) :
BaseCommand(name = "test", help = "Run tests within the given module(s)", helpLink = helpLink) {
val modules: List<URI> by
argument(name = "<modules>", help = "Module paths or URIs to evaluate.")
.convert { parseModuleName(it) }
.convert { BaseOptions.parseModuleName(it) }
.multiple()
private val projectOptions by ProjectOptions()

56
pkl-cli/src/test/kotlin/org/pkl/cli/CliEvaluatorTest.kt
View File

@@ -28,6 +28,9 @@ import org.junit.jupiter.api.AfterAll
import org.junit.jupiter.api.AfterEach
import org.junit.jupiter.api.Test
import org.junit.jupiter.api.assertThrows
import org.junit.jupiter.api.condition.DisabledOnOs
import org.junit.jupiter.api.condition.EnabledOnOs
import org.junit.jupiter.api.condition.OS
import org.junit.jupiter.api.io.TempDir
import org.junit.jupiter.params.ParameterizedTest
import org.junit.jupiter.params.provider.EnumSource
@@ -424,7 +427,10 @@ result = someLib.x
checkOutputFile(outputFiles[0], "result.pcf", contents)
}
// Can't reliably create symlinks on Windows.
// Might get errors like "A required privilege is not held by the client".
@Test
@DisabledOnOs(OS.WINDOWS)
fun `moduleDir is relative to workingDir even through symlinks`() {
val contents = "foo = 42"
val realWorkingDir = tempDir.resolve("workingDir").createDirectories()
@@ -978,6 +984,56 @@ result = someLib.x
.hasMessageContaining("resolve to the same file path")
}
@Test
@EnabledOnOs(OS.WINDOWS)
fun `multiple-file output throws when using invalid Windows characters`() {
val moduleUri =
writePklFile(
"test.pkl",
"""
output {
files {
["foo:bar"] { text = "bar" }
}
}
"""
.trimIndent()
)
val options =
CliEvaluatorOptions(
CliBaseOptions(sourceModules = listOf(moduleUri), workingDir = tempDir),
multipleFileOutputPath = ".output"
)
assertThatCode { evalToConsole(options) }
.hasMessageContaining("Path spec `foo:bar` contains illegal character `:`.")
}
@Test
@EnabledOnOs(OS.WINDOWS)
fun `multiple-file output - cannot use backslash as dir separator on Windows`() {
val moduleUri =
writePklFile(
"test.pkl",
"""
output {
files {
["foo\\bar"] { text = "bar" }
}
}
"""
.trimIndent()
)
val options =
CliEvaluatorOptions(
CliBaseOptions(sourceModules = listOf(moduleUri), workingDir = tempDir),
multipleFileOutputPath = ".output"
)
assertThatCode { evalToConsole(options) }
.hasMessageContaining("Path spec `foo\\bar` contains illegal character `\\`.")
}
@Test
fun `evaluate output expression`() {
val moduleUri =

5
pkl-cli/src/test/kotlin/org/pkl/cli/CliMainTest.kt
View File

@@ -24,6 +24,8 @@ import org.assertj.core.api.Assertions.assertThat
import org.assertj.core.api.AssertionsForClassTypes.assertThatCode
import org.junit.jupiter.api.Test
import org.junit.jupiter.api.assertThrows
import org.junit.jupiter.api.condition.DisabledOnOs
import org.junit.jupiter.api.condition.OS
import org.junit.jupiter.api.io.TempDir
import org.pkl.cli.commands.EvalCommand
import org.pkl.cli.commands.RootCommand
@@ -54,7 +56,10 @@ class CliMainTest {
assertThatCode { cmd.parse(arrayOf("eval")) }.hasMessage("""Missing argument "<modules>"""")
}
// Can't reliably create symlinks on Windows.
// Might get errors like "A required privilege is not held by the client".
@Test
@DisabledOnOs(OS.WINDOWS)
fun `output to symlinked directory works`(@TempDir tempDir: Path) {
val code =
"""

41
pkl-cli/src/test/kotlin/org/pkl/cli/CliProjectPackagerTest.kt
View File

@@ -15,6 +15,7 @@
*/
package org.pkl.cli
import java.io.File
import java.io.StringWriter
import java.net.URI
import java.nio.file.FileSystems
@@ -27,6 +28,8 @@ import org.assertj.core.api.Assertions.assertThatCode
import org.junit.jupiter.api.AfterAll
import org.junit.jupiter.api.Test
import org.junit.jupiter.api.assertThrows
import org.junit.jupiter.api.condition.DisabledOnOs
import org.junit.jupiter.api.condition.OS
import org.junit.jupiter.api.io.TempDir
import org.pkl.commons.cli.CliBaseOptions
import org.pkl.commons.cli.CliException
@@ -35,6 +38,7 @@ import org.pkl.commons.readString
import org.pkl.commons.test.FileTestUtils
import org.pkl.commons.test.PackageServer
import org.pkl.commons.writeString
import org.pkl.core.util.IoUtils
class CliProjectPackagerTest {
companion object {
@@ -690,7 +694,11 @@ class CliProjectPackagerTest {
)
}
// Absolute path imports on Windows must use an absolute URI (e.g. file:///C:/Foo/Bar), because
// they must contain drive letters, which conflict with schemes.
// We skip validation for absolute URIs, so effectively we skip this check on Windows.
@Test
@DisabledOnOs(OS.WINDOWS)
fun `import path verification -- absolute import from root dir`(@TempDir tempDir: Path) {
tempDir.writeFile(
"main.pkl",
@@ -738,6 +746,7 @@ class CliProjectPackagerTest {
}
@Test
@DisabledOnOs(OS.WINDOWS)
fun `import path verification -- absolute read from root dir`(@TempDir tempDir: Path) {
tempDir.writeFile(
"main.pkl",
@@ -858,17 +867,18 @@ class CliProjectPackagerTest {
consoleWriter = out
)
.run()
val sep = File.separatorChar
assertThat(out.toString())
.isEqualTo(
.isEqualToNormalizingNewlines(
"""
.out/project1@1.0.0/project1@1.0.0.zip
.out/project1@1.0.0/project1@1.0.0.zip.sha256
.out/project1@1.0.0/project1@1.0.0
.out/project1@1.0.0/project1@1.0.0.sha256
.out/project2@2.0.0/project2@2.0.0.zip
.out/project2@2.0.0/project2@2.0.0.zip.sha256
.out/project2@2.0.0/project2@2.0.0
.out/project2@2.0.0/project2@2.0.0.sha256
.out${sep}project1@1.0.0${sep}project1@1.0.0.zip
.out${sep}project1@1.0.0${sep}project1@1.0.0.zip.sha256
.out${sep}project1@1.0.0${sep}project1@1.0.0
.out${sep}project1@1.0.0${sep}project1@1.0.0.sha256
.out${sep}project2@2.0.0${sep}project2@2.0.0.zip
.out${sep}project2@2.0.0${sep}project2@2.0.0.zip.sha256
.out${sep}project2@2.0.0${sep}project2@2.0.0
.out${sep}project2@2.0.0${sep}project2@2.0.0.sha256
"""
.trimIndent()
@@ -956,13 +966,14 @@ class CliProjectPackagerTest {
consoleWriter = out
)
.run()
val sep = File.separatorChar
assertThat(out.toString())
.isEqualTo(
.isEqualToNormalizingNewlines(
"""
.out/mangos@1.0.0/mangos@1.0.0.zip
.out/mangos@1.0.0/mangos@1.0.0.zip.sha256
.out/mangos@1.0.0/mangos@1.0.0
.out/mangos@1.0.0/mangos@1.0.0.sha256
.out${sep}mangos@1.0.0${sep}mangos@1.0.0.zip
.out${sep}mangos@1.0.0${sep}mangos@1.0.0.zip.sha256
.out${sep}mangos@1.0.0${sep}mangos@1.0.0
.out${sep}mangos@1.0.0${sep}mangos@1.0.0.sha256
"""
.trimIndent()
@@ -971,7 +982,7 @@ class CliProjectPackagerTest {
private fun Path.zipFilePaths(): List<String> {
return FileSystems.newFileSystem(URI("jar:${toUri()}"), emptyMap<String, String>()).use { fs ->
Files.walk(fs.getPath("/")).map { it.toString() }.collect(Collectors.toList())
Files.walk(fs.getPath("/")).map(IoUtils::toNormalizedPathString).collect(Collectors.toList())
}
}
}

11
pkl-cli/src/test/kotlin/org/pkl/cli/CliProjectResolverTest.kt
View File

@@ -15,6 +15,7 @@
*/
package org.pkl.cli
import java.io.File
import java.io.StringWriter
import java.nio.file.Path
import org.assertj.core.api.Assertions.assertThat
@@ -26,6 +27,7 @@ import org.pkl.commons.cli.CliBaseOptions
import org.pkl.commons.cli.CliException
import org.pkl.commons.test.FileTestUtils
import org.pkl.commons.test.PackageServer
import org.pkl.core.util.IoUtils
class CliProjectResolverTest {
companion object {
@@ -354,7 +356,7 @@ class CliProjectResolverTest {
)
assertThat(errOut.toString())
.isEqualTo(
"WARN: local dependency `package://localhost:0/fruit@1.0.0` was overridden to remote dependency `package://localhost:0/fruit@1.0.5`.\n"
"WARN: local dependency `package://localhost:0/fruit@1.0.0` was overridden to remote dependency `package://localhost:0/fruit@1.0.5`.${IoUtils.getLineSeparator()}"
)
}
@@ -401,11 +403,12 @@ class CliProjectResolverTest {
errWriter = errOut
)
.run()
val sep = File.separatorChar
assertThat(consoleOut.toString())
.isEqualTo(
.isEqualToNormalizingNewlines(
"""
$tempDir/project1/PklProject.deps.json
$tempDir/project2/PklProject.deps.json
$tempDir${sep}project1${sep}PklProject.deps.json
$tempDir${sep}project2${sep}PklProject.deps.json
"""
.trimIndent()

31
pkl-commons-cli/src/main/kotlin/org/pkl/commons/cli/commands/BaseCommand.kt
View File

@@ -17,11 +17,6 @@ package org.pkl.commons.cli.commands
import com.github.ajalt.clikt.core.CliktCommand
import com.github.ajalt.clikt.parameters.groups.provideDelegate
import java.net.URI
import java.net.URISyntaxException
import org.pkl.commons.cli.CliException
import org.pkl.core.runtime.VmUtils
import org.pkl.core.util.IoUtils
abstract class BaseCommand(name: String, helpLink: String, help: String = "") :
CliktCommand(
@@ -30,30 +25,4 @@ abstract class BaseCommand(name: String, helpLink: String, help: String = "") :
epilog = "For more information, visit $helpLink",
) {
val baseOptions by BaseOptions()
/**
* Parses [moduleName] into a URI. If scheme is not present, we expect that this is a file path
* and encode any possibly invalid characters. If a scheme is present, we expect that this is a
* valid URI.
*/
protected fun parseModuleName(moduleName: String): URI =
when (moduleName) {
"-" -> VmUtils.REPL_TEXT_URI
else ->
try {
IoUtils.toUri(moduleName)
} catch (e: URISyntaxException) {
val message = buildString {
append("Module URI `$moduleName` has invalid syntax (${e.reason}).")
if (e.index > -1) {
append("\n\n")
append(moduleName)
append("\n")
append(" ".repeat(e.index))
append("^")
}
}
throw CliException(message)
}
}
}

38
pkl-commons-cli/src/main/kotlin/org/pkl/commons/cli/commands/BaseOptions.kt
View File

@@ -22,14 +22,50 @@ import com.github.ajalt.clikt.parameters.types.long
import com.github.ajalt.clikt.parameters.types.path
import java.io.File
import java.net.URI
import java.net.URISyntaxException
import java.nio.file.Path
import java.time.Duration
import java.util.regex.Pattern
import org.pkl.commons.cli.CliBaseOptions
import org.pkl.commons.cli.CliException
import org.pkl.core.runtime.VmUtils
import org.pkl.core.util.IoUtils
@Suppress("MemberVisibilityCanBePrivate")
class BaseOptions : OptionGroup() {
companion object {
/**
* Parses [moduleName] into a URI. If scheme is not present, we expect that this is a file path
* and encode any possibly invalid characters, and also normalize directory separators. If a
* scheme is present, we expect that this is a valid URI.
*/
fun parseModuleName(moduleName: String): URI =
when (moduleName) {
"-" -> VmUtils.REPL_TEXT_URI
else ->
// Don't use `IoUtils.toUri` here becaus we need to normalize `\` paths to `/` on Windows.
try {
if (IoUtils.isUriLike(moduleName)) URI(moduleName)
// Can't just use URI constructor, because URI(null, null, "C:/foo/bar", null) turns
// into `URI("C", null, "/foo/bar", null)`.
else if (IoUtils.isWindowsAbsolutePath(moduleName)) Path.of(moduleName).toUri()
else URI(null, null, IoUtils.toNormalizedPathString(Path.of(moduleName)), null)
} catch (e: URISyntaxException) {
val message = buildString {
append("Module URI `$moduleName` has invalid syntax (${e.reason}).")
if (e.index > -1) {
append("\n\n")
append(moduleName)
append("\n")
append(" ".repeat(e.index))
append("^")
}
}
throw CliException(message)
}
}
}
private val defaults = CliBaseOptions()
private val output =
@@ -114,7 +150,7 @@ class BaseOptions : OptionGroup() {
val settings: URI? by
option(names = arrayOf("--settings"), help = "Pkl settings module to use.").single().convert {
IoUtils.toUri(it)
parseModuleName(it)
}
val timeout: Duration? by

2
pkl-commons-cli/src/main/kotlin/org/pkl/commons/cli/commands/ModulesCommand.kt
View File

@@ -29,7 +29,7 @@ abstract class ModulesCommand(name: String, helpLink: String, help: String = "")
) {
open val modules: List<URI> by
argument(name = "<modules>", help = "Module paths or URIs to evaluate.")
.convert { parseModuleName(it) }
.convert { BaseOptions.parseModuleName(it) }
.multiple(required = true)
protected val projectOptions by ProjectOptions()

5
pkl-commons-test/pkl-commons-test.gradle.kts
View File

@@ -58,6 +58,11 @@ for (packageDir in file("src/main/files/packages").listFiles()!!) {
}
doLast {
val outputFile = destinationDir.get().asFile.resolve("${packageDir.name}.json")
if (buildInfo.os.isWindows) {
val contents = outputFile.readText()
// workaround for https://github.com/gradle/gradle/issues/1151
outputFile.writeText(contents.replace("\r\n", "\n"))
}
shasumFile.get().asFile.writeText(outputFile.computeChecksum())
}
}

3
pkl-commons-test/src/main/kotlin/org/pkl/commons/test/InputOutputTestEngine.kt
View File

@@ -31,6 +31,7 @@ import org.junit.platform.engine.support.hierarchical.EngineExecutionContext
import org.junit.platform.engine.support.hierarchical.HierarchicalTestEngine
import org.junit.platform.engine.support.hierarchical.Node
import org.junit.platform.engine.support.hierarchical.Node.DynamicTestExecutor
import org.pkl.commons.toNormalizedPathString
abstract class InputOutputTestEngine :
HierarchicalTestEngine<InputOutputTestEngine.ExecutionContext>() {
@@ -106,7 +107,7 @@ abstract class InputOutputTestEngine :
): TestDescriptor {
dirNode.inputDir.useDirectoryEntries { children ->
for (child in children) {
val testPath = child.toString()
val testPath = child.toNormalizedPathString()
val testName = child.fileName.toString()
if (child.isRegularFile()) {
if (

10
pkl-commons/src/main/kotlin/org/pkl/commons/Paths.kt
View File

@@ -71,3 +71,13 @@ fun Path.deleteRecursively() {
walk().use { paths -> paths.sorted(Comparator.reverseOrder()).forEach { it.deleteIfExists() } }
}
}
private val isWindows by lazy { System.getProperty("os.name").contains("Windows") }
/** Copy implementation from IoUtils.toNormalizedPathString */
fun Path.toNormalizedPathString(): String {
if (isWindows) {
return toString().replace("\\", "/")
}
return toString()
}

19
pkl-commons/src/main/kotlin/org/pkl/commons/Strings.kt
View File

@@ -15,15 +15,24 @@
*/
package org.pkl.commons
import java.io.File
import java.net.URI
import java.nio.file.Path
import java.util.regex.Pattern
fun String.toPath(): Path = Path.of(this)
private val uriLike = Pattern.compile("\\w+:[^\\\\].*")
private val windowsPathLike = Pattern.compile("\\w:\\\\.*")
/** Copy of org.pkl.core.util.IoUtils.toUri */
fun String.toUri(): URI =
if (contains(":")) {
URI(this)
} else {
URI(null, null, this, null)
fun String.toUri(): URI {
if (uriLike.matcher(this).matches()) {
return URI(this)
}
if (windowsPathLike.matcher(this).matches()) {
return File(this).toURI()
}
return URI(null, null, this, null)
}

18
pkl-core/pkl-core.gradle.kts
View File

@@ -267,6 +267,21 @@ val testAlpineExecutableAmd64 by tasks.registering(Test::class) {
}
}
val testWindowsExecutableAmd64 by tasks.registering(Test::class) {
dependsOn(":pkl-cli:windowsExecutableAmd64")
inputs.dir("src/test/files/LanguageSnippetTests/input")
inputs.dir("src/test/files/LanguageSnippetTests/input-helper")
inputs.dir("src/test/files/LanguageSnippetTests/output")
testClassesDirs = files(tasks.test.get().testClassesDirs)
classpath = tasks.test.get().classpath
useJUnitPlatform {
includeEngines("WindowsLanguageSnippetTestsEngine")
}
}
tasks.testNative {
when {
buildInfo.os.isMacOsX -> {
@@ -284,6 +299,9 @@ tasks.testNative {
dependsOn(testAlpineExecutableAmd64)
}
}
buildInfo.os.isWindows -> {
dependsOn(testWindowsExecutableAmd64)
}
}
}

1
pkl-core/src/main/java/org/pkl/core/Platform.java
View File

@@ -31,6 +31,7 @@ public final class Platform {
var pklVersion = Release.current().version().toString();
var osName = System.getProperty("os.name");
if (osName.equals("Mac OS X")) osName = "macOS";
if (osName.contains("Windows")) osName = "Windows";
var osVersion = System.getProperty("os.version");
var architecture = System.getProperty("os.arch");

1
pkl-core/src/main/java/org/pkl/core/Release.java
View File

@@ -49,6 +49,7 @@ public final class Release {
var commitId = properties.getProperty("commitId");
var osName = System.getProperty("os.name");
if (osName.equals("Mac OS X")) osName = "macOS";
if (osName.contains("Windows")) osName = "Windows";
var osVersion = System.getProperty("os.version");
var os = osName + " " + osVersion;
var flavor = TruffleOptions.AOT ? "native" : "Java " + System.getProperty("java.version");

15
pkl-core/src/main/java/org/pkl/core/ast/builder/AstBuilder.java
View File

@@ -1794,10 +1794,17 @@ public final class AstBuilder extends AbstractAstBuilder<Object> {
try {
resolvedUri = IoUtils.resolve(context.getSecurityManager(), moduleKey, parsedUri);
} catch (FileNotFoundException e) {
throw exceptionBuilder()
.evalError("cannotFindModule", importUri)
.withSourceSection(createSourceSection(importUriCtx))
.build();
var exceptionBuilder =
exceptionBuilder()
.evalError("cannotFindModule", importUri)
.withSourceSection(createSourceSection(importUriCtx));
var path = parsedUri.getPath();
if (path != null && path.contains("\\")) {
exceptionBuilder.withHint(
"To resolve modules in nested directories, use `/` as the directory separator.");
}
throw exceptionBuilder.build();
} catch (URISyntaxException e) {
throw exceptionBuilder()
.evalError("invalidModuleUri", importUri)

21
pkl-core/src/main/java/org/pkl/core/module/ModuleKeyFactories.java
View File

@@ -18,8 +18,8 @@ package org.pkl.core.module;
import java.net.URI;
import java.net.URISyntaxException;
import java.nio.file.FileSystemNotFoundException;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.spi.FileSystemProvider;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
@@ -141,15 +141,20 @@ public final class ModuleKeyFactories {
private static class File implements ModuleKeyFactory {
@Override
public Optional<ModuleKey> create(URI uri) {
Path path;
try {
path = Path.of(uri);
} catch (FileSystemNotFoundException | IllegalArgumentException e) {
// none of the installed file system providers can handle this URI
// skip loading providers if the scheme is `file`.
if (uri.getScheme().equalsIgnoreCase("file")) {
return Optional.of(ModuleKeys.file(uri));
}
// don't handle jar-file URIs (these are handled by GenericUrl).
if (uri.getScheme().equalsIgnoreCase("jar")) {
return Optional.empty();
}
return Optional.of(ModuleKeys.file(uri, path));
for (FileSystemProvider provider : FileSystemProvider.installedProviders()) {
if (provider.getScheme().equalsIgnoreCase(uri.getScheme())) {
return Optional.of(ModuleKeys.file(uri));
}
}
return Optional.empty();
}
}

26
pkl-core/src/main/java/org/pkl/core/module/ModuleKeys.java
View File

@@ -16,9 +16,11 @@
package org.pkl.core.module;
import com.oracle.truffle.api.CompilerDirectives.TruffleBoundary;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
import java.net.JarURLConnection;
import java.net.URI;
import java.net.URISyntaxException;
import java.net.http.HttpRequest;
@@ -88,8 +90,8 @@ public final class ModuleKeys {
}
/** Creates a module key for a {@code file:} module. */
public static ModuleKey file(URI uri, Path path) {
return new File(uri, path);
public static ModuleKey file(URI uri) {
return new File(uri);
}
/**
@@ -290,12 +292,10 @@ public final class ModuleKeys {
private static class File extends DependencyAwareModuleKey {
final URI uri;
final Path path;
File(URI uri, Path path) {
File(URI uri) {
super(uri);
this.uri = uri;
this.path = path;
}
@Override
@@ -316,7 +316,13 @@ public final class ModuleKeys {
public ResolvedModuleKey resolve(SecurityManager securityManager)
throws IOException, SecurityManagerException {
securityManager.checkResolveModule(uri);
var realPath = path.toRealPath();
// Disallow paths that contain `\\` characters if on Windows
// (require `/` as the path separator on all OSes)
var uriPath = uri.getPath();
if (java.io.File.separatorChar == '\\' && uriPath != null && uriPath.contains("\\")) {
throw new FileNotFoundException();
}
var realPath = Path.of(uri).toRealPath();
var resolvedUri = realPath.toUri();
securityManager.checkResolveModule(resolvedUri);
return ResolvedModuleKeys.file(this, resolvedUri, realPath);
@@ -325,7 +331,7 @@ public final class ModuleKeys {
@Override
protected Map<String, ? extends Dependency> getDependencies() {
var projectDepsManager = VmContext.get(null).getProjectDependenciesManager();
if (projectDepsManager == null || !projectDepsManager.hasPath(path)) {
if (projectDepsManager == null || !projectDepsManager.hasPath(Path.of(uri))) {
throw new PackageLoadError("cannotResolveDependencyNoProject");
}
return projectDepsManager.getDependencies();
@@ -519,6 +525,12 @@ public final class ModuleKeys {
var url = IoUtils.toUrl(uri);
var conn = url.openConnection();
conn.connect();
if (conn instanceof JarURLConnection && IoUtils.isWindows()) {
// On Windows, opening a JarURLConnection prevents the jar file from being deleted, unless
// cacheing is disabled.
// See https://bugs.openjdk.org/browse/JDK-8239054
conn.setUseCaches(false);
}
try (InputStream stream = conn.getInputStream()) {
URI redirected;
try {

5
pkl-core/src/main/java/org/pkl/core/module/ModulePathResolver.java
View File

@@ -30,6 +30,7 @@ import java.util.Map;
import javax.annotation.concurrent.GuardedBy;
import org.pkl.core.module.PathElement.TreePathElement;
import org.pkl.core.runtime.FileSystemManager;
import org.pkl.core.util.IoUtils;
import org.pkl.core.util.LateInit;
/**
@@ -152,8 +153,8 @@ public final class ModulePathResolver implements AutoCloseable {
// in case of duplicate path, first entry wins (cf. class loader)
stream.forEach(
(path) -> {
var relativized = basePath.relativize(path);
fileCache.putIfAbsent(relativized.toString(), path);
var relativized = IoUtils.relativize(path, basePath);
fileCache.putIfAbsent(IoUtils.toNormalizedPathString(relativized), path);
var element = cachedPathElementRoot;
for (var i = 0; i < relativized.getNameCount(); i++) {
var name = relativized.getName(i).toString();

6
pkl-core/src/main/java/org/pkl/core/module/ProjectDependenciesManager.java
View File

@@ -207,13 +207,15 @@ public final class ProjectDependenciesManager {
if (projectDeps == null) {
var depsPath = getProjectDepsFile();
if (!Files.exists(depsPath)) {
throw new VmExceptionBuilder().evalError("missingProjectDepsJson", projectDir).build();
throw new VmExceptionBuilder()
.evalError("missingProjectDepsJson", projectDir.toUri())
.build();
}
try {
projectDeps = ProjectDeps.parse(depsPath);
} catch (IOException | URISyntaxException | JsonParseException e) {
throw new VmExceptionBuilder()
.evalError("invalidProjectDepsJson", depsPath, e.getMessage())
.evalError("invalidProjectDepsJson", depsPath.toUri(), e.getMessage())
.withCause(e)
.build();
}

13
pkl-core/src/main/java/org/pkl/core/module/ResolvedModuleKeys.java
View File

@@ -15,10 +15,12 @@
*/
package org.pkl.core.module;
import java.io.File;
import java.io.IOException;
import java.net.URI;
import java.net.URL;
import java.nio.charset.StandardCharsets;
import java.nio.file.AccessDeniedException;
import java.nio.file.Files;
import java.nio.file.Path;
import org.pkl.core.util.IoUtils;
@@ -75,7 +77,16 @@ public final class ResolvedModuleKeys {
@Override
public String loadSource() throws IOException {
return Files.readString(path, StandardCharsets.UTF_8);
try {
return Files.readString(path, StandardCharsets.UTF_8);
} catch (AccessDeniedException e) {
// Windows throws `AccessDeniedException` when reading directories.
// Sync error between different OSes.
if (Files.isDirectory(path)) {
throw new IOException("Is a directory");
}
throw e;
}
}
}

2
pkl-core/src/main/java/org/pkl/core/packages/Dependency.java
View File

@@ -50,7 +50,7 @@ public abstract class Dependency {
public Path resolveAssetPath(Path projectDir, PackageAssetUri packageAssetUri) {
// drop 1 to remove leading `/`
var assetPath = packageAssetUri.getAssetPath().toString().substring(1);
var assetPath = packageAssetUri.getAssetPath().substring(1);
return projectDir.resolve(path).resolve(assetPath);
}

12
pkl-core/src/main/java/org/pkl/core/packages/PackageAssetUri.java
View File

@@ -20,6 +20,7 @@ import java.net.URISyntaxException;
import java.nio.file.Path;
import org.pkl.core.Version;
import org.pkl.core.util.ErrorMessages;
import org.pkl.core.util.IoUtils;
/**
* The canonical URI of an asset within a package, i.e., a package URI with a fragment path. For
@@ -28,7 +29,7 @@ import org.pkl.core.util.ErrorMessages;
public final class PackageAssetUri {
private final URI uri;
private final PackageUri packageUri;
private final Path assetPath;
private final String assetPath;
public static PackageAssetUri create(URI uri) {
try {
@@ -41,7 +42,7 @@ public final class PackageAssetUri {
public PackageAssetUri(PackageUri packageUri, String assetPath) {
this.uri = packageUri.getUri().resolve("#" + assetPath);
this.packageUri = packageUri;
this.assetPath = Path.of(assetPath);
this.assetPath = assetPath;
}
public PackageAssetUri(String uri) throws URISyntaxException {
@@ -60,7 +61,7 @@ public final class PackageAssetUri {
throw new URISyntaxException(
uri.toString(), ErrorMessages.create("cannotHaveRelativeFragment", fragment, uri));
}
this.assetPath = Path.of(fragment);
this.assetPath = fragment;
}
public URI getUri() {
@@ -71,7 +72,7 @@ public final class PackageAssetUri {
return packageUri;
}
public Path getAssetPath() {
public String getAssetPath() {
return assetPath;
}
@@ -102,6 +103,7 @@ public final class PackageAssetUri {
}
public PackageAssetUri resolve(String path) {
return new PackageAssetUri(packageUri, assetPath.resolve(path).toString());
var resolvedPath = IoUtils.toNormalizedPathString(Path.of(assetPath).resolve(path));
return new PackageAssetUri(packageUri, resolvedPath);
}
}

11
pkl-core/src/main/java/org/pkl/core/packages/PackageResolvers.java
View File

@@ -335,8 +335,7 @@ final class PackageResolvers {
var entries = cachedEntries.get(packageUri);
// need to normalize here but not in `doListElments` nor `doHasElement` because
// `TreePathElement.getElement` does normalization already.
var path = uri.getAssetPath().normalize().toString();
assert path.startsWith("/");
var path = IoUtils.toNormalizedPathString(Path.of(uri.getAssetPath()).normalize());
return entries.get(path).array();
}
@@ -496,7 +495,9 @@ final class PackageResolvers {
downloadMetadata(packageUri, requestUri, tmpPath, checksums);
Files.createDirectories(cachePath.getParent());
Files.move(tmpPath, cachePath, StandardCopyOption.ATOMIC_MOVE);
Files.setPosixFilePermissions(cachePath, FILE_PERMISSIONS);
if (!IoUtils.isWindows()) {
Files.setPosixFilePermissions(cachePath, FILE_PERMISSIONS);
}
return cachePath;
} finally {
Files.deleteIfExists(tmpPath);
@@ -545,7 +546,9 @@ final class PackageResolvers {
verifyPackageZipBytes(packageUri, dependencyMetadata, checksumBytes);
Files.createDirectories(cachePath.getParent());
Files.move(tmpPath, cachePath, StandardCopyOption.ATOMIC_MOVE);
Files.setPosixFilePermissions(cachePath, FILE_PERMISSIONS);
if (!IoUtils.isWindows()) {
Files.setPosixFilePermissions(cachePath, FILE_PERMISSIONS);
}
return cachePath;
} finally {
Files.deleteIfExists(tmpPath);

5
pkl-core/src/main/java/org/pkl/core/project/ProjectDependenciesResolver.java
View File

@@ -33,6 +33,7 @@ import org.pkl.core.packages.PackageUri;
import org.pkl.core.util.EconomicMaps;
import org.pkl.core.util.EconomicSets;
import org.pkl.core.util.ErrorMessages;
import org.pkl.core.util.IoUtils;
import org.pkl.core.util.Nullable;
/**
@@ -78,7 +79,7 @@ public final class ProjectDependenciesResolver {
private void log(String message) {
try {
logWriter.write(message + "\n");
logWriter.write(message + IoUtils.getLineSeparator());
} catch (IOException e) {
throw new UncheckedIOException(e);
}
@@ -130,7 +131,7 @@ public final class ProjectDependenciesResolver {
var packageUri = declaredDependencies.getMyPackageUri();
assert packageUri != null;
var projectDir = Path.of(declaredDependencies.getProjectFileUri()).getParent();
var relativePath = this.project.getProjectDir().relativize(projectDir);
var relativePath = IoUtils.relativize(projectDir, this.project.getProjectDir());
var localDependency = new LocalDependency(packageUri.toProjectPackageUri(), relativePath);
updateDependency(localDependency);
buildResolvedDependencies(declaredDependencies);

3
pkl-core/src/main/java/org/pkl/core/project/ProjectDeps.java
View File

@@ -35,6 +35,7 @@ import org.pkl.core.packages.PackageLoadError;
import org.pkl.core.packages.PackageUtils;
import org.pkl.core.runtime.VmExceptionBuilder;
import org.pkl.core.util.EconomicMaps;
import org.pkl.core.util.IoUtils;
import org.pkl.core.util.Nullable;
import org.pkl.core.util.json.Json;
import org.pkl.core.util.json.Json.FormatException;
@@ -196,7 +197,7 @@ public final class ProjectDeps {
jsonWriter.beginObject();
jsonWriter.name("type").value("local");
jsonWriter.name("uri").value(localDependency.getPackageUri().toString());
jsonWriter.name("path").value(localDependency.getPath().toString());
jsonWriter.name("path").value(IoUtils.toNormalizedPathString(localDependency.getPath()));
jsonWriter.endObject();
}

41
pkl-core/src/main/java/org/pkl/core/project/ProjectPackager.java
View File

@@ -20,6 +20,8 @@ import java.io.IOException;
import java.io.OutputStream;
import java.io.UncheckedIOException;
import java.io.Writer;
import java.net.URI;
import java.net.URISyntaxException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.security.DigestOutputStream;
@@ -119,13 +121,18 @@ public final class ProjectPackager {
this.outputWriter = outputWriter;
}
private void writeLine(String line) throws IOException {
outputWriter.write(line);
outputWriter.write(IoUtils.getLineSeparator());
}
public void createPackages() throws IOException {
for (var project : projects) {
var packageResult = doPackage(project);
outputWriter.write(workingDir.relativize(packageResult.getMetadataFile()) + "\n");
outputWriter.write(workingDir.relativize(packageResult.getMetadataChecksumFile()) + "\n");
outputWriter.write(workingDir.relativize(packageResult.getZipFile()) + "\n");
outputWriter.write(workingDir.relativize(packageResult.getZipChecksumFile()) + "\n");
writeLine(IoUtils.relativize(packageResult.getMetadataFile(), workingDir).toString());
writeLine(IoUtils.relativize(packageResult.getMetadataChecksumFile(), workingDir).toString());
writeLine(IoUtils.relativize(packageResult.getZipFile(), workingDir).toString());
writeLine(IoUtils.relativize(packageResult.getZipChecksumFile(), workingDir).toString());
outputWriter.flush();
}
}
@@ -302,8 +309,8 @@ public final class ProjectPackager {
}
try (var zos = new ZipOutputStream(digestOutputStream)) {
for (var file : files) {
var relativePath = project.getProjectDir().relativize(file);
var zipEntry = new ZipEntry(relativePath.toString());
var relativePath = IoUtils.relativize(file, project.getProjectDir());
var zipEntry = new ZipEntry(IoUtils.toNormalizedPathString(relativePath));
zipEntry.setTimeLocal(ZIP_ENTRY_MTIME);
zos.putNextEntry(zipEntry);
Files.copy(file, zos);
@@ -342,8 +349,8 @@ public final class ProjectPackager {
.filter(Files::isRegularFile)
.filter(
(it) -> {
var fileNameRelativeToProjectRoot =
project.getProjectDir().relativize(it).toString();
var relativePath = IoUtils.relativize(it, project.getProjectDir());
var fileNameRelativeToProjectRoot = IoUtils.toNormalizedPathString(relativePath);
for (var pattern : excludePatterns) {
if (pattern.matcher(it.getFileName().toString()).matches()) {
return false;
@@ -363,7 +370,7 @@ public final class ProjectPackager {
}
private boolean isAbsoluteImport(String importStr) {
return importStr.matches("\\w:.*") || importStr.startsWith("@");
return importStr.matches("\\w+:.*") || importStr.startsWith("@");
}
/**
@@ -386,8 +393,17 @@ public final class ProjectPackager {
if (isAbsoluteImport(importStr)) {
continue;
}
var importPath = Path.of(importStr);
if (importPath.isAbsolute() && !project.getProjectDir().toString().equals("/")) {
URI importUri;
try {
importUri = IoUtils.toUri(importStr);
} catch (URISyntaxException e) {
throw new VmExceptionBuilder()
.evalError("invalidModuleUri", importStr)
.withSourceSection(sourceSection)
.build()
.toPklException(stackFrameTransformer);
}
if (importStr.startsWith("/") && !project.getProjectDir().toString().equals("/")) {
throw new VmExceptionBuilder()
.evalError("invalidRelativeProjectImport", importStr)
.withSourceSection(sourceSection)
@@ -395,6 +411,7 @@ public final class ProjectPackager {
.toPklException(stackFrameTransformer);
}
var currentPath = pklModulePath.getParent();
var importPath = Path.of(importUri.getPath());
// It's not good enough to just check the normalized path to see whether it exists within the
// root dir.
// It's possible that the import path resolves to a path outside the project dir,
@@ -416,7 +433,7 @@ public final class ProjectPackager {
private @Nullable List<Pair<String, SourceSection>> getImportsAndReads(Path pklModulePath) {
try {
var moduleKey = ModuleKeys.file(pklModulePath.toUri(), pklModulePath);
var moduleKey = ModuleKeys.file(pklModulePath.toUri());
var resolvedModuleKey = ResolvedModuleKeys.file(moduleKey, moduleKey.getUri(), pklModulePath);
return ImportsAndReadsParser.parse(moduleKey, resolvedModuleKey);
} catch (IOException e) {

14
pkl-core/src/main/java/org/pkl/core/runtime/ModuleCache.java
View File

@@ -195,10 +195,16 @@ public final class ModuleCache {
} catch (SecurityManagerException | PackageLoadError e) {
throw new VmExceptionBuilder().withOptionalLocation(importNode).withCause(e).build();
} catch (FileNotFoundException | NoSuchFileException e) {
throw new VmExceptionBuilder()
.withOptionalLocation(importNode)
.evalError("cannotFindModule", module.getUri())
.build();
var exceptionBuilder =
new VmExceptionBuilder()
.withOptionalLocation(importNode)
.evalError("cannotFindModule", module.getUri());
var path = module.getUri().getPath();
if (path != null && path.contains("\\")) {
exceptionBuilder.withHint(
"To resolve modules in nested directories, use `/` as the directory separator.");
}
throw exceptionBuilder.build();
} catch (IOException e) {
throw new VmExceptionBuilder()
.withOptionalLocation(importNode)

165
pkl-core/src/main/java/org/pkl/core/util/IoUtils.java
View File

@@ -35,6 +35,7 @@ import java.util.regex.Pattern;
import java.util.zip.ZipEntry;
import java.util.zip.ZipOutputStream;
import org.pkl.core.PklBugException;
import org.pkl.core.Platform;
import org.pkl.core.SecurityManager;
import org.pkl.core.SecurityManagerException;
import org.pkl.core.module.ModuleKey;
@@ -43,7 +44,10 @@ import org.pkl.core.runtime.VmExceptionBuilder;
public final class IoUtils {
private static final Pattern uriLike = Pattern.compile("\\w+:.*");
// Don't match paths like `C:\`, which are drive letters on Windows.
private static final Pattern uriLike = Pattern.compile("\\w+:[^\\\\].*");
private static final Pattern windowsPathLike = Pattern.compile("\\w:\\\\.*");
private IoUtils() {}
@@ -66,12 +70,20 @@ public final class IoUtils {
return uriLike.matcher(str).matches();
}
public static boolean isWindowsAbsolutePath(String str) {
if (!isWindows()) return false;
return windowsPathLike.matcher(str).matches();
}
/**
* Converts the given string to a {@link URI}. This method MUST be used for constructing module
* and resource URIs. Unlike {@code new URI(str)}, it correctly escapes paths of relative URIs.
*/
public static URI toUri(String str) throws URISyntaxException {
return isUriLike(str) ? new URI(str) : new URI(null, null, str, null);
if (isUriLike(str)) {
return new URI(str);
}
return new URI(null, null, str, null);
}
/** Like {@link #toUri(String)}, except without checked exceptions. */
@@ -150,7 +162,8 @@ public final class IoUtils {
new SimpleFileVisitor<>() {
public FileVisitResult visitFile(Path file, BasicFileAttributes attrs)
throws IOException {
zipStream.putNextEntry(new ZipEntry(sourceDir.relativize(file).toString()));
var relativePath = relativize(file, sourceDir);
zipStream.putNextEntry(new ZipEntry(toNormalizedPathString(relativePath)));
Files.copy(file, zipStream);
zipStream.closeEntry();
return FileVisitResult.CONTINUE;
@@ -180,6 +193,10 @@ public final class IoUtils {
return System.getProperty("line.separator");
}
public static Boolean isWindows() {
return Platform.current().operatingSystem().name().equals("Windows");
}
public static String getName(String path) {
var lastSep = Math.max(path.lastIndexOf('/'), path.lastIndexOf('\\'));
return path.substring(lastSep + 1);
@@ -362,7 +379,9 @@ public final class IoUtils {
}
}
// URI.relativize() won't construct relative paths containing ".."
// URI.relativize won't construct relative paths containing `..`.
// Can't use Path.relativize because certain URI characters will throw InvalidPathException
// on Windows.
public static URI relativize(URI uri, URI base) {
if (uri.isOpaque()
|| base.isOpaque()
@@ -370,19 +389,60 @@ public final class IoUtils {
|| !Objects.equals(uri.getAuthority(), base.getAuthority())) {
return uri;
}
var basePath = Path.of(base.getPath());
if (!base.getRawPath().endsWith("/")) basePath = basePath.getParent();
var resultPath = basePath.relativize(Path.of(uri.getPath()));
var uriPath = uri.normalize().getPath();
var basePath = base.normalize().getPath();
try {
return new URI(
null, null, null, -1, resultPath.toString(), uri.getQuery(), uri.getFragment());
if (basePath.isEmpty()) {
return uri;
}
var uriParts = Arrays.asList(uriPath.split("/"));
var baseParts = Arrays.asList(basePath.split("/"));
if (!basePath.endsWith("/")) {
// strip the last path segment of the base uri, unless it ends in a slash. `/foo/bar.pkl` ->
// `/foo`
baseParts = baseParts.subList(0, baseParts.size() - 1);
}
if (uriParts.equals(baseParts)) {
return new URI(null, null, null, -1, "", uri.getQuery(), uri.getFragment());
}
var start = 0;
while (start < Math.min(uriParts.size(), baseParts.size())) {
if (!uriParts.get(start).equals(baseParts.get(start))) {
break;
}
start++;
}
var uriPartsRemaining = uriParts.subList(start, uriParts.size());
var basePartsRemainig = baseParts.subList(start, baseParts.size());
if (basePartsRemainig.isEmpty()) {
return new URI(
null,
null,
null,
-1,
String.join("/", uriPartsRemaining),
uri.getQuery(),
uri.getFragment());
}
var resultingPath =
"../".repeat(basePartsRemainig.size()) + String.join("/", uriPartsRemaining);
return new URI(null, null, null, -1, resultingPath, uri.getQuery(), uri.getFragment());
} catch (URISyntaxException e) {
throw new IllegalArgumentException(e);
// Impossible; started from a valid URI to begin with.
throw PklBugException.unreachableCode();
}
}
// On Windows, `Path.relativize` will fail if the two paths have different roots.
public static Path relativize(Path path, Path base) {
if (isWindows()) {
if (path.isAbsolute() && base.isAbsolute() && !path.getRoot().equals(base.getRoot())) {
return path;
}
}
return base.relativize(path);
}
public static boolean isWhitespace(String str) {
return str.codePoints().allMatch(Character::isWhitespace);
}
@@ -597,6 +657,63 @@ public final class IoUtils {
return newUri;
}
public static boolean isReservedFilenameChar(char character) {
if (isWindows()) {
return isReservedWindowsFilenameChar(character);
}
// posix; only NULL and `/` are reserved.
return character == 0 || character == '/';
}
/** Tells if this character cannot be used for filenames on Windows. */
public static boolean isReservedWindowsFilenameChar(char character) {
return switch (character) {
case 0,
1,
2,
3,
4,
5,
6,
7,
8,
9,
10,
11,
12,
13,
14,
15,
16,
17,
18,
19,
20,
21,
22,
23,
24,
25,
26,
27,
28,
29,
30,
31,
'<',
'>',
':',
'"',
'\\',
'/',
'|',
'?',
'*' ->
true;
default -> false;
};
}
/**
* Windows reserves characters {@code <>:"\|?*} in filenames.
*
@@ -608,19 +725,27 @@ public final class IoUtils {
var sb = new StringBuilder();
for (var i = 0; i < path.length(); i++) {
var character = path.charAt(i);
switch (character) {
case '<', '>', ':', '"', '\\', '|', '?', '*' -> {
sb.append('(');
sb.append(ByteArrayUtils.toHex(new byte[] {(byte) character}));
sb.append(")");
}
case '(' -> sb.append("((");
default -> sb.append(path.charAt(i));
if (isReservedWindowsFilenameChar(character) && character != '/') {
sb.append('(');
sb.append(ByteArrayUtils.toHex(new byte[] {(byte) character}));
sb.append(")");
} else if (character == '(') {
sb.append("((");
} else {
sb.append(character);
}
}
return sb.toString();
}
/** Returns a path string that uses unix-like path separators. */
public static String toNormalizedPathString(Path path) {
if (isWindows()) {
return path.toString().replace("\\", "/");
}
return path.toString();
}
private static int getExclamationMarkIndex(String jarUri) {
var index = jarUri.indexOf('!');
if (index == -1) {

2
pkl-core/src/test/files/LanguageSnippetTests/input/api/releaseModule.pkl vendored
View File

@@ -12,7 +12,7 @@ facts {
}
["versionInfo"] {
current.versionInfo.contains("macOS") || current.versionInfo.contains("Linux")
current.versionInfo.contains("macOS") || current.versionInfo.contains("Linux") || current.versionInfo.contains("Windows")
}
["commitId"] {

2
pkl-core/src/test/files/LanguageSnippetTests/input/errors/invalidImportBackslashSep.pkl vendored Normal file
View File

@@ -0,0 +1,2 @@
// In all OSes, the directory separator is forward slash.
res = import(#"..\basic\baseModule.pkl"#)

12
pkl-core/src/test/files/LanguageSnippetTests/output/errors/invalidImportBackslashSep.err vendored Normal file
View File

@@ -0,0 +1,12 @@
–– Pkl Error ––
Cannot find module `file:///$snippetsDir/input/errors/..%5Cbasic%5CbaseModule.pkl`.
x | res = import(#"..\basic\baseModule.pkl"#)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
at invalidImportBackslashSep#res (file:///$snippetsDir/input/errors/invalidImportBackslashSep.pkl)
To resolve modules in nested directories, use `/` as the directory separator.
xxx | text = renderer.renderDocument(value)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
at pkl.base#Module.output.text (pkl:base)

2
pkl-core/src/test/files/LanguageSnippetTests/output/projects/badProjectDeps1/bug.err vendored
View File

@@ -1,5 +1,5 @@
–– Pkl Error ––
Cannot resolve dependency because file `/$snippetsDir/input/projects/badProjectDeps1/PklProject.deps.json` is malformed.
Cannot resolve dependency because file `file:///$snippetsDir/input/projects/badProjectDeps1/PklProject.deps.json` is malformed.
Run `pkl project resolve` to re-create this file.
x | import "@bird/Bird.pkl"

2
pkl-core/src/test/files/LanguageSnippetTests/output/projects/badProjectDeps2/bug.err vendored
View File

@@ -1,5 +1,5 @@
–– Pkl Error ––
Cannot resolve dependency because file `/$snippetsDir/input/projects/badProjectDeps2/PklProject.deps.json` is malformed.
Cannot resolve dependency because file `file:///$snippetsDir/input/projects/badProjectDeps2/PklProject.deps.json` is malformed.
Run `pkl project resolve` to re-create this file.
x | import "@bird/Bird.pkl"

2
pkl-core/src/test/files/LanguageSnippetTests/output/projects/missingProjectDeps/bug.err vendored
View File

@@ -1,5 +1,5 @@
–– Pkl Error ––
Cannot resolve dependency because file `PklProject.deps.json` is missing in project directory `/$snippetsDir/input/projects/missingProjectDeps`.
Cannot resolve dependency because file `PklProject.deps.json` is missing in project directory `file:///$snippetsDir/input/projects/missingProjectDeps/`.
x | import "@birds/Bird.pkl"
^^^^^^^^^^^^^^^^^

12
pkl-core/src/test/kotlin/org/pkl/core/EvaluatorTest.kt
View File

@@ -74,11 +74,12 @@ class EvaluatorTest {
@Test
fun `evaluate non-existing file`() {
val file = File("/non/existing")
val e = assertThrows<PklException> {
evaluator.evaluate(file(File("/non/existing")))
evaluator.evaluate(file(file))
}
assertThat(e)
.hasMessageContaining("Cannot find module `file:///non/existing`.")
.hasMessageContaining("Cannot find module `${file.toPath().toUri()}`.")
}
@Test
@@ -92,13 +93,14 @@ class EvaluatorTest {
@Test
fun `evaluate non-existing path`() {
val path = "/non/existing".toPath()
val e = assertThrows<PklException> {
evaluator.evaluate(path("/non/existing".toPath()))
evaluator.evaluate(path(path))
}
assertThat(e)
.hasMessageContaining("Cannot find module `file:///non/existing`.")
.hasMessageContaining("Cannot find module `${path.toUri()}`.")
}
@Test
fun `evaluate zip file system path`(@TempDir tempDir: Path) {
val zipFile = createModulesZip(tempDir)

3
pkl-core/src/test/kotlin/org/pkl/core/LanguageSnippetTests.kt
View File

@@ -13,3 +13,6 @@ class LinuxLanguageSnippetTests
@Testable
class AlpineLanguageSnippetTests
@Testable
class WindowsLanguageSnippetTests

22
pkl-core/src/test/kotlin/org/pkl/core/LanguageSnippetTestsEngine.kt
View File

@@ -51,7 +51,7 @@ abstract class AbstractLanguageSnippetTestsEngine : InputOutputTestEngine() {
else parent?.getProjectDir()
override fun expectedOutputFileFor(inputFile: Path): Path {
val relativePath = inputDir.relativize(inputFile).toString()
val relativePath = IoUtils.relativize(inputFile, inputDir).toString()
val stdoutPath =
if (relativePath.matches(hiddenExtensionRegex)) relativePath.dropLast(4)
else relativePath.dropLast(3) + "pcf"
@@ -62,12 +62,12 @@ abstract class AbstractLanguageSnippetTestsEngine : InputOutputTestEngine() {
// disable SHA verification for packages
IoUtils.setTestMode()
}
override fun afterAll() {
packageServer.close()
}
protected fun String.stripFilePaths() = replace(snippetsDir.toString(), "/\$snippetsDir")
protected fun String.stripFilePaths() = replace(snippetsDir.toUri().toString(), "file:///\$snippetsDir/")
protected fun String.stripLineNumbers() = replace(lineNumberRegex) { result ->
// replace line number with equivalent number of 'x' characters to keep formatting intact
@@ -82,6 +82,11 @@ abstract class AbstractLanguageSnippetTestsEngine : InputOutputTestEngine() {
protected fun String.stripStdlibLocationSha(): String =
replace("https://github.com/apple/pkl/blob/${Release.current().commitId()}/stdlib/", "https://github.com/apple/pkl/blob/\$commitId/stdlib/")
protected fun String.withUnixLineEndings(): String {
return if (System.lineSeparator() == "\r\n") replace("\r\n", "\n")
else this
}
}
class LanguageSnippetTestsEngine : AbstractLanguageSnippetTestsEngine() {
@@ -143,7 +148,7 @@ class LanguageSnippetTestsEngine : AbstractLanguageSnippetTestsEngine() {
.stripVersionCheckErrorMessage()
}
val stderr = logWriter.toString()
val stderr = logWriter.toString().withUnixLineEndings()
return (success && stderr.isBlank()) to (output + stderr).stripFilePaths().stripWebsite().stripStdlibLocationSha()
}
@@ -216,7 +221,7 @@ abstract class AbstractNativeLanguageSnippetTestsEngine : AbstractLanguageSnippe
val process = builder.start()
return try {
val (out, err) = listOf(process.inputStream, process.errorStream)
.map { it.reader().readText() }
.map { it.reader().readText().withUnixLineEndings() }
val success = process.waitFor() == 0 && err.isBlank()
success to (out + err)
.stripFilePaths()
@@ -254,3 +259,8 @@ class AlpineLanguageSnippetTestsEngine : AbstractNativeLanguageSnippetTestsEngin
override val pklExecutablePath: Path = rootProjectDir.resolve("pkl-cli/build/executable/pkl-alpine-linux-amd64")
override val testClass: KClass<*> = AlpineLanguageSnippetTests::class
}
class WindowsLanguageSnippetTestsEngine : AbstractNativeLanguageSnippetTestsEngine() {
override val pklExecutablePath: Path = rootProjectDir.resolve("pkl-cli/build/executable/pkl-windows-amd64.exe")
override val testClass: KClass<*> = WindowsLanguageSnippetTests::class
}

16
pkl-core/src/test/kotlin/org/pkl/core/SecurityManagersTest.kt
View File

@@ -181,11 +181,11 @@ class SecurityManagersTest {
rootDir
)
manager.checkResolveModule(URI("file:///foo/bar/baz.pkl"))
manager.checkReadResource(URI("file:///foo/bar/baz.pkl"))
manager.checkResolveModule(Path.of("/foo/bar/baz.pkl").toUri())
manager.checkReadResource(Path.of("/foo/bar/baz.pkl").toUri())
manager.checkResolveModule(URI("file:///foo/bar/qux/../baz.pkl"))
manager.checkReadResource(URI("file:///foo/bar/qux/../baz.pkl"))
manager.checkResolveModule(Path.of("/foo/bar/qux/../baz.pkl").toUri())
manager.checkReadResource(Path.of("/foo/bar/qux/../baz.pkl").toUri())
}
@Test
@@ -233,17 +233,17 @@ class SecurityManagersTest {
)
assertThrows<SecurityManagerException> {
manager.checkResolveModule(URI("file:///foo/baz.pkl"))
manager.checkResolveModule(Path.of("/foo/baz.pkl").toUri())
}
assertThrows<SecurityManagerException> {
manager.checkReadResource(URI("file:///foo/baz.pkl"))
manager.checkReadResource(Path.of("/foo/baz.pkl").toUri())
}
assertThrows<SecurityManagerException> {
manager.checkResolveModule(URI("file:///foo/bar/../baz.pkl"))
manager.checkResolveModule(Path.of("/foo/bar/../baz.pkl").toUri())
}
assertThrows<SecurityManagerException> {
manager.checkReadResource(URI("file:///foo/bar/../baz.pkl"))
manager.checkReadResource(Path.of("/foo/bar/../baz.pkl").toUri())
}
}
}

2
pkl-core/src/test/kotlin/org/pkl/core/module/ModuleKeysTest.kt
View File

@@ -57,7 +57,7 @@ class ModuleKeysTest {
file.writeString("age = 40")
val uri = file.toUri()
val key = ModuleKeys.file(uri, file.toAbsolutePath())
val key = ModuleKeys.file(uri)
assertThat(key.uri).isEqualTo(uri)
assertThat(key.isCached).isTrue

5
pkl-core/src/test/kotlin/org/pkl/core/project/ProjectDependenciesResolverTest.kt
View File

@@ -6,6 +6,7 @@ import org.junit.jupiter.api.Test
import org.junit.jupiter.api.assertThrows
import org.pkl.commons.test.FileTestUtils
import org.pkl.commons.test.PackageServer
import org.pkl.commons.toPath
import org.pkl.core.http.HttpClient
import org.pkl.core.PklException
import org.pkl.core.SecurityManagers
@@ -34,7 +35,7 @@ class ProjectDependenciesResolverTest {
@Test
fun resolveDependencies() {
val project2Path = Path.of(javaClass.getResource("project2/PklProject")!!.path)
val project2Path = javaClass.getResource("project2/PklProject")!!.toURI().toPath()
val project = Project.loadFromPath(project2Path)
val packageResolver = PackageResolver.getInstance(SecurityManagers.defaultManager, httpClient, null)
val deps = ProjectDependenciesResolver(project, packageResolver, System.out.writer()).resolve()
@@ -72,7 +73,7 @@ class ProjectDependenciesResolverTest {
@Test
fun `fails if project declares a package with an incorrect checksum`() {
val projectPath = Path.of(javaClass.getResource("badProjectChecksum/PklProject")!!.path)
val projectPath = javaClass.getResource("badProjectChecksum/PklProject")!!.toURI().toPath()
val project = Project.loadFromPath(projectPath)
val packageResolver = PackageResolver.getInstance(SecurityManagers.defaultManager, httpClient, null)
val e = assertThrows<PklException> {

2
pkl-core/src/test/kotlin/org/pkl/core/project/ProjectTest.kt
View File

@@ -137,7 +137,7 @@ class ProjectTest {
@Test
fun `evaluate project module -- invalid checksum`() {
PackageServer().use { server ->
val projectDir = Path.of(javaClass.getResource("badProjectChecksum2/")!!.path)
val projectDir = Path.of(javaClass.getResource("badProjectChecksum2/")!!.toURI())
val project = Project.loadFromPath(projectDir.resolve("PklProject"))
val httpClient = HttpClient.builder()
.addCertificates(FileTestUtils.selfSignedCertificate)

43
pkl-core/src/test/kotlin/org/pkl/core/util/IoUtilsTest.kt
View File

@@ -117,70 +117,69 @@ class IoUtilsTest {
@Test
fun `relativize file URLs`() {
// perhaps URI("") would be a more precise result
assertThat(
IoUtils.relativize(
URI("file://foo/bar/baz.pkl"),
URI("file://foo/bar/baz.pkl")
URI("file:///foo/bar/baz.pkl"),
URI("file:///foo/bar/baz.pkl")
)
).isEqualTo(URI("baz.pkl"))
assertThat(
IoUtils.relativize(
URI("file://foo/bar/baz.pkl"),
URI("file://foo/bar/qux.pkl")
URI("file:///foo/bar/baz.pkl"),
URI("file:///foo/bar/qux.pkl")
)
).isEqualTo(URI("baz.pkl"))
assertThat(
IoUtils.relativize(
URI("file://foo/bar/baz.pkl"),
URI("file://foo/bar/")
URI("file:///foo/bar/baz.pkl"),
URI("file:///foo/bar/")
)
).isEqualTo(URI("baz.pkl"))
assertThat(
IoUtils.relativize(
URI("file://foo/bar/baz.pkl"),
URI("file://foo/bar")
URI("file:///foo/bar/baz.pkl"),
URI("file:///foo/bar")
)
).isEqualTo(URI("bar/baz.pkl"))
// URI.relativize() returns an absolute URI here
assertThat(
IoUtils.relativize(
URI("file://foo/bar/baz.pkl"),
URI("file://foo/qux/")
URI("file:///foo/bar/baz.pkl"),
URI("file:///foo/qux/")
)
).isEqualTo(URI("../bar/baz.pkl"))
assertThat(
IoUtils.relativize(
URI("file://foo/bar/baz.pkl"),
URI("file://foo/qux/qux2/")
URI("file:///foo/bar/baz.pkl"),
URI("file:///foo/qux/qux2/")
)
).isEqualTo(URI("../../bar/baz.pkl"))
assertThat(
IoUtils.relativize(
URI("file://foo/bar/baz.pkl"),
URI("file://foo/qux/qux2")
URI("file:///foo/bar/baz.pkl"),
URI("file:///foo/qux/qux2")
)
).isEqualTo(URI("../bar/baz.pkl"))
assertThat(
IoUtils.relativize(
URI("file://foo/bar/baz.pkl"),
URI("file://qux/qux2/")
URI("file:///foo/bar/baz.pkl"),
URI("file:///qux/qux2/")
)
).isEqualTo(URI("file://foo/bar/baz.pkl"))
).isEqualTo(URI("../../foo/bar/baz.pkl"))
assertThat(
IoUtils.relativize(
URI("file://foo/bar/baz.pkl"),
URI("https://foo/bar/baz.pkl")
URI("file:///foo/bar/baz.pkl"),
URI("https:///foo/bar/baz.pkl")
)
).isEqualTo(URI("file://foo/bar/baz.pkl"))
).isEqualTo(URI("file:///foo/bar/baz.pkl"))
}
@Test
@@ -343,7 +342,7 @@ class IoUtilsTest {
val file3 = tempDir.resolve("base1/dir2/foo.pkl").createParentDirectories().createFile()
val uri = file2.toUri()
val key = ModuleKeys.file(uri, file2)
val key = ModuleKeys.file(uri)
assertThat(IoUtils.resolve(FakeSecurityManager, key, URI("..."))).isEqualTo(file1.toUri())
assertThat(IoUtils.resolve(FakeSecurityManager, key, URI(".../foo.pkl"))).isEqualTo(file1.toUri())

1
pkl-core/src/test/resources/META-INF/services/org.junit.platform.engine.TestEngine
View File

@@ -4,3 +4,4 @@ org.pkl.core.MacAarch64LanguageSnippetTestsEngine
org.pkl.core.LinuxAmd64LanguageSnippetTestsEngine
org.pkl.core.LinuxAarch64LanguageSnippetTestsEngine
org.pkl.core.AlpineLanguageSnippetTestsEngine
org.pkl.core.WindowsLanguageSnippetTestsEngine

8
pkl-doc/src/main/kotlin/org/pkl/doc/CliDocGenerator.kt
View File

@@ -22,6 +22,7 @@ import kotlin.Pair
import org.pkl.commons.cli.CliBaseOptions.Companion.getProjectFile
import org.pkl.commons.cli.CliCommand
import org.pkl.commons.cli.CliException
import org.pkl.commons.toPath
import org.pkl.core.*
import org.pkl.core.module.ModuleKeyFactories
import org.pkl.core.packages.*
@@ -136,7 +137,7 @@ class CliDocGenerator(private val options: CliDocGeneratorOptions) : CliCommand(
val packageUris = mutableListOf<PackageUri>()
for (moduleUri in options.base.normalizedSourceModules) {
if (moduleUri.scheme == "file") {
val dir = Path.of(moduleUri).parent
val dir = moduleUri.toPath().parent
val projectFile = dir.getProjectFile(options.base.normalizedRootDir)
if (projectFile != null) {
pklProjectPaths.add(projectFile)
@@ -229,13 +230,12 @@ class CliDocGenerator(private val options: CliDocGeneratorOptions) : CliCommand(
DocPackageInfo.fromPkl(module).apply {
evaluator.collectImportedModules(overviewImports)
}
schemasByDocPackageInfoAndPath[docPackageInfo to Path.of(uri.path).parent] =
mutableSetOf()
schemasByDocPackageInfoAndPath[docPackageInfo to uri.toPath().parent] = mutableSetOf()
}
for (uri in regularModuleUris) {
val entry =
schemasByDocPackageInfoAndPath.keys.find { uri.path.startsWith(it.second.toString()) }
schemasByDocPackageInfoAndPath.keys.find { uri.toPath().startsWith(it.second) }
?: throw CliException("Could not find a doc-package-info.pkl for module $uri")
val schema =
evaluator.evaluateSchema(ModuleSource.uri(uri)).apply {

3
pkl-doc/src/main/kotlin/org/pkl/doc/DocGenerator.kt
View File

@@ -26,6 +26,7 @@ import org.pkl.commons.deleteRecursively
import org.pkl.core.ModuleSchema
import org.pkl.core.PClassInfo
import org.pkl.core.Version
import org.pkl.core.util.IoUtils
/**
* Entry point for the low-level Pkldoc API.
@@ -126,7 +127,7 @@ class DocGenerator(
val dest = basePath.resolve("current")
if (dest.exists() && dest.isSameFileAs(src)) continue
dest.deleteIfExists()
dest.createSymbolicLinkPointingTo(basePath.relativize(src))
dest.createSymbolicLinkPointingTo(IoUtils.relativize(src, basePath))
}
}
}

1
pkl-doc/src/main/kotlin/org/pkl/doc/Main.kt
View File

@@ -28,6 +28,7 @@ import java.net.URI
import java.nio.file.Path
import org.pkl.commons.cli.cliMain
import org.pkl.commons.cli.commands.BaseCommand
import org.pkl.commons.cli.commands.BaseOptions.Companion.parseModuleName
import org.pkl.commons.cli.commands.ProjectOptions
import org.pkl.core.Release

16
pkl-doc/src/test/kotlin/org/pkl/doc/CliDocGeneratorTest.kt
View File

@@ -37,6 +37,7 @@ import org.pkl.commons.test.PackageServer
import org.pkl.commons.test.listFilesRecursively
import org.pkl.commons.toPath
import org.pkl.core.Version
import org.pkl.core.util.IoUtils
import org.pkl.doc.DocGenerator.Companion.current
class CliDocGeneratorTest {
@@ -92,11 +93,17 @@ class CliDocGeneratorTest {
private val actualOutputFiles: List<Path> by lazy { actualOutputDir.listFilesRecursively() }
private val expectedRelativeOutputFiles: List<String> by lazy {
expectedOutputFiles.map { expectedOutputDir.relativize(it).toString() }
expectedOutputFiles.map { path ->
IoUtils.toNormalizedPathString(expectedOutputDir.relativize(path)).let { str ->
// Git will by default clone symlinks as shortcuts on Windows, and shortcuts have a
// `.lnk` extension.
if (IoUtils.isWindows() && str.endsWith(".lnk")) str.dropLast(4) else str
}
}
}
private val actualRelativeOutputFiles: List<String> by lazy {
actualOutputFiles.map { actualOutputDir.relativize(it).toString() }
actualOutputFiles.map { IoUtils.toNormalizedPathString(actualOutputDir.relativize(it)) }
}
private val binaryFileExtensions =
@@ -219,6 +226,11 @@ class CliDocGeneratorTest {
.withFailMessage("Test bug: $actualFile should exist but does not.")
.exists()
// symlinks on Git and Windows is rather finnicky; they create shortcuts by default unless
// a core Git option is set. Also, by default, symlinks require administrator privileges to run.
// We'll just test that the symlink got created but skip verifying that it points to the right
// place.
if (actualFile.isSymbolicLink() && IoUtils.isWindows()) return
val expectedFile = expectedOutputDir.resolve(relativeFilePath)
if (expectedFile.exists()) {
when {

3
pkl-doc/src/test/kotlin/org/pkl/doc/DocScopeTest.kt
View File

@@ -16,6 +16,7 @@
package org.pkl.doc
import java.net.URI
import java.nio.file.Path
import org.assertj.core.api.Assertions.assertThat
import org.junit.jupiter.api.AfterAll
import org.junit.jupiter.api.Test
@@ -221,6 +222,6 @@ class DocScopeTest {
val scope = SiteScope(listOf(), mapOf(), { evaluator.evaluateSchema(uri(it)) }, outputDir)
// used to return `/non/index.html`
assertThat(scope.url.path).isEqualTo("/non/existing/index.html")
assertThat(scope.url.toPath()).isEqualTo(Path.of("/non/existing/index.html").toAbsolutePath())
}
}

22
pkl-executor/pkl-executor.gradle.kts
View File

@@ -1,4 +1,5 @@
import java.nio.file.Files
import java.nio.file.LinkOption
plugins {
pklAllProjects
@@ -62,12 +63,23 @@ val prepareHistoricalDistributions by tasks.registering {
val distributionDir = outputDir.get().asFile.toPath()
.also(Files::createDirectories)
for (file in pklHistoricalDistributions.files) {
val link = distributionDir.resolve(file.name)
if (!Files.isSymbolicLink(link)) {
if (Files.exists(link)) {
Files.delete(link)
val target = distributionDir.resolve(file.name)
// Create normal files on Windows, symlink on macOS/linux (need admin priveleges to create
// symlinks on Windows)
if (buildInfo.os.isWindows) {
if (!Files.isRegularFile(target, LinkOption.NOFOLLOW_LINKS)) {
if (Files.exists(target)) {
Files.delete(target)
}
Files.copy(file.toPath(), target)
}
} else {
if (!Files.isSymbolicLink(target)) {
if (Files.exists(target)) {
Files.delete(target)
}
Files.createSymbolicLink(target, file.toPath())
}
Files.createSymbolicLink(link, file.toPath())
}
}
}

10
pkl-executor/src/main/java/org/pkl/executor/EmbeddedExecutor.java
View File

@@ -157,7 +157,15 @@ final class EmbeddedExecutor implements Executor {
private static Path toDisplayPath(Path modulePath, ExecutorOptions options) {
var rootDir = options.getRootDir();
return rootDir == null ? modulePath : rootDir.relativize(modulePath);
return rootDir == null ? modulePath : relativize(modulePath, rootDir);
}
// On Windows, `Path.relativize` will fail if the two paths have different roots.
private static Path relativize(Path path, Path base) {
if (path.isAbsolute() && base.isAbsolute() && !path.getRoot().equals(base.getRoot())) {
return path;
}
return base.relativize(path);
}
@Override

4
pkl-executor/src/test/kotlin/org/pkl/executor/EmbeddedExecutorTest.kt
View File

@@ -15,6 +15,7 @@ import org.pkl.commons.test.FilteringClassLoader
import org.pkl.commons.test.PackageServer
import org.pkl.commons.toPath
import org.pkl.core.Release
import java.io.File
import java.nio.file.Files
import java.nio.file.Path
import java.time.Duration
@@ -197,9 +198,10 @@ class EmbeddedExecutorTest {
Executors.embedded(listOf("/non/existing".toPath()))
}
val sep = File.separatorChar
assertThat(e.message)
.contains("Cannot find Jar file")
.contains("/non/existing")
.contains("${sep}non${sep}existing")
}
@Test

2
pkl-gradle/src/main/java/org/pkl/gradle/task/ModulesTask.java
View File

@@ -134,7 +134,7 @@ public abstract class ModulesTask extends BasePklTask {
*/
private URI parsedModuleNotationToUri(Object notation) {
if (notation instanceof File file) {
return IoUtils.createUri(file.getPath());
return IoUtils.createUri(IoUtils.toNormalizedPathString(file.toPath()));
} else if (notation instanceof URI uri) {
return uri;
}

10
pkl-gradle/src/test/kotlin/org/pkl/gradle/EvaluatorsTest.kt
View File

@@ -2,7 +2,9 @@ package org.pkl.gradle
import org.assertj.core.api.Assertions
import org.pkl.commons.readString
import org.pkl.commons.readString
import org.pkl.commons.test.PackageServer
import org.pkl.commons.toNormalizedPathString
import org.assertj.core.api.Assertions.assertThat
import org.junit.jupiter.api.Test
import org.junit.jupiter.api.io.TempDir
@@ -199,8 +201,8 @@ class EvaluatorsTest : AbstractTest() {
@Test
fun `source module URIs`() {
val pklFile = writeFile(
"test.pkl", """
writeFile(
"testDir/test.pkl", """
person {
name = "Pigeon"
age = 20 + 10
@@ -218,7 +220,7 @@ class EvaluatorsTest : AbstractTest() {
evaluators {
evalTest {
sourceModules = [uri("modulepath:/test.pkl")]
modulePath.from "${pklFile.parent}"
modulePath.from layout.projectDirectory.dir("testDir")
outputFile = layout.projectDirectory.file("test.pcf")
settingsModule = "pkl:settings"
}
@@ -387,7 +389,7 @@ class EvaluatorsTest : AbstractTest() {
@Test
fun `explicitly set cache dir`(@TempDir tempDir: Path) {
writeBuildFile("pcf", """
moduleCacheDir = file("$tempDir")
moduleCacheDir = file("${tempDir.toUri()}")
""".trimIndent())
writeFile(
"test.pkl",

6
pkl-gradle/src/test/kotlin/org/pkl/gradle/TestsTest.kt
View File

@@ -1,6 +1,7 @@
package org.pkl.gradle
import org.assertj.core.api.Assertions.assertThat
import org.pkl.commons.toNormalizedPathString
import org.junit.jupiter.api.Test
import java.nio.file.Path
import kotlin.io.path.readText
@@ -46,8 +47,7 @@ class TestsTest : AbstractTest() {
val output = runTask("evalTest", expectFailure = true).output.stripFilesAndLines()
assertThat(output.trimStart()).startsWith("""
> Task :evalTest FAILED
assertThat(output).contains("""
module test (file:///file, line x)
test ❌
Error:
@@ -143,7 +143,7 @@ class TestsTest : AbstractTest() {
val pklFile = writePklFile(contents = bigTest)
writeFile("test.pkl-expected.pcf", bigTestExpected)
writeBuildFile("junitReportsDir = file('${pklFile.parent}/build')")
writeBuildFile("junitReportsDir = file('${pklFile.parent.toNormalizedPathString()}/build')")
runTask("evalTest", expectFailure = true)

3
pkl-server/src/test/kotlin/org/pkl/server/BinaryEvaluatorSnippetTests.kt
View File

@@ -58,7 +58,8 @@ class BinaryEvaluatorSnippetTestEngine : InputOutputTestEngine() {
null
)
private fun String.stripFilePaths() = replace(snippetsDir.toString(), "/\$snippetsDir")
private fun String.stripFilePaths() =
replace(snippetsDir.toUri().toString(), "file:///\$snippetsDir/")
override fun generateOutputFor(inputFile: Path): Pair<Boolean, String> {
val bytes = evaluator.evaluate(ModuleSource.path(inputFile), null)

2
stdlib/platform.pkl
View File

@@ -66,7 +66,7 @@ class VirtualMachine {
/// The operating system of a platform.
class OperatingSystem {
/// The name of this operating system.
name: String
name: "macOS"|"Linux"|"Windows"|String
/// The version of this operating system.
version: String