starred/pkl
1
0
Fork 0
mirror of https://github.com/apple/pkl.git synced 2026-03-31 06:03:11 +02:00
Code Issues 198 Packages Projects Releases 10 Wiki Activity

Add support for Windows (#492)

Browse Source 
This adds support for Windows.
The in-language path separator is still `/`, to ensure Pkl programs are cross-platform.

Log lines are written using CRLF endings on Windows.
Modules that are combined with `--module-output-separator` uses LF endings to ensure
consistent rendering across platforms.

`jpkl` does not work on Windows as a direct executable.
However, it can work with `java -jar jpkl`.

Additional details:

* Adjust git settings for Windows
* Add native executable for pkl cli
* Add jdk17 windows Gradle check in CI
* Adjust CI test reports to be staged within Gradle rather than by shell script.
* Fix: encode more characters that are not safe Windows paths
* Skip running tests involving symbolic links on Windows (these require administrator privileges to run).
* Introduce custom implementation of `IoUtils.relativize`
* Allow Gradle to initialize ExecutableJar `Property` values
* Add Gradle flag to enable remote JVM debugging

Co-authored-by: Philip K.F. Hölzenspies <holzensp@gmail.com>
This commit is contained in:
Daniel Chao
2024-05-28 15:56:20 -07:00
committed by GitHub
parent 5e4ccfd4e8
commit 8ec06e631f
76 changed files with 905 additions and 402 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
pkl-core/src/main/java/org/pkl/core/Platform.java
View File

@@ -31,6 +31,7 @@ public final class Platform {
var pklVersion = Release.current().version().toString();
var osName = System.getProperty("os.name");
if (osName.equals("Mac OS X")) osName = "macOS";
if (osName.contains("Windows")) osName = "Windows";
var osVersion = System.getProperty("os.version");
var architecture = System.getProperty("os.arch");

1
pkl-core/src/main/java/org/pkl/core/Release.java
View File

@@ -49,6 +49,7 @@ public final class Release {
var commitId = properties.getProperty("commitId");
var osName = System.getProperty("os.name");
if (osName.equals("Mac OS X")) osName = "macOS";
if (osName.contains("Windows")) osName = "Windows";
var osVersion = System.getProperty("os.version");
var os = osName + " " + osVersion;
var flavor = TruffleOptions.AOT ? "native" : "Java " + System.getProperty("java.version");

15
pkl-core/src/main/java/org/pkl/core/ast/builder/AstBuilder.java
View File

@@ -1794,10 +1794,17 @@ public final class AstBuilder extends AbstractAstBuilder<Object> {
try {
resolvedUri = IoUtils.resolve(context.getSecurityManager(), moduleKey, parsedUri);
} catch (FileNotFoundException e) {
throw exceptionBuilder()
.evalError("cannotFindModule", importUri)
.withSourceSection(createSourceSection(importUriCtx))
.build();
var exceptionBuilder =
exceptionBuilder()
.evalError("cannotFindModule", importUri)
.withSourceSection(createSourceSection(importUriCtx));
var path = parsedUri.getPath();
if (path != null && path.contains("\\")) {
exceptionBuilder.withHint(
"To resolve modules in nested directories, use `/` as the directory separator.");
}
throw exceptionBuilder.build();
} catch (URISyntaxException e) {
throw exceptionBuilder()
.evalError("invalidModuleUri", importUri)

21
pkl-core/src/main/java/org/pkl/core/module/ModuleKeyFactories.java
View File

@@ -18,8 +18,8 @@ package org.pkl.core.module;
import java.net.URI;
import java.net.URISyntaxException;
import java.nio.file.FileSystemNotFoundException;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.spi.FileSystemProvider;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
@@ -141,15 +141,20 @@ public final class ModuleKeyFactories {
private static class File implements ModuleKeyFactory {
@Override
public Optional<ModuleKey> create(URI uri) {
Path path;
try {
path = Path.of(uri);
} catch (FileSystemNotFoundException | IllegalArgumentException e) {
// none of the installed file system providers can handle this URI
// skip loading providers if the scheme is `file`.
if (uri.getScheme().equalsIgnoreCase("file")) {
return Optional.of(ModuleKeys.file(uri));
}
// don't handle jar-file URIs (these are handled by GenericUrl).
if (uri.getScheme().equalsIgnoreCase("jar")) {
return Optional.empty();
}
return Optional.of(ModuleKeys.file(uri, path));
for (FileSystemProvider provider : FileSystemProvider.installedProviders()) {
if (provider.getScheme().equalsIgnoreCase(uri.getScheme())) {
return Optional.of(ModuleKeys.file(uri));
}
}
return Optional.empty();
}
}

26
pkl-core/src/main/java/org/pkl/core/module/ModuleKeys.java
View File

@@ -16,9 +16,11 @@
package org.pkl.core.module;
import com.oracle.truffle.api.CompilerDirectives.TruffleBoundary;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
import java.net.JarURLConnection;
import java.net.URI;
import java.net.URISyntaxException;
import java.net.http.HttpRequest;
@@ -88,8 +90,8 @@ public final class ModuleKeys {
}
/** Creates a module key for a {@code file:} module. */
public static ModuleKey file(URI uri, Path path) {
return new File(uri, path);
public static ModuleKey file(URI uri) {
return new File(uri);
}
/**
@@ -290,12 +292,10 @@ public final class ModuleKeys {
private static class File extends DependencyAwareModuleKey {
final URI uri;
final Path path;
File(URI uri, Path path) {
File(URI uri) {
super(uri);
this.uri = uri;
this.path = path;
}
@Override
@@ -316,7 +316,13 @@ public final class ModuleKeys {
public ResolvedModuleKey resolve(SecurityManager securityManager)
throws IOException, SecurityManagerException {
securityManager.checkResolveModule(uri);
var realPath = path.toRealPath();
// Disallow paths that contain `\\` characters if on Windows
// (require `/` as the path separator on all OSes)
var uriPath = uri.getPath();
if (java.io.File.separatorChar == '\\' && uriPath != null && uriPath.contains("\\")) {
throw new FileNotFoundException();
}
var realPath = Path.of(uri).toRealPath();
var resolvedUri = realPath.toUri();
securityManager.checkResolveModule(resolvedUri);
return ResolvedModuleKeys.file(this, resolvedUri, realPath);
@@ -325,7 +331,7 @@ public final class ModuleKeys {
@Override
protected Map<String, ? extends Dependency> getDependencies() {
var projectDepsManager = VmContext.get(null).getProjectDependenciesManager();
if (projectDepsManager == null || !projectDepsManager.hasPath(path)) {
if (projectDepsManager == null || !projectDepsManager.hasPath(Path.of(uri))) {
throw new PackageLoadError("cannotResolveDependencyNoProject");
}
return projectDepsManager.getDependencies();
@@ -519,6 +525,12 @@ public final class ModuleKeys {
var url = IoUtils.toUrl(uri);
var conn = url.openConnection();
conn.connect();
if (conn instanceof JarURLConnection && IoUtils.isWindows()) {
// On Windows, opening a JarURLConnection prevents the jar file from being deleted, unless
// cacheing is disabled.
// See https://bugs.openjdk.org/browse/JDK-8239054
conn.setUseCaches(false);
}
try (InputStream stream = conn.getInputStream()) {
URI redirected;
try {

5
pkl-core/src/main/java/org/pkl/core/module/ModulePathResolver.java
View File

@@ -30,6 +30,7 @@ import java.util.Map;
import javax.annotation.concurrent.GuardedBy;
import org.pkl.core.module.PathElement.TreePathElement;
import org.pkl.core.runtime.FileSystemManager;
import org.pkl.core.util.IoUtils;
import org.pkl.core.util.LateInit;
/**
@@ -152,8 +153,8 @@ public final class ModulePathResolver implements AutoCloseable {
// in case of duplicate path, first entry wins (cf. class loader)
stream.forEach(
(path) -> {
var relativized = basePath.relativize(path);
fileCache.putIfAbsent(relativized.toString(), path);
var relativized = IoUtils.relativize(path, basePath);
fileCache.putIfAbsent(IoUtils.toNormalizedPathString(relativized), path);
var element = cachedPathElementRoot;
for (var i = 0; i < relativized.getNameCount(); i++) {
var name = relativized.getName(i).toString();

6
pkl-core/src/main/java/org/pkl/core/module/ProjectDependenciesManager.java
View File

@@ -207,13 +207,15 @@ public final class ProjectDependenciesManager {
if (projectDeps == null) {
var depsPath = getProjectDepsFile();
if (!Files.exists(depsPath)) {
throw new VmExceptionBuilder().evalError("missingProjectDepsJson", projectDir).build();
throw new VmExceptionBuilder()
.evalError("missingProjectDepsJson", projectDir.toUri())
.build();
}
try {
projectDeps = ProjectDeps.parse(depsPath);
} catch (IOException | URISyntaxException | JsonParseException e) {
throw new VmExceptionBuilder()
.evalError("invalidProjectDepsJson", depsPath, e.getMessage())
.evalError("invalidProjectDepsJson", depsPath.toUri(), e.getMessage())
.withCause(e)
.build();
}

13
pkl-core/src/main/java/org/pkl/core/module/ResolvedModuleKeys.java
View File

@@ -15,10 +15,12 @@
*/
package org.pkl.core.module;
import java.io.File;
import java.io.IOException;
import java.net.URI;
import java.net.URL;
import java.nio.charset.StandardCharsets;
import java.nio.file.AccessDeniedException;
import java.nio.file.Files;
import java.nio.file.Path;
import org.pkl.core.util.IoUtils;
@@ -75,7 +77,16 @@ public final class ResolvedModuleKeys {
@Override
public String loadSource() throws IOException {
return Files.readString(path, StandardCharsets.UTF_8);
try {
return Files.readString(path, StandardCharsets.UTF_8);
} catch (AccessDeniedException e) {
// Windows throws `AccessDeniedException` when reading directories.
// Sync error between different OSes.
if (Files.isDirectory(path)) {
throw new IOException("Is a directory");
}
throw e;
}
}
}

2
pkl-core/src/main/java/org/pkl/core/packages/Dependency.java
View File

@@ -50,7 +50,7 @@ public abstract class Dependency {
public Path resolveAssetPath(Path projectDir, PackageAssetUri packageAssetUri) {
// drop 1 to remove leading `/`
var assetPath = packageAssetUri.getAssetPath().toString().substring(1);
var assetPath = packageAssetUri.getAssetPath().substring(1);
return projectDir.resolve(path).resolve(assetPath);
}

12
pkl-core/src/main/java/org/pkl/core/packages/PackageAssetUri.java
View File

@@ -20,6 +20,7 @@ import java.net.URISyntaxException;
import java.nio.file.Path;
import org.pkl.core.Version;
import org.pkl.core.util.ErrorMessages;
import org.pkl.core.util.IoUtils;
/**
* The canonical URI of an asset within a package, i.e., a package URI with a fragment path. For
@@ -28,7 +29,7 @@ import org.pkl.core.util.ErrorMessages;
public final class PackageAssetUri {
private final URI uri;
private final PackageUri packageUri;
private final Path assetPath;
private final String assetPath;
public static PackageAssetUri create(URI uri) {
try {
@@ -41,7 +42,7 @@ public final class PackageAssetUri {
public PackageAssetUri(PackageUri packageUri, String assetPath) {
this.uri = packageUri.getUri().resolve("#" + assetPath);
this.packageUri = packageUri;
this.assetPath = Path.of(assetPath);
this.assetPath = assetPath;
}
public PackageAssetUri(String uri) throws URISyntaxException {
@@ -60,7 +61,7 @@ public final class PackageAssetUri {
throw new URISyntaxException(
uri.toString(), ErrorMessages.create("cannotHaveRelativeFragment", fragment, uri));
}
this.assetPath = Path.of(fragment);
this.assetPath = fragment;
}
public URI getUri() {
@@ -71,7 +72,7 @@ public final class PackageAssetUri {
return packageUri;
}
public Path getAssetPath() {
public String getAssetPath() {
return assetPath;
}
@@ -102,6 +103,7 @@ public final class PackageAssetUri {
}
public PackageAssetUri resolve(String path) {
return new PackageAssetUri(packageUri, assetPath.resolve(path).toString());
var resolvedPath = IoUtils.toNormalizedPathString(Path.of(assetPath).resolve(path));
return new PackageAssetUri(packageUri, resolvedPath);
}
}

11
pkl-core/src/main/java/org/pkl/core/packages/PackageResolvers.java
View File

@@ -335,8 +335,7 @@ final class PackageResolvers {
var entries = cachedEntries.get(packageUri);
// need to normalize here but not in `doListElments` nor `doHasElement` because
// `TreePathElement.getElement` does normalization already.
var path = uri.getAssetPath().normalize().toString();
assert path.startsWith("/");
var path = IoUtils.toNormalizedPathString(Path.of(uri.getAssetPath()).normalize());
return entries.get(path).array();
}
@@ -496,7 +495,9 @@ final class PackageResolvers {
downloadMetadata(packageUri, requestUri, tmpPath, checksums);
Files.createDirectories(cachePath.getParent());
Files.move(tmpPath, cachePath, StandardCopyOption.ATOMIC_MOVE);
Files.setPosixFilePermissions(cachePath, FILE_PERMISSIONS);
if (!IoUtils.isWindows()) {
Files.setPosixFilePermissions(cachePath, FILE_PERMISSIONS);
}
return cachePath;
} finally {
Files.deleteIfExists(tmpPath);
@@ -545,7 +546,9 @@ final class PackageResolvers {
verifyPackageZipBytes(packageUri, dependencyMetadata, checksumBytes);
Files.createDirectories(cachePath.getParent());
Files.move(tmpPath, cachePath, StandardCopyOption.ATOMIC_MOVE);
Files.setPosixFilePermissions(cachePath, FILE_PERMISSIONS);
if (!IoUtils.isWindows()) {
Files.setPosixFilePermissions(cachePath, FILE_PERMISSIONS);
}
return cachePath;
} finally {
Files.deleteIfExists(tmpPath);

5
pkl-core/src/main/java/org/pkl/core/project/ProjectDependenciesResolver.java
View File

@@ -33,6 +33,7 @@ import org.pkl.core.packages.PackageUri;
import org.pkl.core.util.EconomicMaps;
import org.pkl.core.util.EconomicSets;
import org.pkl.core.util.ErrorMessages;
import org.pkl.core.util.IoUtils;
import org.pkl.core.util.Nullable;
/**
@@ -78,7 +79,7 @@ public final class ProjectDependenciesResolver {
private void log(String message) {
try {
logWriter.write(message + "\n");
logWriter.write(message + IoUtils.getLineSeparator());
} catch (IOException e) {
throw new UncheckedIOException(e);
}
@@ -130,7 +131,7 @@ public final class ProjectDependenciesResolver {
var packageUri = declaredDependencies.getMyPackageUri();
assert packageUri != null;
var projectDir = Path.of(declaredDependencies.getProjectFileUri()).getParent();
var relativePath = this.project.getProjectDir().relativize(projectDir);
var relativePath = IoUtils.relativize(projectDir, this.project.getProjectDir());
var localDependency = new LocalDependency(packageUri.toProjectPackageUri(), relativePath);
updateDependency(localDependency);
buildResolvedDependencies(declaredDependencies);

3
pkl-core/src/main/java/org/pkl/core/project/ProjectDeps.java
View File

@@ -35,6 +35,7 @@ import org.pkl.core.packages.PackageLoadError;
import org.pkl.core.packages.PackageUtils;
import org.pkl.core.runtime.VmExceptionBuilder;
import org.pkl.core.util.EconomicMaps;
import org.pkl.core.util.IoUtils;
import org.pkl.core.util.Nullable;
import org.pkl.core.util.json.Json;
import org.pkl.core.util.json.Json.FormatException;
@@ -196,7 +197,7 @@ public final class ProjectDeps {
jsonWriter.beginObject();
jsonWriter.name("type").value("local");
jsonWriter.name("uri").value(localDependency.getPackageUri().toString());
jsonWriter.name("path").value(localDependency.getPath().toString());
jsonWriter.name("path").value(IoUtils.toNormalizedPathString(localDependency.getPath()));
jsonWriter.endObject();
}

41
pkl-core/src/main/java/org/pkl/core/project/ProjectPackager.java
View File

@@ -20,6 +20,8 @@ import java.io.IOException;
import java.io.OutputStream;
import java.io.UncheckedIOException;
import java.io.Writer;
import java.net.URI;
import java.net.URISyntaxException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.security.DigestOutputStream;
@@ -119,13 +121,18 @@ public final class ProjectPackager {
this.outputWriter = outputWriter;
}
private void writeLine(String line) throws IOException {
outputWriter.write(line);
outputWriter.write(IoUtils.getLineSeparator());
}
public void createPackages() throws IOException {
for (var project : projects) {
var packageResult = doPackage(project);
outputWriter.write(workingDir.relativize(packageResult.getMetadataFile()) + "\n");
outputWriter.write(workingDir.relativize(packageResult.getMetadataChecksumFile()) + "\n");
outputWriter.write(workingDir.relativize(packageResult.getZipFile()) + "\n");
outputWriter.write(workingDir.relativize(packageResult.getZipChecksumFile()) + "\n");
writeLine(IoUtils.relativize(packageResult.getMetadataFile(), workingDir).toString());
writeLine(IoUtils.relativize(packageResult.getMetadataChecksumFile(), workingDir).toString());
writeLine(IoUtils.relativize(packageResult.getZipFile(), workingDir).toString());
writeLine(IoUtils.relativize(packageResult.getZipChecksumFile(), workingDir).toString());
outputWriter.flush();
}
}
@@ -302,8 +309,8 @@ public final class ProjectPackager {
}
try (var zos = new ZipOutputStream(digestOutputStream)) {
for (var file : files) {
var relativePath = project.getProjectDir().relativize(file);
var zipEntry = new ZipEntry(relativePath.toString());
var relativePath = IoUtils.relativize(file, project.getProjectDir());
var zipEntry = new ZipEntry(IoUtils.toNormalizedPathString(relativePath));
zipEntry.setTimeLocal(ZIP_ENTRY_MTIME);
zos.putNextEntry(zipEntry);
Files.copy(file, zos);
@@ -342,8 +349,8 @@ public final class ProjectPackager {
.filter(Files::isRegularFile)
.filter(
(it) -> {
var fileNameRelativeToProjectRoot =
project.getProjectDir().relativize(it).toString();
var relativePath = IoUtils.relativize(it, project.getProjectDir());
var fileNameRelativeToProjectRoot = IoUtils.toNormalizedPathString(relativePath);
for (var pattern : excludePatterns) {
if (pattern.matcher(it.getFileName().toString()).matches()) {
return false;
@@ -363,7 +370,7 @@ public final class ProjectPackager {
}
private boolean isAbsoluteImport(String importStr) {
return importStr.matches("\\w:.*") || importStr.startsWith("@");
return importStr.matches("\\w+:.*") || importStr.startsWith("@");
}
/**
@@ -386,8 +393,17 @@ public final class ProjectPackager {
if (isAbsoluteImport(importStr)) {
continue;
}
var importPath = Path.of(importStr);
if (importPath.isAbsolute() && !project.getProjectDir().toString().equals("/")) {
URI importUri;
try {
importUri = IoUtils.toUri(importStr);
} catch (URISyntaxException e) {
throw new VmExceptionBuilder()
.evalError("invalidModuleUri", importStr)
.withSourceSection(sourceSection)
.build()
.toPklException(stackFrameTransformer);
}
if (importStr.startsWith("/") && !project.getProjectDir().toString().equals("/")) {
throw new VmExceptionBuilder()
.evalError("invalidRelativeProjectImport", importStr)
.withSourceSection(sourceSection)
@@ -395,6 +411,7 @@ public final class ProjectPackager {
.toPklException(stackFrameTransformer);
}
var currentPath = pklModulePath.getParent();
var importPath = Path.of(importUri.getPath());
// It's not good enough to just check the normalized path to see whether it exists within the
// root dir.
// It's possible that the import path resolves to a path outside the project dir,
@@ -416,7 +433,7 @@ public final class ProjectPackager {
private @Nullable List<Pair<String, SourceSection>> getImportsAndReads(Path pklModulePath) {
try {
var moduleKey = ModuleKeys.file(pklModulePath.toUri(), pklModulePath);
var moduleKey = ModuleKeys.file(pklModulePath.toUri());
var resolvedModuleKey = ResolvedModuleKeys.file(moduleKey, moduleKey.getUri(), pklModulePath);
return ImportsAndReadsParser.parse(moduleKey, resolvedModuleKey);
} catch (IOException e) {

14
pkl-core/src/main/java/org/pkl/core/runtime/ModuleCache.java
View File

@@ -195,10 +195,16 @@ public final class ModuleCache {
} catch (SecurityManagerException | PackageLoadError e) {
throw new VmExceptionBuilder().withOptionalLocation(importNode).withCause(e).build();
} catch (FileNotFoundException | NoSuchFileException e) {
throw new VmExceptionBuilder()
.withOptionalLocation(importNode)
.evalError("cannotFindModule", module.getUri())
.build();
var exceptionBuilder =
new VmExceptionBuilder()
.withOptionalLocation(importNode)
.evalError("cannotFindModule", module.getUri());
var path = module.getUri().getPath();
if (path != null && path.contains("\\")) {
exceptionBuilder.withHint(
"To resolve modules in nested directories, use `/` as the directory separator.");
}
throw exceptionBuilder.build();
} catch (IOException e) {
throw new VmExceptionBuilder()
.withOptionalLocation(importNode)

165
pkl-core/src/main/java/org/pkl/core/util/IoUtils.java
View File

@@ -35,6 +35,7 @@ import java.util.regex.Pattern;
import java.util.zip.ZipEntry;
import java.util.zip.ZipOutputStream;
import org.pkl.core.PklBugException;
import org.pkl.core.Platform;
import org.pkl.core.SecurityManager;
import org.pkl.core.SecurityManagerException;
import org.pkl.core.module.ModuleKey;
@@ -43,7 +44,10 @@ import org.pkl.core.runtime.VmExceptionBuilder;
public final class IoUtils {
private static final Pattern uriLike = Pattern.compile("\\w+:.*");
// Don't match paths like `C:\`, which are drive letters on Windows.
private static final Pattern uriLike = Pattern.compile("\\w+:[^\\\\].*");
private static final Pattern windowsPathLike = Pattern.compile("\\w:\\\\.*");
private IoUtils() {}
@@ -66,12 +70,20 @@ public final class IoUtils {
return uriLike.matcher(str).matches();
}
public static boolean isWindowsAbsolutePath(String str) {
if (!isWindows()) return false;
return windowsPathLike.matcher(str).matches();
}
/**
* Converts the given string to a {@link URI}. This method MUST be used for constructing module
* and resource URIs. Unlike {@code new URI(str)}, it correctly escapes paths of relative URIs.
*/
public static URI toUri(String str) throws URISyntaxException {
return isUriLike(str) ? new URI(str) : new URI(null, null, str, null);
if (isUriLike(str)) {
return new URI(str);
}
return new URI(null, null, str, null);
}
/** Like {@link #toUri(String)}, except without checked exceptions. */
@@ -150,7 +162,8 @@ public final class IoUtils {
new SimpleFileVisitor<>() {
public FileVisitResult visitFile(Path file, BasicFileAttributes attrs)
throws IOException {
zipStream.putNextEntry(new ZipEntry(sourceDir.relativize(file).toString()));
var relativePath = relativize(file, sourceDir);
zipStream.putNextEntry(new ZipEntry(toNormalizedPathString(relativePath)));
Files.copy(file, zipStream);
zipStream.closeEntry();
return FileVisitResult.CONTINUE;
@@ -180,6 +193,10 @@ public final class IoUtils {
return System.getProperty("line.separator");
}
public static Boolean isWindows() {
return Platform.current().operatingSystem().name().equals("Windows");
}
public static String getName(String path) {
var lastSep = Math.max(path.lastIndexOf('/'), path.lastIndexOf('\\'));
return path.substring(lastSep + 1);
@@ -362,7 +379,9 @@ public final class IoUtils {
}
}
// URI.relativize() won't construct relative paths containing ".."
// URI.relativize won't construct relative paths containing `..`.
// Can't use Path.relativize because certain URI characters will throw InvalidPathException
// on Windows.
public static URI relativize(URI uri, URI base) {
if (uri.isOpaque()
|| base.isOpaque()
@@ -370,19 +389,60 @@ public final class IoUtils {
|| !Objects.equals(uri.getAuthority(), base.getAuthority())) {
return uri;
}
var basePath = Path.of(base.getPath());
if (!base.getRawPath().endsWith("/")) basePath = basePath.getParent();
var resultPath = basePath.relativize(Path.of(uri.getPath()));
var uriPath = uri.normalize().getPath();
var basePath = base.normalize().getPath();
try {
return new URI(
null, null, null, -1, resultPath.toString(), uri.getQuery(), uri.getFragment());
if (basePath.isEmpty()) {
return uri;
}
var uriParts = Arrays.asList(uriPath.split("/"));
var baseParts = Arrays.asList(basePath.split("/"));
if (!basePath.endsWith("/")) {
// strip the last path segment of the base uri, unless it ends in a slash. `/foo/bar.pkl` ->
// `/foo`
baseParts = baseParts.subList(0, baseParts.size() - 1);
}
if (uriParts.equals(baseParts)) {
return new URI(null, null, null, -1, "", uri.getQuery(), uri.getFragment());
}
var start = 0;
while (start < Math.min(uriParts.size(), baseParts.size())) {
if (!uriParts.get(start).equals(baseParts.get(start))) {
break;
}
start++;
}
var uriPartsRemaining = uriParts.subList(start, uriParts.size());
var basePartsRemainig = baseParts.subList(start, baseParts.size());
if (basePartsRemainig.isEmpty()) {
return new URI(
null,
null,
null,
-1,
String.join("/", uriPartsRemaining),
uri.getQuery(),
uri.getFragment());
}
var resultingPath =
"../".repeat(basePartsRemainig.size()) + String.join("/", uriPartsRemaining);
return new URI(null, null, null, -1, resultingPath, uri.getQuery(), uri.getFragment());
} catch (URISyntaxException e) {
throw new IllegalArgumentException(e);
// Impossible; started from a valid URI to begin with.
throw PklBugException.unreachableCode();
}
}
// On Windows, `Path.relativize` will fail if the two paths have different roots.
public static Path relativize(Path path, Path base) {
if (isWindows()) {
if (path.isAbsolute() && base.isAbsolute() && !path.getRoot().equals(base.getRoot())) {
return path;
}
}
return base.relativize(path);
}
public static boolean isWhitespace(String str) {
return str.codePoints().allMatch(Character::isWhitespace);
}
@@ -597,6 +657,63 @@ public final class IoUtils {
return newUri;
}
public static boolean isReservedFilenameChar(char character) {
if (isWindows()) {
return isReservedWindowsFilenameChar(character);
}
// posix; only NULL and `/` are reserved.
return character == 0 || character == '/';
}
/** Tells if this character cannot be used for filenames on Windows. */
public static boolean isReservedWindowsFilenameChar(char character) {
return switch (character) {
case 0,
1,
2,
3,
4,
5,
6,
7,
8,
9,
10,
11,
12,
13,
14,
15,
16,
17,
18,
19,
20,
21,
22,
23,
24,
25,
26,
27,
28,
29,
30,
31,
'<',
'>',
':',
'"',
'\\',
'/',
'|',
'?',
'*' ->
true;
default -> false;
};
}
/**
* Windows reserves characters {@code <>:"\|?*} in filenames.
*
@@ -608,19 +725,27 @@ public final class IoUtils {
var sb = new StringBuilder();
for (var i = 0; i < path.length(); i++) {
var character = path.charAt(i);
switch (character) {
case '<', '>', ':', '"', '\\', '|', '?', '*' -> {
sb.append('(');
sb.append(ByteArrayUtils.toHex(new byte[] {(byte) character}));
sb.append(")");
}
case '(' -> sb.append("((");
default -> sb.append(path.charAt(i));
if (isReservedWindowsFilenameChar(character) && character != '/') {
sb.append('(');
sb.append(ByteArrayUtils.toHex(new byte[] {(byte) character}));
sb.append(")");
} else if (character == '(') {
sb.append("((");
} else {
sb.append(character);
}
}
return sb.toString();
}
/** Returns a path string that uses unix-like path separators. */
public static String toNormalizedPathString(Path path) {
if (isWindows()) {
return path.toString().replace("\\", "/");
}
return path.toString();
}
private static int getExclamationMarkIndex(String jarUri) {
var index = jarUri.indexOf('!');
if (index == -1) {

2
pkl-core/src/test/files/LanguageSnippetTests/input/api/releaseModule.pkl vendored
View File

@@ -12,7 +12,7 @@ facts {
}
["versionInfo"] {
current.versionInfo.contains("macOS") || current.versionInfo.contains("Linux")
current.versionInfo.contains("macOS") || current.versionInfo.contains("Linux") || current.versionInfo.contains("Windows")
}
["commitId"] {

2
pkl-core/src/test/files/LanguageSnippetTests/input/errors/invalidImportBackslashSep.pkl vendored Normal file
View File

@@ -0,0 +1,2 @@
// In all OSes, the directory separator is forward slash.
res = import(#"..\basic\baseModule.pkl"#)

12
pkl-core/src/test/files/LanguageSnippetTests/output/errors/invalidImportBackslashSep.err vendored Normal file
View File

@@ -0,0 +1,12 @@
–– Pkl Error ––
Cannot find module `file:///$snippetsDir/input/errors/..%5Cbasic%5CbaseModule.pkl`.
x | res = import(#"..\basic\baseModule.pkl"#)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
at invalidImportBackslashSep#res (file:///$snippetsDir/input/errors/invalidImportBackslashSep.pkl)
To resolve modules in nested directories, use `/` as the directory separator.
xxx | text = renderer.renderDocument(value)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
at pkl.base#Module.output.text (pkl:base)

2
pkl-core/src/test/files/LanguageSnippetTests/output/projects/badProjectDeps1/bug.err vendored
View File

@@ -1,5 +1,5 @@
–– Pkl Error ––
Cannot resolve dependency because file `/$snippetsDir/input/projects/badProjectDeps1/PklProject.deps.json` is malformed.
Cannot resolve dependency because file `file:///$snippetsDir/input/projects/badProjectDeps1/PklProject.deps.json` is malformed.
Run `pkl project resolve` to re-create this file.
x | import "@bird/Bird.pkl"

2
pkl-core/src/test/files/LanguageSnippetTests/output/projects/badProjectDeps2/bug.err vendored
View File

@@ -1,5 +1,5 @@
–– Pkl Error ––
Cannot resolve dependency because file `/$snippetsDir/input/projects/badProjectDeps2/PklProject.deps.json` is malformed.
Cannot resolve dependency because file `file:///$snippetsDir/input/projects/badProjectDeps2/PklProject.deps.json` is malformed.
Run `pkl project resolve` to re-create this file.
x | import "@bird/Bird.pkl"

2
pkl-core/src/test/files/LanguageSnippetTests/output/projects/missingProjectDeps/bug.err vendored
View File

@@ -1,5 +1,5 @@
–– Pkl Error ––
Cannot resolve dependency because file `PklProject.deps.json` is missing in project directory `/$snippetsDir/input/projects/missingProjectDeps`.
Cannot resolve dependency because file `PklProject.deps.json` is missing in project directory `file:///$snippetsDir/input/projects/missingProjectDeps/`.
x | import "@birds/Bird.pkl"
^^^^^^^^^^^^^^^^^

12
pkl-core/src/test/kotlin/org/pkl/core/EvaluatorTest.kt
View File

@@ -74,11 +74,12 @@ class EvaluatorTest {
@Test
fun `evaluate non-existing file`() {
val file = File("/non/existing")
val e = assertThrows<PklException> {
evaluator.evaluate(file(File("/non/existing")))
evaluator.evaluate(file(file))
}
assertThat(e)
.hasMessageContaining("Cannot find module `file:///non/existing`.")
.hasMessageContaining("Cannot find module `${file.toPath().toUri()}`.")
}
@Test
@@ -92,13 +93,14 @@ class EvaluatorTest {
@Test
fun `evaluate non-existing path`() {
val path = "/non/existing".toPath()
val e = assertThrows<PklException> {
evaluator.evaluate(path("/non/existing".toPath()))
evaluator.evaluate(path(path))
}
assertThat(e)
.hasMessageContaining("Cannot find module `file:///non/existing`.")
.hasMessageContaining("Cannot find module `${path.toUri()}`.")
}
@Test
fun `evaluate zip file system path`(@TempDir tempDir: Path) {
val zipFile = createModulesZip(tempDir)

3
pkl-core/src/test/kotlin/org/pkl/core/LanguageSnippetTests.kt
View File

@@ -13,3 +13,6 @@ class LinuxLanguageSnippetTests
@Testable
class AlpineLanguageSnippetTests
@Testable
class WindowsLanguageSnippetTests

22
pkl-core/src/test/kotlin/org/pkl/core/LanguageSnippetTestsEngine.kt
View File

@@ -51,7 +51,7 @@ abstract class AbstractLanguageSnippetTestsEngine : InputOutputTestEngine() {
else parent?.getProjectDir()
override fun expectedOutputFileFor(inputFile: Path): Path {
val relativePath = inputDir.relativize(inputFile).toString()
val relativePath = IoUtils.relativize(inputFile, inputDir).toString()
val stdoutPath =
if (relativePath.matches(hiddenExtensionRegex)) relativePath.dropLast(4)
else relativePath.dropLast(3) + "pcf"
@@ -62,12 +62,12 @@ abstract class AbstractLanguageSnippetTestsEngine : InputOutputTestEngine() {
// disable SHA verification for packages
IoUtils.setTestMode()
}
override fun afterAll() {
packageServer.close()
}
protected fun String.stripFilePaths() = replace(snippetsDir.toString(), "/\$snippetsDir")
protected fun String.stripFilePaths() = replace(snippetsDir.toUri().toString(), "file:///\$snippetsDir/")
protected fun String.stripLineNumbers() = replace(lineNumberRegex) { result ->
// replace line number with equivalent number of 'x' characters to keep formatting intact
@@ -82,6 +82,11 @@ abstract class AbstractLanguageSnippetTestsEngine : InputOutputTestEngine() {
protected fun String.stripStdlibLocationSha(): String =
replace("https://github.com/apple/pkl/blob/${Release.current().commitId()}/stdlib/", "https://github.com/apple/pkl/blob/\$commitId/stdlib/")
protected fun String.withUnixLineEndings(): String {
return if (System.lineSeparator() == "\r\n") replace("\r\n", "\n")
else this
}
}
class LanguageSnippetTestsEngine : AbstractLanguageSnippetTestsEngine() {
@@ -143,7 +148,7 @@ class LanguageSnippetTestsEngine : AbstractLanguageSnippetTestsEngine() {
.stripVersionCheckErrorMessage()
}
val stderr = logWriter.toString()
val stderr = logWriter.toString().withUnixLineEndings()
return (success && stderr.isBlank()) to (output + stderr).stripFilePaths().stripWebsite().stripStdlibLocationSha()
}
@@ -216,7 +221,7 @@ abstract class AbstractNativeLanguageSnippetTestsEngine : AbstractLanguageSnippe
val process = builder.start()
return try {
val (out, err) = listOf(process.inputStream, process.errorStream)
.map { it.reader().readText() }
.map { it.reader().readText().withUnixLineEndings() }
val success = process.waitFor() == 0 && err.isBlank()
success to (out + err)
.stripFilePaths()
@@ -254,3 +259,8 @@ class AlpineLanguageSnippetTestsEngine : AbstractNativeLanguageSnippetTestsEngin
override val pklExecutablePath: Path = rootProjectDir.resolve("pkl-cli/build/executable/pkl-alpine-linux-amd64")
override val testClass: KClass<*> = AlpineLanguageSnippetTests::class
}
class WindowsLanguageSnippetTestsEngine : AbstractNativeLanguageSnippetTestsEngine() {
override val pklExecutablePath: Path = rootProjectDir.resolve("pkl-cli/build/executable/pkl-windows-amd64.exe")
override val testClass: KClass<*> = WindowsLanguageSnippetTests::class
}

16
pkl-core/src/test/kotlin/org/pkl/core/SecurityManagersTest.kt
View File

@@ -181,11 +181,11 @@ class SecurityManagersTest {
rootDir
)
manager.checkResolveModule(URI("file:///foo/bar/baz.pkl"))
manager.checkReadResource(URI("file:///foo/bar/baz.pkl"))
manager.checkResolveModule(Path.of("/foo/bar/baz.pkl").toUri())
manager.checkReadResource(Path.of("/foo/bar/baz.pkl").toUri())
manager.checkResolveModule(URI("file:///foo/bar/qux/../baz.pkl"))
manager.checkReadResource(URI("file:///foo/bar/qux/../baz.pkl"))
manager.checkResolveModule(Path.of("/foo/bar/qux/../baz.pkl").toUri())
manager.checkReadResource(Path.of("/foo/bar/qux/../baz.pkl").toUri())
}
@Test
@@ -233,17 +233,17 @@ class SecurityManagersTest {
)
assertThrows<SecurityManagerException> {
manager.checkResolveModule(URI("file:///foo/baz.pkl"))
manager.checkResolveModule(Path.of("/foo/baz.pkl").toUri())
}
assertThrows<SecurityManagerException> {
manager.checkReadResource(URI("file:///foo/baz.pkl"))
manager.checkReadResource(Path.of("/foo/baz.pkl").toUri())
}
assertThrows<SecurityManagerException> {
manager.checkResolveModule(URI("file:///foo/bar/../baz.pkl"))
manager.checkResolveModule(Path.of("/foo/bar/../baz.pkl").toUri())
}
assertThrows<SecurityManagerException> {
manager.checkReadResource(URI("file:///foo/bar/../baz.pkl"))
manager.checkReadResource(Path.of("/foo/bar/../baz.pkl").toUri())
}
}
}

2
pkl-core/src/test/kotlin/org/pkl/core/module/ModuleKeysTest.kt
View File

@@ -57,7 +57,7 @@ class ModuleKeysTest {
file.writeString("age = 40")
val uri = file.toUri()
val key = ModuleKeys.file(uri, file.toAbsolutePath())
val key = ModuleKeys.file(uri)
assertThat(key.uri).isEqualTo(uri)
assertThat(key.isCached).isTrue

5
pkl-core/src/test/kotlin/org/pkl/core/project/ProjectDependenciesResolverTest.kt
View File

@@ -6,6 +6,7 @@ import org.junit.jupiter.api.Test
import org.junit.jupiter.api.assertThrows
import org.pkl.commons.test.FileTestUtils
import org.pkl.commons.test.PackageServer
import org.pkl.commons.toPath
import org.pkl.core.http.HttpClient
import org.pkl.core.PklException
import org.pkl.core.SecurityManagers
@@ -34,7 +35,7 @@ class ProjectDependenciesResolverTest {
@Test
fun resolveDependencies() {
val project2Path = Path.of(javaClass.getResource("project2/PklProject")!!.path)
val project2Path = javaClass.getResource("project2/PklProject")!!.toURI().toPath()
val project = Project.loadFromPath(project2Path)
val packageResolver = PackageResolver.getInstance(SecurityManagers.defaultManager, httpClient, null)
val deps = ProjectDependenciesResolver(project, packageResolver, System.out.writer()).resolve()
@@ -72,7 +73,7 @@ class ProjectDependenciesResolverTest {
@Test
fun `fails if project declares a package with an incorrect checksum`() {
val projectPath = Path.of(javaClass.getResource("badProjectChecksum/PklProject")!!.path)
val projectPath = javaClass.getResource("badProjectChecksum/PklProject")!!.toURI().toPath()
val project = Project.loadFromPath(projectPath)
val packageResolver = PackageResolver.getInstance(SecurityManagers.defaultManager, httpClient, null)
val e = assertThrows<PklException> {

2
pkl-core/src/test/kotlin/org/pkl/core/project/ProjectTest.kt
View File

@@ -137,7 +137,7 @@ class ProjectTest {
@Test
fun `evaluate project module -- invalid checksum`() {
PackageServer().use { server ->
val projectDir = Path.of(javaClass.getResource("badProjectChecksum2/")!!.path)
val projectDir = Path.of(javaClass.getResource("badProjectChecksum2/")!!.toURI())
val project = Project.loadFromPath(projectDir.resolve("PklProject"))
val httpClient = HttpClient.builder()
.addCertificates(FileTestUtils.selfSignedCertificate)

43
pkl-core/src/test/kotlin/org/pkl/core/util/IoUtilsTest.kt
View File

@@ -117,70 +117,69 @@ class IoUtilsTest {
@Test
fun `relativize file URLs`() {
// perhaps URI("") would be a more precise result
assertThat(
IoUtils.relativize(
URI("file://foo/bar/baz.pkl"),
URI("file://foo/bar/baz.pkl")
URI("file:///foo/bar/baz.pkl"),
URI("file:///foo/bar/baz.pkl")
)
).isEqualTo(URI("baz.pkl"))
assertThat(
IoUtils.relativize(
URI("file://foo/bar/baz.pkl"),
URI("file://foo/bar/qux.pkl")
URI("file:///foo/bar/baz.pkl"),
URI("file:///foo/bar/qux.pkl")
)
).isEqualTo(URI("baz.pkl"))
assertThat(
IoUtils.relativize(
URI("file://foo/bar/baz.pkl"),
URI("file://foo/bar/")
URI("file:///foo/bar/baz.pkl"),
URI("file:///foo/bar/")
)
).isEqualTo(URI("baz.pkl"))
assertThat(
IoUtils.relativize(
URI("file://foo/bar/baz.pkl"),
URI("file://foo/bar")
URI("file:///foo/bar/baz.pkl"),
URI("file:///foo/bar")
)
).isEqualTo(URI("bar/baz.pkl"))
// URI.relativize() returns an absolute URI here
assertThat(
IoUtils.relativize(
URI("file://foo/bar/baz.pkl"),
URI("file://foo/qux/")
URI("file:///foo/bar/baz.pkl"),
URI("file:///foo/qux/")
)
).isEqualTo(URI("../bar/baz.pkl"))
assertThat(
IoUtils.relativize(
URI("file://foo/bar/baz.pkl"),
URI("file://foo/qux/qux2/")
URI("file:///foo/bar/baz.pkl"),
URI("file:///foo/qux/qux2/")
)
).isEqualTo(URI("../../bar/baz.pkl"))
assertThat(
IoUtils.relativize(
URI("file://foo/bar/baz.pkl"),
URI("file://foo/qux/qux2")
URI("file:///foo/bar/baz.pkl"),
URI("file:///foo/qux/qux2")
)
).isEqualTo(URI("../bar/baz.pkl"))
assertThat(
IoUtils.relativize(
URI("file://foo/bar/baz.pkl"),
URI("file://qux/qux2/")
URI("file:///foo/bar/baz.pkl"),
URI("file:///qux/qux2/")
)
).isEqualTo(URI("file://foo/bar/baz.pkl"))
).isEqualTo(URI("../../foo/bar/baz.pkl"))
assertThat(
IoUtils.relativize(
URI("file://foo/bar/baz.pkl"),
URI("https://foo/bar/baz.pkl")
URI("file:///foo/bar/baz.pkl"),
URI("https:///foo/bar/baz.pkl")
)
).isEqualTo(URI("file://foo/bar/baz.pkl"))
).isEqualTo(URI("file:///foo/bar/baz.pkl"))
}
@Test
@@ -343,7 +342,7 @@ class IoUtilsTest {
val file3 = tempDir.resolve("base1/dir2/foo.pkl").createParentDirectories().createFile()
val uri = file2.toUri()
val key = ModuleKeys.file(uri, file2)
val key = ModuleKeys.file(uri)
assertThat(IoUtils.resolve(FakeSecurityManager, key, URI("..."))).isEqualTo(file1.toUri())
assertThat(IoUtils.resolve(FakeSecurityManager, key, URI(".../foo.pkl"))).isEqualTo(file1.toUri())

1
pkl-core/src/test/resources/META-INF/services/org.junit.platform.engine.TestEngine
View File

@@ -4,3 +4,4 @@ org.pkl.core.MacAarch64LanguageSnippetTestsEngine
org.pkl.core.LinuxAmd64LanguageSnippetTestsEngine
org.pkl.core.LinuxAarch64LanguageSnippetTestsEngine
org.pkl.core.AlpineLanguageSnippetTestsEngine
org.pkl.core.WindowsLanguageSnippetTestsEngine