From d5ba8fa73623a1e54c596b5ff93aa143046a5183 Mon Sep 17 00:00:00 2001 From: Daniel Chao Date: Tue, 4 Jun 2024 16:52:20 -0700 Subject: [PATCH] Support scheme-agnostic projects (#486) This adds changes to support loading project dependencies in non-file based projects. The design for this feature can be found in SPICE-0005: https://github.com/apple/pkl-evolution/pull/6 Changes: * Consider all imports prefixed with `@` as dependency notation. * Bugfix: fix resolution of glob expressions in a local dependency. * Adjust pkl.Project: - Allow local dependencies from a scheme-local paths. - Disallow certain evaluator settings if not loaded as a file-based module. * Breaking API change: `ProjectDependenciesManager` constructor now requires `ModuleResolver` and `SecurityManager`. --- .../main/java/org/pkl/core/EvaluatorImpl.java | 3 +- .../org/pkl/core/ast/builder/AstBuilder.java | 2 + .../ast/expression/unary/ReadGlobNode.java | 6 +- .../java/org/pkl/core/module/ModuleKeys.java | 195 ++++++++---------- .../module/ProjectDependenciesManager.java | 64 ++++-- .../pkl/core/module/ResolvedModuleKeys.java | 34 +++ .../org/pkl/core/packages/Dependency.java | 6 +- .../java/org/pkl/core/project/Project.java | 107 +++++++--- .../org/pkl/core/project/ProjectPackager.java | 11 +- .../java/org/pkl/core/repl/ReplServer.java | 4 +- .../pkl/core/resource/ResourceReaders.java | 39 ++-- .../org/pkl/core/runtime/ResourceManager.java | 29 ++- .../pkl/core/runtime/VmExceptionBuilder.java | 2 +- .../java/org/pkl/core/util/GlobResolver.java | 10 +- .../main/java/org/pkl/core/util/IoUtils.java | 85 ++++++-- .../org/pkl/core/errorMessages.properties | 35 +++- .../input/projects/project1/PklProject | 1 + .../projects/project1/PklProject.deps.json | 5 + .../input/projects/project1/globbing.pkl | 4 + .../projects/project3/PklProject.deps.json | 5 + .../input/projects/project6/PklProject | 8 + .../projects/project6/PklProject.deps.json | 4 + .../input/projects/project6/children.pkl | 1 + .../input/projects/project6/children/a.pkl | 1 + .../input/projects/project6/children/b.pkl | 1 + .../input/projects/project6/children/c.pkl | 1 + .../output/projects/badProjectDeps4/bug.err | 10 +- .../projects/missingProjectDeps/bug.err | 5 +- .../output/projects/project1/globbing.pcf | 15 ++ .../output/projects/project6/children.pcf | 11 + .../output/projects/project6/children/a.pcf | 1 + .../output/projects/project6/children/b.pcf | 1 + .../output/projects/project6/children/c.pcf | 1 + .../test/kotlin/org/pkl/core/EvaluatorTest.kt | 158 ++++++++++++++ .../pkl/core/LanguageSnippetTestsEngine.kt | 23 ++- .../org/pkl/core/project/ProjectTest.kt | 10 +- .../org/pkl/core/project/project5/PklProject | 8 + .../project/project5/PklProject.deps.json | 17 ++ .../org/pkl/core/project/project5/main.pkl | 5 + .../org/pkl/core/project/project6/PklProject | 5 + .../project/project6/PklProject.deps.json | 10 + .../project/project6/globIntoDependency.pkl | 3 + .../project/project6/globWithinDependency.pkl | 3 + .../org/pkl/core/project/project7/PklProject | 8 + .../org/pkl/core/project/project7/main.pkl | 1 + .../org/pkl/core/project/project7/moduleA.pkl | 0 .../org/pkl/core/project/project7/moduleB.pkl | 0 .../org/pkl/executor/EmbeddedExecutorTest.kt | 2 +- stdlib/Project.pkl | 39 ++-- 49 files changed, 764 insertions(+), 235 deletions(-) create mode 100644 pkl-core/src/test/files/LanguageSnippetTests/input/projects/project6/PklProject create mode 100644 pkl-core/src/test/files/LanguageSnippetTests/input/projects/project6/PklProject.deps.json create mode 100644 pkl-core/src/test/files/LanguageSnippetTests/input/projects/project6/children.pkl create mode 100644 pkl-core/src/test/files/LanguageSnippetTests/input/projects/project6/children/a.pkl create mode 100644 pkl-core/src/test/files/LanguageSnippetTests/input/projects/project6/children/b.pkl create mode 100644 pkl-core/src/test/files/LanguageSnippetTests/input/projects/project6/children/c.pkl create mode 100644 pkl-core/src/test/files/LanguageSnippetTests/output/projects/project6/children.pcf create mode 100644 pkl-core/src/test/files/LanguageSnippetTests/output/projects/project6/children/a.pcf create mode 100644 pkl-core/src/test/files/LanguageSnippetTests/output/projects/project6/children/b.pcf create mode 100644 pkl-core/src/test/files/LanguageSnippetTests/output/projects/project6/children/c.pcf create mode 100644 pkl-core/src/test/resources/org/pkl/core/project/project5/PklProject create mode 100644 pkl-core/src/test/resources/org/pkl/core/project/project5/PklProject.deps.json create mode 100644 pkl-core/src/test/resources/org/pkl/core/project/project5/main.pkl create mode 100644 pkl-core/src/test/resources/org/pkl/core/project/project6/PklProject create mode 100644 pkl-core/src/test/resources/org/pkl/core/project/project6/PklProject.deps.json create mode 100644 pkl-core/src/test/resources/org/pkl/core/project/project6/globIntoDependency.pkl create mode 100644 pkl-core/src/test/resources/org/pkl/core/project/project6/globWithinDependency.pkl create mode 100644 pkl-core/src/test/resources/org/pkl/core/project/project7/PklProject create mode 100644 pkl-core/src/test/resources/org/pkl/core/project/project7/main.pkl create mode 100644 pkl-core/src/test/resources/org/pkl/core/project/project7/moduleA.pkl create mode 100644 pkl-core/src/test/resources/org/pkl/core/project/project7/moduleB.pkl diff --git a/pkl-core/src/main/java/org/pkl/core/EvaluatorImpl.java b/pkl-core/src/main/java/org/pkl/core/EvaluatorImpl.java index 9f9c2640..3bf4b9be 100644 --- a/pkl-core/src/main/java/org/pkl/core/EvaluatorImpl.java +++ b/pkl-core/src/main/java/org/pkl/core/EvaluatorImpl.java @@ -105,7 +105,8 @@ public class EvaluatorImpl implements Evaluator { packageResolver, projectDependencies == null ? null - : new ProjectDependenciesManager(projectDependencies))); + : new ProjectDependenciesManager( + projectDependencies, moduleResolver, securityManager))); }); this.timeout = timeout; // NOTE: would probably make sense to share executor between evaluators diff --git a/pkl-core/src/main/java/org/pkl/core/ast/builder/AstBuilder.java b/pkl-core/src/main/java/org/pkl/core/ast/builder/AstBuilder.java index c9a6b6e1..8ed49b91 100644 --- a/pkl-core/src/main/java/org/pkl/core/ast/builder/AstBuilder.java +++ b/pkl-core/src/main/java/org/pkl/core/ast/builder/AstBuilder.java @@ -1825,6 +1825,8 @@ public final class AstBuilder extends AbstractAstBuilder { } catch (VmException e) { throw exceptionBuilder() .evalError(e.getMessage(), e.getMessageArguments()) + .withCause(e.getCause()) + .withHint(e.getHint()) .withSourceSection(createSourceSection(importUriCtx)) .build(); } diff --git a/pkl-core/src/main/java/org/pkl/core/ast/expression/unary/ReadGlobNode.java b/pkl-core/src/main/java/org/pkl/core/ast/expression/unary/ReadGlobNode.java index 1a58ed82..225ba491 100644 --- a/pkl-core/src/main/java/org/pkl/core/ast/expression/unary/ReadGlobNode.java +++ b/pkl-core/src/main/java/org/pkl/core/ast/expression/unary/ReadGlobNode.java @@ -22,6 +22,7 @@ import com.oracle.truffle.api.frame.FrameDescriptor; import com.oracle.truffle.api.nodes.NodeInfo; import com.oracle.truffle.api.source.SourceSection; import java.io.IOException; +import java.net.URISyntaxException; import org.graalvm.collections.EconomicMap; import org.pkl.core.SecurityManagerException; import org.pkl.core.ast.member.SharedMemberNode; @@ -33,6 +34,7 @@ import org.pkl.core.runtime.VmMapping; import org.pkl.core.runtime.VmObjectBuilder; import org.pkl.core.util.GlobResolver; import org.pkl.core.util.GlobResolver.InvalidGlobPatternException; +import org.pkl.core.util.IoUtils; import org.pkl.core.util.LateInit; @NodeInfo(shortName = "read*") @@ -73,7 +75,7 @@ public abstract class ReadGlobNode extends AbstractReadNode { var globUri = parseUri(globPattern); var context = VmContext.get(this); try { - var resolvedUri = currentModule.resolveUri(globUri); + var resolvedUri = IoUtils.resolve(context.getSecurityManager(), currentModule, globUri); var reader = context.getResourceManager().getReader(resolvedUri, this); if (!reader.isGlobbable()) { throw exceptionBuilder().evalError("cannotGlobUri", globUri, globUri.getScheme()).build(); @@ -94,7 +96,7 @@ public abstract class ReadGlobNode extends AbstractReadNode { return cachedResult; } catch (IOException e) { throw exceptionBuilder().evalError("ioErrorResolvingGlob", globPattern).withCause(e).build(); - } catch (SecurityManagerException | HttpClientInitException e) { + } catch (SecurityManagerException | HttpClientInitException | URISyntaxException e) { throw exceptionBuilder().withCause(e).build(); } catch (InvalidGlobPatternException e) { throw exceptionBuilder() diff --git a/pkl-core/src/main/java/org/pkl/core/module/ModuleKeys.java b/pkl-core/src/main/java/org/pkl/core/module/ModuleKeys.java index 60f6cb8c..a1881751 100644 --- a/pkl-core/src/main/java/org/pkl/core/module/ModuleKeys.java +++ b/pkl-core/src/main/java/org/pkl/core/module/ModuleKeys.java @@ -41,7 +41,6 @@ import org.pkl.core.util.ErrorMessages; import org.pkl.core.util.HttpUtils; import org.pkl.core.util.IoUtils; import org.pkl.core.util.Nullable; -import org.pkl.core.util.Pair; /** Utilities for creating and using {@link ModuleKey}s. */ public final class ModuleKeys { @@ -290,14 +289,18 @@ public final class ModuleKeys { } } - private static class File extends DependencyAwareModuleKey { + private static class File implements ModuleKey { final URI uri; File(URI uri) { - super(uri); this.uri = uri; } + @Override + public URI getUri() { + return uri; + } + @Override public boolean hasElement(SecurityManager securityManager, URI uri) throws SecurityManagerException { @@ -329,17 +332,18 @@ public final class ModuleKeys { } @Override - protected Map getDependencies() { - var projectDepsManager = VmContext.get(null).getProjectDependenciesManager(); - if (projectDepsManager == null || !projectDepsManager.hasPath(Path.of(uri))) { - throw new PackageLoadError("cannotResolveDependencyNoProject"); - } - return projectDepsManager.getDependencies(); + public boolean isGlobbable() { + return true; } @Override - protected PackageLoadError cannotFindDependency(String name) { - return new PackageLoadError("cannotFindDependencyInProject", name); + public boolean isLocal() { + return true; + } + + @Override + public boolean hasHierarchicalUris() { + return true; } } @@ -547,40 +551,29 @@ public final class ModuleKeys { } } - /** Base implementation; knows how to resolve dependencies prefixed with @. */ - private abstract static class DependencyAwareModuleKey implements ModuleKey { + private abstract static class AbstractPackage implements ModuleKey { - protected final URI uri; + protected final PackageAssetUri packageAssetUri; - DependencyAwareModuleKey(URI uri) { - this.uri = uri; - } - - @Override - public URI getUri() { - return uri; - } - - protected Pair parseDependencyNotation(String importPath) { - var idx = importPath.indexOf('/'); - if (idx == -1) { - // treat named dependency without a subpath as the root path. - // i.e. resolve to `@foo` to `package://example.com/foo@1.0.0#/` - return Pair.of(importPath.substring(1), "/"); - } - return Pair.of(importPath.substring(1, idx), importPath.substring(idx)); + AbstractPackage(PackageAssetUri packageAssetUri) { + this.packageAssetUri = packageAssetUri; } protected abstract Map getDependencies() throws IOException, SecurityManagerException; @Override - public boolean isLocal() { + public boolean hasHierarchicalUris() { return true; } @Override - public boolean hasHierarchicalUris() { + public boolean hasFragmentPaths() { + return true; + } + + @Override + public boolean isLocal() { return true; } @@ -589,37 +582,36 @@ public final class ModuleKeys { return true; } - private URI resolveDependencyNotation(String notation) - throws IOException, SecurityManagerException { - var parsed = parseDependencyNotation(notation); - var name = parsed.getFirst(); - var path = parsed.getSecond(); - var dependency = getDependencies().get(name); - if (dependency == null) { - throw cannotFindDependency(name); - } - return dependency.getPackageUri().toPackageAssetUri(path).getUri(); + @Override + public URI getUri() { + return packageAssetUri.getUri(); } @Override public URI resolveUri(URI baseUri, URI importUri) throws IOException, SecurityManagerException { - if (importUri.isAbsolute() || !importUri.getPath().startsWith("@")) { + var ssp = importUri.getSchemeSpecificPart(); + if (importUri.isAbsolute() || !ssp.startsWith("@")) { return ModuleKey.super.resolveUri(baseUri, importUri); } - return resolveDependencyNotation(importUri.getPath()); + var parsed = IoUtils.parseDependencyNotation(ssp); + var name = parsed.getFirst(); + var path = parsed.getSecond(); + var dependency = getDependencies().get(name); + if (dependency == null) { + throw new PackageLoadError( + "cannotFindDependencyInPackage", + name, + packageAssetUri.getPackageUri().getDisplayName()); + } + return dependency.getPackageUri().toPackageAssetUri(path).getUri(); } - - protected abstract PackageLoadError cannotFindDependency(String name); } /** Represents a module imported via the {@code package} scheme. */ - private static class Package extends DependencyAwareModuleKey { - - private final PackageAssetUri packageAssetUri; + private static class Package extends AbstractPackage { Package(PackageAssetUri packageAssetUri) { - super(packageAssetUri.getUri()); - this.packageAssetUri = packageAssetUri; + super(packageAssetUri); } private PackageResolver getPackageResolver() { @@ -631,6 +623,7 @@ public final class ModuleKeys { @Override public ResolvedModuleKey resolve(SecurityManager securityManager) throws IOException, SecurityManagerException { + var uri = packageAssetUri.getUri(); securityManager.checkResolveModule(uri); var bytes = getPackageResolver() @@ -654,11 +647,6 @@ public final class ModuleKeys { return getPackageResolver().hasElement(assetUri, assetUri.getPackageUri().getChecksums()); } - @Override - public boolean hasFragmentPaths() { - return true; - } - @Override protected Map getDependencies() throws IOException, SecurityManagerException { @@ -667,12 +655,6 @@ public final class ModuleKeys { packageAssetUri.getPackageUri(), packageAssetUri.getPackageUri().getChecksums()) .getDependencies(); } - - @Override - protected PackageLoadError cannotFindDependency(String name) { - return new PackageLoadError( - "cannotFindDependencyInPackage", name, packageAssetUri.getPackageUri().getDisplayName()); - } } /** @@ -680,15 +662,11 @@ public final class ModuleKeys { * *

The {@code projectpackage} scheme is what project-local dependencies resolve to when * imported using dependency notation (for example, {@code import "@foo/bar.pkl"}). This scheme is - * an internal implementation detail, and we do not expect a project to declare this. + * an internal implementation detail, and we do not expect a module to declare this. */ - private static class ProjectPackage extends DependencyAwareModuleKey { - - private final PackageAssetUri packageAssetUri; - + public static class ProjectPackage extends AbstractPackage { ProjectPackage(PackageAssetUri packageAssetUri) { - super(packageAssetUri.getUri()); - this.packageAssetUri = packageAssetUri; + super(packageAssetUri); } private PackageResolver getPackageResolver() { @@ -697,37 +675,36 @@ public final class ModuleKeys { return packageResolver; } - private ProjectDependenciesManager getProjectDepsResolver() { + private ProjectDependenciesManager getProjectDependenciesManager() { var projectDepsManager = VmContext.get(null).getProjectDependenciesManager(); assert projectDepsManager != null; return projectDepsManager; } - private @Nullable Path getLocalPath(Dependency dependency, PackageAssetUri packageAssetUri) { + private @Nullable URI getLocalUri(Dependency dependency) { + return getLocalUri(dependency, packageAssetUri); + } + + private @Nullable URI getLocalUri(Dependency dependency, PackageAssetUri assetUri) { if (!(dependency instanceof LocalDependency localDependency)) { return null; } - return localDependency.resolveAssetPath( - getProjectDepsResolver().getProjectDir(), packageAssetUri); - } - - private @Nullable Path getLocalPath(Dependency dependency) { - if (!(dependency instanceof LocalDependency)) { - return null; - } - return getLocalPath(dependency, packageAssetUri); + return localDependency.resolveAssetUri( + getProjectDependenciesManager().getProjectBaseUri(), assetUri); } @Override public ResolvedModuleKey resolve(SecurityManager securityManager) throws IOException, SecurityManagerException { - securityManager.checkResolveModule(packageAssetUri.getUri()); + var uri = packageAssetUri.getUri(); + securityManager.checkResolveModule(uri); var dependency = - getProjectDepsResolver().getResolvedDependency(packageAssetUri.getPackageUri()); - var path = getLocalPath(dependency); - if (path != null) { - securityManager.checkResolveModule(path.toUri()); - return ResolvedModuleKeys.file(this, path.toUri(), path); + getProjectDependenciesManager().getResolvedDependency(packageAssetUri.getPackageUri()); + var local = getLocalUri(dependency); + if (local != null) { + var resolved = + VmContext.get(null).getModuleResolver().resolve(local).resolve(securityManager); + return ResolvedModuleKeys.delegated(resolved, this); } var dep = (Dependency.RemoteDependency) dependency; assert dep.getChecksums() != null; @@ -741,11 +718,15 @@ public final class ModuleKeys { securityManager.checkResolveModule(baseUri); var packageAssetUri = PackageAssetUri.create(baseUri); var dependency = - getProjectDepsResolver().getResolvedDependency(packageAssetUri.getPackageUri()); - var path = getLocalPath(dependency, packageAssetUri); - if (path != null) { - securityManager.checkResolveModule(path.toUri()); - return FileResolver.listElements(path); + getProjectDependenciesManager().getResolvedDependency(packageAssetUri.getPackageUri()); + var local = getLocalUri(dependency, packageAssetUri); + if (local != null) { + var moduleKey = VmContext.get(null).getModuleResolver().resolve(local); + if (!moduleKey.isGlobbable()) { + throw new PackageLoadError( + "cannotResolveInLocalDependencyNotGlobbable", local.getScheme()); + } + return moduleKey.listElements(securityManager, local); } var dep = (Dependency.RemoteDependency) dependency; assert dep.getChecksums() != null; @@ -758,42 +739,36 @@ public final class ModuleKeys { securityManager.checkResolveModule(elementUri); var packageAssetUri = PackageAssetUri.create(elementUri); var dependency = - getProjectDepsResolver().getResolvedDependency(packageAssetUri.getPackageUri()); - var path = getLocalPath(dependency, packageAssetUri); - if (path != null) { - securityManager.checkResolveModule(path.toUri()); - return FileResolver.hasElement(path); + getProjectDependenciesManager().getResolvedDependency(packageAssetUri.getPackageUri()); + var local = getLocalUri(dependency, packageAssetUri); + if (local != null) { + var moduleKey = VmContext.get(null).getModuleResolver().resolve(local); + if (!moduleKey.isGlobbable() && !moduleKey.isLocal()) { + throw new PackageLoadError( + "cannotResolveInLocalDependencyNotGlobbableNorLocal", local.getScheme()); + } + return moduleKey.hasElement(securityManager, local); } var dep = (Dependency.RemoteDependency) dependency; assert dep.getChecksums() != null; return getPackageResolver().hasElement(packageAssetUri, dep.getChecksums()); } - @Override - public boolean hasFragmentPaths() { - return true; - } - @Override protected Map getDependencies() throws IOException, SecurityManagerException { var packageUri = packageAssetUri.getPackageUri(); - var projectResolver = getProjectDepsResolver(); + var projectResolver = getProjectDependenciesManager(); if (projectResolver.isLocalPackage(packageUri)) { return projectResolver.getLocalPackageDependencies(packageUri); } var dep = - (Dependency.RemoteDependency) getProjectDepsResolver().getResolvedDependency(packageUri); + (Dependency.RemoteDependency) + getProjectDependenciesManager().getResolvedDependency(packageUri); assert dep.getChecksums() != null; var dependencyMetadata = getPackageResolver().getDependencyMetadata(packageUri, dep.getChecksums()); return projectResolver.getResolvedDependenciesForPackage(packageUri, dependencyMetadata); } - - @Override - protected PackageLoadError cannotFindDependency(String name) { - return new PackageLoadError( - "cannotFindDependencyInPackage", name, packageAssetUri.getPackageUri().getDisplayName()); - } } } diff --git a/pkl-core/src/main/java/org/pkl/core/module/ProjectDependenciesManager.java b/pkl-core/src/main/java/org/pkl/core/module/ProjectDependenciesManager.java index 633ee277..67967477 100644 --- a/pkl-core/src/main/java/org/pkl/core/module/ProjectDependenciesManager.java +++ b/pkl-core/src/main/java/org/pkl/core/module/ProjectDependenciesManager.java @@ -17,13 +17,14 @@ package org.pkl.core.module; import java.io.IOException; import java.net.URI; -import java.net.URISyntaxException; -import java.nio.file.Files; -import java.nio.file.Path; import java.util.HashMap; import java.util.Map; +import java.util.Objects; import javax.annotation.concurrent.GuardedBy; import org.graalvm.collections.EconomicMap; +import org.pkl.core.PklBugException; +import org.pkl.core.SecurityManager; +import org.pkl.core.SecurityManagerException; import org.pkl.core.packages.Dependency; import org.pkl.core.packages.DependencyMetadata; import org.pkl.core.packages.PackageLoadError; @@ -31,8 +32,10 @@ import org.pkl.core.packages.PackageUri; import org.pkl.core.project.CanonicalPackageUri; import org.pkl.core.project.DeclaredDependencies; import org.pkl.core.project.ProjectDeps; +import org.pkl.core.runtime.ModuleResolver; import org.pkl.core.runtime.VmExceptionBuilder; import org.pkl.core.util.EconomicMaps; +import org.pkl.core.util.IoUtils; import org.pkl.core.util.json.Json.JsonParseException; public final class ProjectDependenciesManager { @@ -41,7 +44,9 @@ public final class ProjectDependenciesManager { public static final String PKL_PROJECT_DEPS_FILENAME = "PklProject.deps.json"; private final DeclaredDependencies declaredDependencies; - private final Path projectDir; + private final URI projectBaseUri; + private final ModuleResolver moduleResolver; + private final SecurityManager securityManager; @GuardedBy("lock") private ProjectDeps projectDeps; @@ -59,13 +64,21 @@ public final class ProjectDependenciesManager { private final Object lock = new Object(); - public ProjectDependenciesManager(DeclaredDependencies declaredDependencies) { + public ProjectDependenciesManager( + DeclaredDependencies declaredDependencies, + ModuleResolver moduleResolver, + SecurityManager securityManager) { this.declaredDependencies = declaredDependencies; - this.projectDir = Path.of(declaredDependencies.getProjectFileUri()).getParent(); + // new URI("scheme://host/a/b/c.txt").resolve(".") == new URI("scheme://host/a/b/") + this.projectBaseUri = IoUtils.resolve(declaredDependencies.getProjectFileUri(), "."); + this.moduleResolver = moduleResolver; + this.securityManager = securityManager; } - public boolean hasPath(Path path) { - return path.startsWith(projectDir); + public boolean hasUri(URI uri) { + return projectBaseUri.getScheme().equals(uri.getScheme()) + && Objects.equals(projectBaseUri.getAuthority(), uri.getAuthority()) + && uri.getPath().startsWith(projectBaseUri.getPath()); } private void ensureDependenciesInitialized() { @@ -194,30 +207,39 @@ public final class ProjectDependenciesManager { return dep; } - public Path getProjectDir() { - return projectDir; + public URI getProjectBaseUri() { + return projectBaseUri; } - public Path getProjectDepsFile() { - return projectDir.resolve(PKL_PROJECT_DEPS_FILENAME); + public URI getProjectDepsFileUri() { + return IoUtils.resolve(projectBaseUri, PKL_PROJECT_DEPS_FILENAME); + } + + public URI getProjectFileUri() { + return declaredDependencies.getProjectFileUri(); } private ProjectDeps getProjectDeps() { synchronized (lock) { if (projectDeps == null) { - var depsPath = getProjectDepsFile(); - if (!Files.exists(depsPath)) { - throw new VmExceptionBuilder() - .evalError("missingProjectDepsJson", projectDir.toUri()) - .build(); - } + var depsUri = getProjectDepsFileUri(); + var moduleKey = moduleResolver.resolve(depsUri); try { - projectDeps = ProjectDeps.parse(depsPath); - } catch (IOException | URISyntaxException | JsonParseException e) { + // treat PklProject.deps.json as a module read, rather than introduce a new API. + var depsJson = moduleKey.resolve(securityManager).loadSource(); + projectDeps = ProjectDeps.parse(depsJson); + } catch (IOException e) { throw new VmExceptionBuilder() - .evalError("invalidProjectDepsJson", depsPath.toUri(), e.getMessage()) + .evalError("cannotLoadProjectDepsJson", depsUri) .withCause(e) + .withHint(e.getMessage() != null ? e.getMessage() : ("Encountered error: " + e)) .build(); + } catch (JsonParseException e) { + throw new VmExceptionBuilder() + .evalError("invalidProjectDepsJson", depsUri, e.getMessage()) + .build(); + } catch (SecurityManagerException e) { + throw PklBugException.unreachableCode(); } } return projectDeps; diff --git a/pkl-core/src/main/java/org/pkl/core/module/ResolvedModuleKeys.java b/pkl-core/src/main/java/org/pkl/core/module/ResolvedModuleKeys.java index 8fcaaa96..c3fe4f86 100644 --- a/pkl-core/src/main/java/org/pkl/core/module/ResolvedModuleKeys.java +++ b/pkl-core/src/main/java/org/pkl/core/module/ResolvedModuleKeys.java @@ -54,6 +54,14 @@ public final class ResolvedModuleKeys { return new Virtual(original, uri, sourceText, cached); } + /** + * Creates a resolved module key that behaves like {@code delegate}, except with {@code original} + * as its original module key. + */ + public static ResolvedModuleKey delegated(ResolvedModuleKey delegate, ModuleKey original) { + return new Delegated(delegate, original); + } + private static class File implements ResolvedModuleKey { final ModuleKey original; final URI uri; @@ -145,4 +153,30 @@ public final class ResolvedModuleKeys { return sourceText; } } + + private static class Delegated implements ResolvedModuleKey { + + private final ResolvedModuleKey delegate; + private final ModuleKey original; + + Delegated(ResolvedModuleKey delegate, ModuleKey original) { + this.delegate = delegate; + this.original = original; + } + + @Override + public ModuleKey getOriginal() { + return original; + } + + @Override + public URI getUri() { + return delegate.getUri(); + } + + @Override + public String loadSource() throws IOException { + return delegate.loadSource(); + } + } } diff --git a/pkl-core/src/main/java/org/pkl/core/packages/Dependency.java b/pkl-core/src/main/java/org/pkl/core/packages/Dependency.java index 991462a9..c9ba95e4 100644 --- a/pkl-core/src/main/java/org/pkl/core/packages/Dependency.java +++ b/pkl-core/src/main/java/org/pkl/core/packages/Dependency.java @@ -15,9 +15,11 @@ */ package org.pkl.core.packages; +import java.net.URI; import java.nio.file.Path; import java.util.Objects; import org.pkl.core.Version; +import org.pkl.core.util.IoUtils; import org.pkl.core.util.Nullable; public abstract class Dependency { @@ -48,10 +50,10 @@ public abstract class Dependency { return path; } - public Path resolveAssetPath(Path projectDir, PackageAssetUri packageAssetUri) { + public URI resolveAssetUri(URI projectBaseUri, PackageAssetUri packageAssetUri) { // drop 1 to remove leading `/` var assetPath = packageAssetUri.getAssetPath().substring(1); - return projectDir.resolve(path).resolve(assetPath); + return projectBaseUri.resolve(IoUtils.toNormalizedPathString(path.resolve(assetPath))); } @Override diff --git a/pkl-core/src/main/java/org/pkl/core/project/Project.java b/pkl-core/src/main/java/org/pkl/core/project/Project.java index bc591b96..554cb8aa 100644 --- a/pkl-core/src/main/java/org/pkl/core/project/Project.java +++ b/pkl-core/src/main/java/org/pkl/core/project/Project.java @@ -17,6 +17,7 @@ package org.pkl.core.project; import java.net.URI; import java.net.URISyntaxException; +import java.nio.file.FileSystemNotFoundException; import java.nio.file.Path; import java.util.HashMap; import java.util.List; @@ -27,6 +28,7 @@ import java.util.regex.Pattern; import java.util.stream.Collectors; import org.pkl.core.Composite; import org.pkl.core.Duration; +import org.pkl.core.Evaluator; import org.pkl.core.EvaluatorBuilder; import org.pkl.core.ModuleSource; import org.pkl.core.PClassInfo; @@ -41,9 +43,11 @@ import org.pkl.core.Version; import org.pkl.core.module.ModuleKeyFactories; import org.pkl.core.packages.Checksums; import org.pkl.core.packages.Dependency.RemoteDependency; +import org.pkl.core.packages.PackageLoadError; import org.pkl.core.packages.PackageUri; import org.pkl.core.packages.PackageUtils; import org.pkl.core.resource.ResourceReaders; +import org.pkl.core.util.IoUtils; import org.pkl.core.util.Nullable; /** Java representation of module {@code pkl.Project}. */ @@ -52,8 +56,8 @@ public final class Project { private final DeclaredDependencies dependencies; private final EvaluatorSettings evaluatorSettings; private final URI projectFileUri; - private final Path projectDir; - private final List tests; + private final URI projectBaseUri; + private final List tests; private final Map localProjectDependencies; /** @@ -81,10 +85,7 @@ public final class Project { .addEnvironmentVariables(envVars) .setTimeout(timeout) .build()) { - var output = evaluator.evaluateOutputValueAs(ModuleSource.path(path), PClassInfo.Project); - return Project.parseProject(output); - } catch (URISyntaxException e) { - throw new PklException(e.getMessage(), e); + return load(evaluator, ModuleSource.path(path)); } } @@ -103,6 +104,31 @@ public final class Project { return loadFromPath(path, SecurityManagers.defaultManager, null); } + /** Loads a project from the given source. */ + public static Project load(ModuleSource moduleSource) { + try (var evaluator = + EvaluatorBuilder.unconfigured() + .setSecurityManager(SecurityManagers.defaultManager) + .setStackFrameTransformer(StackFrameTransformers.defaultTransformer) + .addModuleKeyFactory(ModuleKeyFactories.standardLibrary) + .addModuleKeyFactory(ModuleKeyFactories.file) + .addModuleKeyFactory(ModuleKeyFactories.classPath(Project.class.getClassLoader())) + .addResourceReader(ResourceReaders.environmentVariable()) + .addResourceReader(ResourceReaders.file()) + .build()) { + return load(evaluator, moduleSource); + } + } + + public static Project load(Evaluator evaluator, ModuleSource moduleSource) { + try { + var output = evaluator.evaluateOutputValueAs(moduleSource, PClassInfo.Project); + return Project.parseProject(output); + } catch (URISyntaxException e) { + throw new PklException(e.getMessage(), e); + } + } + private static DeclaredDependencies parseDependencies( PObject module, URI projectFileUri, @Nullable PackageUri packageUri) throws URISyntaxException { @@ -143,7 +169,7 @@ public final class Project { var pkgObj = getNullableProperty(module, "package"); var projectFileUri = URI.create((String) module.getProperty("projectFileUri")); var dependencies = parseDependencies(module, projectFileUri, null); - var projectDir = Path.of(projectFileUri).getParent(); + var projectBaseUri = IoUtils.resolve(projectFileUri, "."); Package pkg = null; if (pkgObj != null) { pkg = parsePackage((PObject) pkgObj); @@ -152,12 +178,12 @@ public final class Project { getProperty( module, "evaluatorSettings", - (settings) -> parseEvaluatorSettings(settings, projectDir)); + (settings) -> parseEvaluatorSettings(settings, projectBaseUri)); @SuppressWarnings("unchecked") var testPathStrs = (List) getProperty(module, "tests"); var tests = testPathStrs.stream() - .map((it) -> projectDir.resolve(it).normalize()) + .map((it) -> projectBaseUri.resolve(it).normalize()) .collect(Collectors.toList()); var localProjectDependencies = parseLocalProjectDependencies(module); return new Project( @@ -165,7 +191,7 @@ public final class Project { dependencies, evaluatorSettings, projectFileUri, - projectDir, + projectBaseUri, tests, localProjectDependencies); } @@ -185,7 +211,7 @@ public final class Project { } @SuppressWarnings("unchecked") - private static EvaluatorSettings parseEvaluatorSettings(Object settings, Path projectDir) { + private static EvaluatorSettings parseEvaluatorSettings(Object settings, URI projectBaseUri) { var pSettings = (PObject) settings; var externalProperties = getNullableProperty(pSettings, "externalProperties", Project::asMap); var env = getNullableProperty(pSettings, "env", Project::asMap); @@ -194,16 +220,18 @@ public final class Project { getNullableProperty(pSettings, "allowedResources", Project::asPatternList); var noCache = (Boolean) getNullableProperty(pSettings, "noCache"); var modulePathStrs = (List) getNullableProperty(pSettings, "modulePath"); + var timeout = (Duration) getNullableProperty(pSettings, "timeout"); + List modulePath = null; if (modulePathStrs != null) { modulePath = modulePathStrs.stream() - .map((it) -> projectDir.resolve(it).normalize()) + .map((it) -> resolveNullablePath(it, projectBaseUri, "modulePath")) .collect(Collectors.toList()); } - var timeout = (Duration) getNullableProperty(pSettings, "timeout"); - var moduleCacheDir = getNullablePath(pSettings, "moduleCacheDir", projectDir); - var rootDir = getNullablePath(pSettings, "rootDir", projectDir); + + var moduleCacheDir = getNullablePath(pSettings, "moduleCacheDir", projectBaseUri); + var rootDir = getNullablePath(pSettings, "rootDir", projectBaseUri); return new EvaluatorSettings( externalProperties, env, @@ -261,10 +289,28 @@ public final class Project { return new URI((String) value); } + /** + * Resolve a path string against projectBaseUri. + * + * @throws PackageLoadError if projectBaseUri is not a {@code file:} URI. + */ + private static @Nullable Path resolveNullablePath( + @Nullable String path, URI projectBaseUri, String propertyName) { + if (path == null) { + return null; + } + try { + return Path.of(projectBaseUri).resolve(path).normalize(); + } catch (FileSystemNotFoundException e) { + throw new PackageLoadError( + "relativePathPropertyDefinedByProjectFromNonFileUri", projectBaseUri, propertyName); + } + } + private static @Nullable Path getNullablePath( - Composite object, String propertyName, Path projectDir) { - return getNullableProperty( - object, propertyName, (obj) -> projectDir.resolve((String) obj).normalize()); + Composite object, String propertyName, URI projectBaseUri) { + return resolveNullablePath( + (String) getNullableProperty(object, propertyName), projectBaseUri, propertyName); } @SuppressWarnings("unchecked") @@ -309,14 +355,14 @@ public final class Project { DeclaredDependencies dependencies, EvaluatorSettings evaluatorSettings, URI projectFileUri, - Path projectDir, - List tests, + URI projectBaseUri, + List tests, Map localProjectDependencies) { this.pkg = pkg; this.dependencies = dependencies; this.evaluatorSettings = evaluatorSettings; this.projectFileUri = projectFileUri; - this.projectDir = projectDir; + this.projectBaseUri = projectBaseUri; this.tests = tests; this.localProjectDependencies = localProjectDependencies; } @@ -334,7 +380,16 @@ public final class Project { } public List getTests() { - return tests; + return tests.stream() + .map( + (it) -> { + try { + return Path.of(it); + } catch (FileSystemNotFoundException e) { + throw new PackageLoadError("invalidUsageOfProjectFromNonFileUri"); + } + }) + .collect(Collectors.toList()); } @Override @@ -366,11 +421,17 @@ public final class Project { return localProjectDependencies; } + public URI getProjectBaseUri() { + return projectBaseUri; + } + public Path getProjectDir() { - return projectDir; + assert projectBaseUri.getScheme().equalsIgnoreCase("file"); + return Path.of(projectBaseUri); } public static class EvaluatorSettings { + private final @Nullable Map externalProperties; private final @Nullable Map env; private final @Nullable List allowedModules; diff --git a/pkl-core/src/main/java/org/pkl/core/project/ProjectPackager.java b/pkl-core/src/main/java/org/pkl/core/project/ProjectPackager.java index 28492eae..8491bf09 100644 --- a/pkl-core/src/main/java/org/pkl/core/project/ProjectPackager.java +++ b/pkl-core/src/main/java/org/pkl/core/project/ProjectPackager.java @@ -45,6 +45,7 @@ import org.pkl.core.SecurityManagerException; import org.pkl.core.StackFrameTransformer; import org.pkl.core.ast.builder.ImportsAndReadsParser; import org.pkl.core.http.HttpClient; +import org.pkl.core.module.ModuleKeyFactories; import org.pkl.core.module.ModuleKeys; import org.pkl.core.module.ProjectDependenciesManager; import org.pkl.core.module.ResolvedModuleKeys; @@ -55,6 +56,7 @@ import org.pkl.core.packages.DependencyMetadata; import org.pkl.core.packages.PackageLoadError; import org.pkl.core.packages.PackageResolver; import org.pkl.core.packages.PackageUri; +import org.pkl.core.runtime.ModuleResolver; import org.pkl.core.runtime.VmExceptionBuilder; import org.pkl.core.util.ByteArrayUtils; import org.pkl.core.util.ErrorMessages; @@ -98,6 +100,7 @@ public final class ProjectPackager { private final Path workingDir; private final String outputPathPattern; private final StackFrameTransformer stackFrameTransformer; + private final SecurityManager securityManager; private final PackageResolver packageResolver; private final boolean skipPublishCheck; private final Writer outputWriter; @@ -115,6 +118,7 @@ public final class ProjectPackager { this.workingDir = workingDir; this.outputPathPattern = outputPathPattern; this.stackFrameTransformer = stackFrameTransformer; + this.securityManager = securityManager; // intentionally use InMemoryPackageResolver this.packageResolver = PackageResolver.getInstance(securityManager, httpClient, null); this.skipPublishCheck = skipPublishCheck; @@ -226,7 +230,12 @@ public final class ProjectPackager { new HashMap( project.getDependencies().getLocalDependencies().size() + project.getDependencies().getRemoteDependencies().size()); - var projectDependenciesManager = new ProjectDependenciesManager(project.getDependencies()); + // module resolver is only used for reading PklProject.deps.json, so provide one that reads + // files. + var moduleResolver = new ModuleResolver(List.of(ModuleKeyFactories.file)); + var projectDependenciesManager = + new ProjectDependenciesManager( + project.getDependencies(), moduleResolver, this.securityManager); for (var entry : project.getDependencies().getRemoteDependencies().entrySet()) { var resolved = (RemoteDependency) diff --git a/pkl-core/src/main/java/org/pkl/core/repl/ReplServer.java b/pkl-core/src/main/java/org/pkl/core/repl/ReplServer.java index 711f1b5e..a51220ac 100644 --- a/pkl-core/src/main/java/org/pkl/core/repl/ReplServer.java +++ b/pkl-core/src/main/java/org/pkl/core/repl/ReplServer.java @@ -89,7 +89,9 @@ public class ReplServer implements AutoCloseable { var languageRef = new MutableReference(null); packageResolver = PackageResolver.getInstance(securityManager, httpClient, moduleCacheDir); projectDependenciesManager = - projectDependencies == null ? null : new ProjectDependenciesManager(projectDependencies); + projectDependencies == null + ? null + : new ProjectDependenciesManager(projectDependencies, moduleResolver, securityManager); polyglotContext = VmUtils.createContext( () -> { diff --git a/pkl-core/src/main/java/org/pkl/core/resource/ResourceReaders.java b/pkl-core/src/main/java/org/pkl/core/resource/ResourceReaders.java index 6f9707fe..85610057 100644 --- a/pkl-core/src/main/java/org/pkl/core/resource/ResourceReaders.java +++ b/pkl-core/src/main/java/org/pkl/core/resource/ResourceReaders.java @@ -22,7 +22,6 @@ import java.net.URISyntaxException; import java.net.http.HttpRequest; import java.net.http.HttpResponse.BodyHandlers; import java.nio.file.Files; -import java.nio.file.Path; import java.util.ArrayList; import java.util.Collections; import java.util.List; @@ -39,6 +38,7 @@ import org.pkl.core.packages.Dependency.LocalDependency; import org.pkl.core.packages.PackageAssetUri; import org.pkl.core.packages.PackageResolver; import org.pkl.core.runtime.VmContext; +import org.pkl.core.runtime.VmExceptionBuilder; import org.pkl.core.util.ErrorMessages; import org.pkl.core.util.HttpUtils; import org.pkl.core.util.IoUtils; @@ -486,10 +486,9 @@ public final class ResourceReaders { throws IOException, URISyntaxException, SecurityManagerException { var assetUri = new PackageAssetUri(uri); var dependency = getProjectDepsResolver().getResolvedDependency(assetUri.getPackageUri()); - var path = getLocalPath(dependency, assetUri); - if (path != null) { - var bytes = Files.readAllBytes(path); - return Optional.of(new Resource(uri, bytes)); + var local = getLocalUri(dependency, assetUri); + if (local != null) { + return VmContext.get(null).getResourceManager().read(local, null); } var remoteDep = (Dependency.RemoteDependency) dependency; var bytes = getPackageResolver().getBytes(assetUri, true, remoteDep.getChecksums()); @@ -518,9 +517,15 @@ public final class ResourceReaders { var packageAssetUri = PackageAssetUri.create(baseUri); var dependency = getProjectDepsResolver().getResolvedDependency(packageAssetUri.getPackageUri()); - var path = getLocalPath(dependency, packageAssetUri); - if (path != null) { - return FileResolver.listElements(path); + var local = getLocalUri(dependency, packageAssetUri); + if (local != null) { + var reader = VmContext.get(null).getResourceManager().getResourceReader(local); + if (reader == null) { + throw new VmExceptionBuilder() + .evalError("noResourceReaderRegistered", local.getScheme()) + .build(); + } + return reader.listElements(securityManager, local); } var remoteDep = (Dependency.RemoteDependency) dependency; return getPackageResolver() @@ -534,9 +539,15 @@ public final class ResourceReaders { var packageAssetUri = PackageAssetUri.create(elementUri); var dependency = getProjectDepsResolver().getResolvedDependency(packageAssetUri.getPackageUri()); - var path = getLocalPath(dependency, packageAssetUri); - if (path != null) { - return FileResolver.hasElement(path); + var local = getLocalUri(dependency, packageAssetUri); + if (local != null) { + var reader = VmContext.get(null).getResourceManager().getResourceReader(local); + if (reader == null) { + throw new VmExceptionBuilder() + .evalError("noResourceReaderRegistered", local.getScheme()) + .build(); + } + return reader.hasElement(securityManager, local); } var remoteDep = (Dependency.RemoteDependency) dependency; return getPackageResolver() @@ -555,12 +566,12 @@ public final class ResourceReaders { return projectDepsManager; } - private @Nullable Path getLocalPath(Dependency dependency, PackageAssetUri packageAssetUri) { + private @Nullable URI getLocalUri(Dependency dependency, PackageAssetUri packageAssetUri) { if (!(dependency instanceof LocalDependency localDependency)) { return null; } - return localDependency.resolveAssetPath( - getProjectDepsResolver().getProjectDir(), packageAssetUri); + return localDependency.resolveAssetUri( + getProjectDepsResolver().getProjectBaseUri(), packageAssetUri); } } diff --git a/pkl-core/src/main/java/org/pkl/core/runtime/ResourceManager.java b/pkl-core/src/main/java/org/pkl/core/runtime/ResourceManager.java index 7c5e64b8..e9e95474 100644 --- a/pkl-core/src/main/java/org/pkl/core/runtime/ResourceManager.java +++ b/pkl-core/src/main/java/org/pkl/core/runtime/ResourceManager.java @@ -31,6 +31,7 @@ import org.pkl.core.packages.PackageLoadError; import org.pkl.core.resource.Resource; import org.pkl.core.resource.ResourceReader; import org.pkl.core.stdlib.VmObjectFactory; +import org.pkl.core.util.Nullable; public final class ResourceManager { private final Map resourceReaders = new HashMap<>(); @@ -67,17 +68,23 @@ public final class ResourceManager { } @TruffleBoundary - public Optional read(URI resourceUri, Node readNode) { + public Optional read(URI resourceUri, @Nullable Node readNode) { return resources.computeIfAbsent( resourceUri.normalize(), uri -> { try { securityManager.checkReadResource(uri); } catch (SecurityManagerException e) { - throw new VmExceptionBuilder().withCause(e).withLocation(readNode).build(); + throw new VmExceptionBuilder().withCause(e).withOptionalLocation(readNode).build(); } - var reader = getReader(resourceUri, readNode); + var reader = resourceReaders.get(uri.getScheme()); + if (reader == null) { + throw new VmExceptionBuilder() + .withOptionalLocation(readNode) + .evalError("noResourceReaderRegistered", resourceUri.getScheme()) + .build(); + } Optional resource; try { @@ -86,16 +93,16 @@ public final class ResourceManager { throw new VmExceptionBuilder() .evalError("ioErrorReadingResource", uri) .withCause(e) - .withLocation(readNode) + .withOptionalLocation(readNode) .build(); } catch (URISyntaxException e) { throw new VmExceptionBuilder() .evalError("invalidResourceUri", resourceUri) .withHint(e.getReason()) - .withLocation(readNode) + .withOptionalLocation(readNode) .build(); } catch (SecurityManagerException | PackageLoadError | HttpClientInitException e) { - throw new VmExceptionBuilder().withCause(e).withLocation(readNode).build(); + throw new VmExceptionBuilder().withCause(e).withOptionalLocation(readNode).build(); } if (resource.isEmpty()) return resource; @@ -108,8 +115,16 @@ public final class ResourceManager { throw new VmExceptionBuilder() .evalError("unsupportedResourceType", reader.getClass().getName(), res.getClass()) - .withLocation(readNode) + .withOptionalLocation(readNode) .build(); }); } + + /** + * Returns a {@link ResourceReader} registered to read the resource at {@code baseUri}, or {@code + * null} if there is none. + */ + public @Nullable ResourceReader getResourceReader(URI baseUri) { + return resourceReaders.get(baseUri.getScheme()); + } } diff --git a/pkl-core/src/main/java/org/pkl/core/runtime/VmExceptionBuilder.java b/pkl-core/src/main/java/org/pkl/core/runtime/VmExceptionBuilder.java index 9f580183..6fba2080 100644 --- a/pkl-core/src/main/java/org/pkl/core/runtime/VmExceptionBuilder.java +++ b/pkl-core/src/main/java/org/pkl/core/runtime/VmExceptionBuilder.java @@ -324,7 +324,7 @@ public final class VmExceptionBuilder { return this; } - public VmExceptionBuilder withHint(String hint) { + public VmExceptionBuilder withHint(@Nullable String hint) { this.hint = hint; return this; } diff --git a/pkl-core/src/main/java/org/pkl/core/util/GlobResolver.java b/pkl-core/src/main/java/org/pkl/core/util/GlobResolver.java index d64712b9..3d432a61 100644 --- a/pkl-core/src/main/java/org/pkl/core/util/GlobResolver.java +++ b/pkl-core/src/main/java/org/pkl/core/util/GlobResolver.java @@ -28,6 +28,7 @@ import java.util.Map; import java.util.WeakHashMap; import java.util.regex.Pattern; import java.util.stream.Collectors; +import org.pkl.core.PklBugException; import org.pkl.core.SecurityManager; import org.pkl.core.SecurityManagerException; import org.pkl.core.module.ModuleKey; @@ -490,7 +491,14 @@ public final class GlobResolver { } return result; } - var baseUri = enclosingModuleKey.resolveUri(enclosingUri, URI.create(basePath)); + URI baseUri; + try { + baseUri = IoUtils.resolve(securityManager, enclosingModuleKey, URI.create(basePath)); + } catch (URISyntaxException e) { + // assertion: this is only thrown if the pattern starts with a triple-dot import. + // the language will throw an error if glob imports is combined with triple-dots. + throw new PklBugException(e); + } resolveHierarchicalGlob( securityManager, diff --git a/pkl-core/src/main/java/org/pkl/core/util/IoUtils.java b/pkl-core/src/main/java/org/pkl/core/util/IoUtils.java index 02359057..5153adf3 100644 --- a/pkl-core/src/main/java/org/pkl/core/util/IoUtils.java +++ b/pkl-core/src/main/java/org/pkl/core/util/IoUtils.java @@ -39,7 +39,9 @@ import org.pkl.core.Platform; import org.pkl.core.SecurityManager; import org.pkl.core.SecurityManagerException; import org.pkl.core.module.ModuleKey; +import org.pkl.core.packages.PackageLoadError; import org.pkl.core.runtime.ReaderBase; +import org.pkl.core.runtime.VmContext; import org.pkl.core.runtime.VmExceptionBuilder; public final class IoUtils { @@ -299,29 +301,15 @@ public final class IoUtils { return resolve(baseUri, importUri); } - /** - * Resolves {@code importUri} against the module key. - * - *

When {@code importUri} contains a triple-dot, it is resolved if the module key returns true - * for both {@link ModuleKey#isLocal()} and {@link ModuleKey#hasHierarchicalUris()}. Otherwise, an - * error is thrown. - */ - public static URI resolve(SecurityManager securityManager, ModuleKey moduleKey, URI importUri) - throws URISyntaxException, IOException, SecurityManagerException { - if (importUri.isAbsolute()) { - return moduleKey.resolveUri(importUri); - } + private static URI resolveTripleDotImport( + SecurityManager securityManager, ModuleKey moduleKey, String tripleDotPath) + throws IOException, SecurityManagerException { var moduleKeyUri = moduleKey.getUri(); - var tripleDotPath = parseTripleDotPath(importUri); - if (tripleDotPath == null) { - return moduleKey.resolveUri(importUri); - } if (!moduleKey.isLocal() || !moduleKey.hasHierarchicalUris()) { throw new VmExceptionBuilder() .evalError("cannotResolveTripleDotImports", moduleKeyUri) .build(); } - var currentPath = moduleKey.hasFragmentPaths() ? moduleKeyUri.getFragment() : moduleKeyUri.getPath(); var effectiveImportPath = @@ -351,6 +339,69 @@ public final class IoUtils { throw new FileNotFoundException(); } + public static Pair parseDependencyNotation(String importPath) { + var idx = importPath.indexOf('/'); + if (idx == -1) { + // treat named dependency without a subpath as the root path. + // i.e. resolve to `@foo` to `package://example.com/foo@1.0.0#/` + return Pair.of(importPath.substring(1), "/"); + } + return Pair.of(importPath.substring(1, idx), importPath.substring(idx)); + } + + private static URI resolveProjectDependency(ModuleKey moduleKey, String notation) { + var parsed = parseDependencyNotation(notation); + var name = parsed.getFirst(); + var path = parsed.getSecond(); + var projectDependenciesManager = VmContext.get(null).getProjectDependenciesManager(); + if (!moduleKey.hasHierarchicalUris() && projectDependenciesManager != null) { + throw new PackageLoadError( + "cannotResolveDependencyWithoutHierarchicalUris", + projectDependenciesManager.getProjectFileUri()); + } + if (projectDependenciesManager == null + || !projectDependenciesManager.hasUri(moduleKey.getUri())) { + throw new PackageLoadError("cannotResolveDependencyNoProject"); + } + var dependency = projectDependenciesManager.getDependencies().get(name); + if (dependency != null) { + return dependency.getPackageUri().toPackageAssetUri(path).getUri(); + } + throw new PackageLoadError("cannotFindDependencyInProject", name); + } + + /** + * Resolves {@code importUri} against the module key. + * + *

When {@code importUri} contains a triple-dot, it is resolved if the module key returns true + * for both {@link ModuleKey#isLocal()} and {@link ModuleKey#hasHierarchicalUris()}. Otherwise, an + * error is thrown. + * + *

When {@code importUri} starts with a {@code @}, it is resolved if the module key supports + * dependency notation () + */ + public static URI resolve(SecurityManager securityManager, ModuleKey moduleKey, URI importUri) + throws URISyntaxException, IOException, SecurityManagerException { + if (importUri.isAbsolute()) { + return moduleKey.resolveUri(importUri); + } + var tripleDotPath = parseTripleDotPath(importUri); + if (tripleDotPath != null) { + return resolveTripleDotImport(securityManager, moduleKey, tripleDotPath); + } + var moduleScheme = moduleKey.getUri().getScheme(); + var isPackage = + moduleScheme.equalsIgnoreCase("package") || moduleScheme.equalsIgnoreCase("projectpackage"); + var relativePart = importUri.getSchemeSpecificPart(); + // Special-case handling of project dependencies. + // We'll allow the Package and ProjectPackage module keys to resolve dependency notation on + // their own. + if (relativePart.startsWith("@") && !isPackage) { + return resolveProjectDependency(moduleKey, relativePart); + } + return moduleKey.resolveUri(importUri); + } + public static URI resolve(URI baseUri, URI newUri) { if (newUri.isAbsolute()) return newUri; diff --git a/pkl-core/src/main/resources/org/pkl/core/errorMessages.properties b/pkl-core/src/main/resources/org/pkl/core/errorMessages.properties index 18b49520..562dd44d 100644 --- a/pkl-core/src/main/resources/org/pkl/core/errorMessages.properties +++ b/pkl-core/src/main/resources/org/pkl/core/errorMessages.properties @@ -542,6 +542,12 @@ Cannot combine glob imports with triple-dot module URIs. cannotGlobUri=\ Cannot expand glob pattern `{0}` because scheme `{1}` is not globbable. +cannotResolveInLocalDependencyNotGlobbable=\ +Cannot resolve import in local dependency because scheme `{0}` is not globbable. + +cannotResolveInLocalDependencyNotGlobbableNorLocal=\ +Cannot resolve import in local dependency because scheme `{0}` is not globbable and is not local. + expectedAnnotationClass=\ Expected an annotation class. @@ -680,6 +686,11 @@ Cannot resolve a triple-dot import from module URI `{0}`.\n\ \n\ Triple-dot imports may only be resolved by module schemes that are considered local, and have hierarchical URIs. +moduleDoesNotSupportDependencies=\ +Module `{0}` does not support importing dependencies.\n\ +\n\ +Dependencies can only be imported in modules that belong to a project, or within a package. + cannotHaveRelativeImport=\ Module `{0}` cannot have a relative import URI. @@ -819,6 +830,9 @@ Only type unions can have a default marker (*). invalidModuleOutputValue=\ Expected `output.value` of module `{2}` to be of type `{0}`, but got type `{1}`. +cannotResolveDependencyWithoutHierarchicalUris=\ +Cannot import dependency because project URI `{0}` does not have a hierarchical path. + cannotResolveDependencyNoProject=\ Cannot import dependency because there is no project found.\n\ \n\ @@ -830,6 +844,11 @@ Cannot find a dependency named `{0}`, because it is not declared in the current \n\ To fix this, add it to the `dependencies` section of your `PklProject` file, and resolve your dependencies. +cannotResolveDependencyFromReaderWithOpaqueUris=\ +Cannot resolve dependencies from module reader with opaque URIs.\n\ +\n\ +Module reader for scheme `{0}` does not support hierarchical URIs. + cannotFindDependencyInPackage=\ Cannot find dependency named `{0}`, because it was not declared in package `{1}`. @@ -856,10 +875,10 @@ For example, `example.com` in URI `project://example.com/my/package@1.0.0`. unexpectedChecksumInPackageUri=\ Did not expect to find a checksum component in this package URI. -missingProjectDepsJson=\ -Cannot resolve dependency because file `PklProject.deps.json` is missing in project directory `{0}`.\n\ +cannotLoadProjectDepsJson=\ +Encountered an error when attempting to load `PklProject.deps.json` at `{0}`.\n\ \n\ -Run `pkl project resolve` to create a new set of dependencies. +Try running `pkl project resolve` within the project directory to create a new set of dependencies. invalidProjectDepsJson=\ Cannot resolve dependency because file `{0}` is malformed.\n\ @@ -978,6 +997,16 @@ No package was declared in project `{0}`.\n\ \n\ Add a `package` section to the PklProject file. +relativePathPropertyDefinedByProjectFromNonFileUri=\ +Invalid property specified in project `{0}`\n\ +\n\ +Property `{1}` is only permitted in PklProject files loaded from `file:` URIs. + +invalidUsageOfProjectFromNonFileUri=\ +Invalid usage of project `{0}`\n\ +\n\ +This action can only be performed with PklProject files loaded from `file:` URIs. + packageTestsFailed=\ Failed to create package `{0}`, because its API tests are failing. diff --git a/pkl-core/src/test/files/LanguageSnippetTests/input/projects/project1/PklProject b/pkl-core/src/test/files/LanguageSnippetTests/input/projects/project1/PklProject index 693c5fa5..c2b97221 100644 --- a/pkl-core/src/test/files/LanguageSnippetTests/input/projects/project1/PklProject +++ b/pkl-core/src/test/files/LanguageSnippetTests/input/projects/project1/PklProject @@ -11,4 +11,5 @@ dependencies { uri = "package://localhost:0/badImportsWithinPackage@1.0.0" } ["project2"] = import("../project2/PklProject") + ["project6"] = import("../project6/PklProject") } diff --git a/pkl-core/src/test/files/LanguageSnippetTests/input/projects/project1/PklProject.deps.json b/pkl-core/src/test/files/LanguageSnippetTests/input/projects/project1/PklProject.deps.json index efbcd85e..55d204bf 100644 --- a/pkl-core/src/test/files/LanguageSnippetTests/input/projects/project1/PklProject.deps.json +++ b/pkl-core/src/test/files/LanguageSnippetTests/input/projects/project1/PklProject.deps.json @@ -20,6 +20,11 @@ "uri": "projectpackage://localhost:0/project2@1.0.0", "path": "../project2/" }, + "package://localhost:12110/project6@1": { + "type": "local", + "uri": "projectpackage://localhost:12110/project6@1.0.0", + "path": "../project6/" + }, "package://localhost:0/badImportsWithinPackage@1": { "type": "remote", "uri": "projectpackage://localhost:0/badImportsWithinPackage@1.0.0", diff --git a/pkl-core/src/test/files/LanguageSnippetTests/input/projects/project1/globbing.pkl b/pkl-core/src/test/files/LanguageSnippetTests/input/projects/project1/globbing.pkl index 570b79ef..2abf6734 100644 --- a/pkl-core/src/test/files/LanguageSnippetTests/input/projects/project1/globbing.pkl +++ b/pkl-core/src/test/files/LanguageSnippetTests/input/projects/project1/globbing.pkl @@ -34,4 +34,8 @@ examples { ["glob-read absolute package uri"] { read*("package://localhost:0/birds@0.5.0#/catalog/*.pkl") } + + ["glob-import behind local project import"] { + import("@project6/children.pkl") + } } diff --git a/pkl-core/src/test/files/LanguageSnippetTests/input/projects/project3/PklProject.deps.json b/pkl-core/src/test/files/LanguageSnippetTests/input/projects/project3/PklProject.deps.json index efbcd85e..55d204bf 100644 --- a/pkl-core/src/test/files/LanguageSnippetTests/input/projects/project3/PklProject.deps.json +++ b/pkl-core/src/test/files/LanguageSnippetTests/input/projects/project3/PklProject.deps.json @@ -20,6 +20,11 @@ "uri": "projectpackage://localhost:0/project2@1.0.0", "path": "../project2/" }, + "package://localhost:12110/project6@1": { + "type": "local", + "uri": "projectpackage://localhost:12110/project6@1.0.0", + "path": "../project6/" + }, "package://localhost:0/badImportsWithinPackage@1": { "type": "remote", "uri": "projectpackage://localhost:0/badImportsWithinPackage@1.0.0", diff --git a/pkl-core/src/test/files/LanguageSnippetTests/input/projects/project6/PklProject b/pkl-core/src/test/files/LanguageSnippetTests/input/projects/project6/PklProject new file mode 100644 index 00000000..a574a7a7 --- /dev/null +++ b/pkl-core/src/test/files/LanguageSnippetTests/input/projects/project6/PklProject @@ -0,0 +1,8 @@ +amends "pkl:Project" + +package { + name = "project6" + baseUri = "package://localhost:12110/project6" + version = "1.0.0" + packageZipUrl = "https://localhost:12110/project6/project6-\(version).zip" +} diff --git a/pkl-core/src/test/files/LanguageSnippetTests/input/projects/project6/PklProject.deps.json b/pkl-core/src/test/files/LanguageSnippetTests/input/projects/project6/PklProject.deps.json new file mode 100644 index 00000000..836079aa --- /dev/null +++ b/pkl-core/src/test/files/LanguageSnippetTests/input/projects/project6/PklProject.deps.json @@ -0,0 +1,4 @@ +{ + "schemaVersion": 1, + "resolvedDependencies": {} +} \ No newline at end of file diff --git a/pkl-core/src/test/files/LanguageSnippetTests/input/projects/project6/children.pkl b/pkl-core/src/test/files/LanguageSnippetTests/input/projects/project6/children.pkl new file mode 100644 index 00000000..2e85df8c --- /dev/null +++ b/pkl-core/src/test/files/LanguageSnippetTests/input/projects/project6/children.pkl @@ -0,0 +1 @@ +children = import*("children/*.pkl") diff --git a/pkl-core/src/test/files/LanguageSnippetTests/input/projects/project6/children/a.pkl b/pkl-core/src/test/files/LanguageSnippetTests/input/projects/project6/children/a.pkl new file mode 100644 index 00000000..ca5afff6 --- /dev/null +++ b/pkl-core/src/test/files/LanguageSnippetTests/input/projects/project6/children/a.pkl @@ -0,0 +1 @@ +name = "a" diff --git a/pkl-core/src/test/files/LanguageSnippetTests/input/projects/project6/children/b.pkl b/pkl-core/src/test/files/LanguageSnippetTests/input/projects/project6/children/b.pkl new file mode 100644 index 00000000..b109b4ad --- /dev/null +++ b/pkl-core/src/test/files/LanguageSnippetTests/input/projects/project6/children/b.pkl @@ -0,0 +1 @@ +name = "b" diff --git a/pkl-core/src/test/files/LanguageSnippetTests/input/projects/project6/children/c.pkl b/pkl-core/src/test/files/LanguageSnippetTests/input/projects/project6/children/c.pkl new file mode 100644 index 00000000..021964c9 --- /dev/null +++ b/pkl-core/src/test/files/LanguageSnippetTests/input/projects/project6/children/c.pkl @@ -0,0 +1 @@ +name = "c" diff --git a/pkl-core/src/test/files/LanguageSnippetTests/output/projects/badProjectDeps4/bug.err b/pkl-core/src/test/files/LanguageSnippetTests/output/projects/badProjectDeps4/bug.err index c39c1e87..fafb5600 100644 --- a/pkl-core/src/test/files/LanguageSnippetTests/output/projects/badProjectDeps4/bug.err +++ b/pkl-core/src/test/files/LanguageSnippetTests/output/projects/badProjectDeps4/bug.err @@ -1,13 +1,13 @@ –– Pkl Error –– -Expected value of type `*RemoteDependency|LocalDependency`, but got a different `pkl.Project`. +Expected value of type `*RemoteDependency|Project(isValidLoadDependency)`, but got a different `pkl.Project`. Value: new ModuleClass { package = null; tests {}; dependencies {}; evaluatorSetting... -xxx | dependencies: Mapping - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +xxx | dependencies: Mapping + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ at pkl.Project#dependencies (pkl:Project) -* Value is not of type `LocalDependency` because: - Type constraint `this.package != null` violated. +* Value is not of type `Project(isValidLoadDependency)` because: + Type constraint `isValidLoadDependency` violated. Value: new ModuleClass { package = null; tests {}; dependencies {}; evaluatorSetti... x | dependencies { diff --git a/pkl-core/src/test/files/LanguageSnippetTests/output/projects/missingProjectDeps/bug.err b/pkl-core/src/test/files/LanguageSnippetTests/output/projects/missingProjectDeps/bug.err index 679ecc06..7a775f21 100644 --- a/pkl-core/src/test/files/LanguageSnippetTests/output/projects/missingProjectDeps/bug.err +++ b/pkl-core/src/test/files/LanguageSnippetTests/output/projects/missingProjectDeps/bug.err @@ -1,8 +1,9 @@ –– Pkl Error –– -Cannot resolve dependency because file `PklProject.deps.json` is missing in project directory `file:///$snippetsDir/input/projects/missingProjectDeps/`. +Encountered an error when attempting to load `PklProject.deps.json` at `file:///$snippetsDir/input/projects/missingProjectDeps/PklProject.deps.json`. +NoSuchFileException: /$snippetsDir/input/projects/missingProjectDeps/PklProject.deps.json x | import "@birds/Bird.pkl" ^^^^^^^^^^^^^^^^^ at bug (file:///$snippetsDir/input/projects/missingProjectDeps/bug.pkl) -Run `pkl project resolve` to create a new set of dependencies. +Try running `pkl project resolve` within the project directory to create a new set of dependencies. diff --git a/pkl-core/src/test/files/LanguageSnippetTests/output/projects/project1/globbing.pcf b/pkl-core/src/test/files/LanguageSnippetTests/output/projects/project1/globbing.pcf index 7fd20dca..3f000af6 100644 --- a/pkl-core/src/test/files/LanguageSnippetTests/output/projects/project1/globbing.pcf +++ b/pkl-core/src/test/files/LanguageSnippetTests/output/projects/project1/globbing.pcf @@ -262,4 +262,19 @@ examples { } } } + ["glob-import behind local project import"] { + new { + children { + ["children/a.pkl"] { + name = "a" + } + ["children/b.pkl"] { + name = "b" + } + ["children/c.pkl"] { + name = "c" + } + } + } + } } diff --git a/pkl-core/src/test/files/LanguageSnippetTests/output/projects/project6/children.pcf b/pkl-core/src/test/files/LanguageSnippetTests/output/projects/project6/children.pcf new file mode 100644 index 00000000..2455f8db --- /dev/null +++ b/pkl-core/src/test/files/LanguageSnippetTests/output/projects/project6/children.pcf @@ -0,0 +1,11 @@ +children { + ["children/a.pkl"] { + name = "a" + } + ["children/b.pkl"] { + name = "b" + } + ["children/c.pkl"] { + name = "c" + } +} diff --git a/pkl-core/src/test/files/LanguageSnippetTests/output/projects/project6/children/a.pcf b/pkl-core/src/test/files/LanguageSnippetTests/output/projects/project6/children/a.pcf new file mode 100644 index 00000000..ca5afff6 --- /dev/null +++ b/pkl-core/src/test/files/LanguageSnippetTests/output/projects/project6/children/a.pcf @@ -0,0 +1 @@ +name = "a" diff --git a/pkl-core/src/test/files/LanguageSnippetTests/output/projects/project6/children/b.pcf b/pkl-core/src/test/files/LanguageSnippetTests/output/projects/project6/children/b.pcf new file mode 100644 index 00000000..b109b4ad --- /dev/null +++ b/pkl-core/src/test/files/LanguageSnippetTests/output/projects/project6/children/b.pcf @@ -0,0 +1 @@ +name = "b" diff --git a/pkl-core/src/test/files/LanguageSnippetTests/output/projects/project6/children/c.pcf b/pkl-core/src/test/files/LanguageSnippetTests/output/projects/project6/children/c.pcf new file mode 100644 index 00000000..021964c9 --- /dev/null +++ b/pkl-core/src/test/files/LanguageSnippetTests/output/projects/project6/children/c.pcf @@ -0,0 +1 @@ +name = "c" diff --git a/pkl-core/src/test/kotlin/org/pkl/core/EvaluatorTest.kt b/pkl-core/src/test/kotlin/org/pkl/core/EvaluatorTest.kt index f0de9784..ff091ef9 100644 --- a/pkl-core/src/test/kotlin/org/pkl/core/EvaluatorTest.kt +++ b/pkl-core/src/test/kotlin/org/pkl/core/EvaluatorTest.kt @@ -6,6 +6,7 @@ import java.net.URI import java.nio.file.Files import java.nio.file.Path import org.assertj.core.api.Assertions.assertThat +import org.assertj.core.api.Assertions.assertThatCode import org.junit.jupiter.api.Test import org.junit.jupiter.api.assertThrows import org.junit.jupiter.api.io.TempDir @@ -15,7 +16,16 @@ import org.pkl.commons.writeString import org.pkl.core.ModuleSource.* import org.pkl.core.util.IoUtils import org.junit.jupiter.api.AfterAll +import org.pkl.commons.test.PackageServer +import org.pkl.core.module.ModuleKey +import org.pkl.core.module.ModuleKeyFactories +import org.pkl.core.module.ModuleKeyFactory +import org.pkl.core.module.ResolvedModuleKey +import org.pkl.core.project.Project +import java.nio.charset.StandardCharsets import java.nio.file.FileSystems +import java.util.* +import java.util.regex.Pattern import kotlin.io.path.writeText class EvaluatorTest { @@ -24,6 +34,28 @@ class EvaluatorTest { private const val sourceText = "name = \"pigeon\"; age = 10 + 20" + private object CustomModuleKeyFactory : ModuleKeyFactory { + override fun create(uri: URI): Optional { + return if (uri.scheme == "custom") Optional.of(CustomModuleKey(uri)) + else Optional.empty() + } + } + + private class CustomModuleKey(private val uri: URI) : ModuleKey, ResolvedModuleKey { + override fun hasHierarchicalUris(): Boolean = true + + override fun isGlobbable(): Boolean = false + + override fun getOriginal(): ModuleKey = this + + override fun getUri(): URI = uri + + override fun loadSource(): String = javaClass.classLoader.getResourceAsStream(uri.path.drop(1))!!.use { it.readAllBytes().toString( + StandardCharsets.UTF_8) } + + override fun resolve(securityManager: SecurityManager): ResolvedModuleKey = this + } + @AfterAll @JvmStatic fun afterAll() { @@ -291,6 +323,132 @@ class EvaluatorTest { assertThat(output["bar/../bark.yml"]?.text).isEqualTo("bark: bark bark") } + @Test + fun `project set from modulepath`(@TempDir cacheDir: Path) { + PackageServer.populateCacheDir(cacheDir) + val evaluatorBuilder = EvaluatorBuilder.preconfigured().setModuleCacheDir(cacheDir) + val project = Project.load(modulePath("/org/pkl/core/project/project5/PklProject")) + val result = evaluatorBuilder.setProjectDependencies(project.dependencies).build().use { evaluator -> + evaluator.evaluateOutputText(modulePath("/org/pkl/core/project/project5/main.pkl")) + } + assertThat(result).isEqualTo(""" + prop1 { + name = "Apple" + } + prop2 { + res = 1 + } + + """.trimIndent()) + } + + @Test + fun `project set from custom ModuleKeyFactory`(@TempDir cacheDir: Path) { + PackageServer.populateCacheDir(cacheDir) + val evaluatorBuilder = with(EvaluatorBuilder.preconfigured()) { + setAllowedModules(SecurityManagers.defaultAllowedModules + Pattern.compile("custom:")) + setAllowedResources(SecurityManagers.defaultAllowedResources + Pattern.compile("custom:")) + setModuleCacheDir(cacheDir) + setModuleKeyFactories( + listOf( + CustomModuleKeyFactory, + ModuleKeyFactories.standardLibrary, + ModuleKeyFactories.pkg, + ModuleKeyFactories.projectpackage, + ModuleKeyFactories.file + ) + ) + } + val project = evaluatorBuilder.build().use { Project.load(it, uri("custom:/org/pkl/core/project/project5/PklProject")) } + + val evaluator = evaluatorBuilder.setProjectDependencies(project.dependencies).build() + val output = evaluator.use { it.evaluateOutputText(uri("custom:/org/pkl/core/project/project5/main.pkl")) } + assertThat(output) + .isEqualTo( + """ + prop1 { + name = "Apple" + } + prop2 { + res = 1 + } + + """ + .trimIndent() + ) + } + + @Test + fun `project base path set to non-hierarchical scheme`() { + class FooBarModuleKey(val moduleUri: URI) : ModuleKey, ResolvedModuleKey { + override fun hasHierarchicalUris(): Boolean = false + override fun isGlobbable(): Boolean = false + override fun getOriginal(): ModuleKey = this + override fun getUri(): URI = moduleUri + override fun loadSource(): String = + if (uri.schemeSpecificPart.endsWith("PklProject")) { + """ + amends "pkl:Project" + """.trimIndent() + } else """ + birds = import("@birds/catalog/Ostritch.pkl") + """.trimIndent() + override fun resolve(securityManager: SecurityManager): ResolvedModuleKey { + return this + } + } + + val fooBayModuleKeyFactory = ModuleKeyFactory { uri -> + if (uri.scheme == "foobar") Optional.of(FooBarModuleKey(uri)) + else Optional.empty() + } + + val evaluatorBuilder = with(EvaluatorBuilder.preconfigured()) { + setAllowedModules(SecurityManagers.defaultAllowedModules + Pattern.compile("foobar:")) + setAllowedResources(SecurityManagers.defaultAllowedResources + Pattern.compile("foobar:")) + setModuleKeyFactories( + listOf( + fooBayModuleKeyFactory, + ModuleKeyFactories.standardLibrary, + ModuleKeyFactories.pkg, + ModuleKeyFactories.projectpackage, + ModuleKeyFactories.file + ) + ) + } + + val project = evaluatorBuilder.build().use { Project.load(it, uri("foobar:foo/PklProject")) } + val evaluator = evaluatorBuilder.setProjectDependencies(project.dependencies).build() + assertThatCode { evaluator.use { it.evaluateOutputText(uri("foobar:baz")) } } + .hasMessageContaining("Cannot import dependency because project URI `foobar:foo/PklProject` does not have a hierarchical path.") + } + + @Test + fun `cannot glob import in local dependency from modulepath`(@TempDir cacheDir: Path) { + PackageServer.populateCacheDir(cacheDir) + val evaluatorBuilder = EvaluatorBuilder.preconfigured().setModuleCacheDir(cacheDir) + val project = Project.load(modulePath("/org/pkl/core/project/project6/PklProject")) + evaluatorBuilder.setProjectDependencies(project.dependencies).build().use { evaluator -> + assertThatCode { + evaluator.evaluateOutputText(modulePath("/org/pkl/core/project/project6/globWithinDependency.pkl")) + }.hasMessageContaining(""" + Cannot resolve import in local dependency because scheme `modulepath` is not globbable. + + 1 | res = import*("*.pkl") + ^^^^^^^^^^^^^^^^ + """.trimIndent()) + assertThatCode { + evaluator.evaluateOutputText(modulePath("/org/pkl/core/project/project6/globIntoDependency.pkl")) + }.hasMessageContaining(""" + –– Pkl Error –– + Cannot resolve import in local dependency because scheme `modulepath` is not globbable. + + 1 | import* "@project7/*.pkl" as proj7Files + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + """.trimIndent()) + } + } + private fun checkModule(module: PModule) { assertThat(module.properties.size).isEqualTo(2) assertThat(module.getProperty("name")).isEqualTo("pigeon") diff --git a/pkl-core/src/test/kotlin/org/pkl/core/LanguageSnippetTestsEngine.kt b/pkl-core/src/test/kotlin/org/pkl/core/LanguageSnippetTestsEngine.kt index ef54f052..7addac13 100644 --- a/pkl-core/src/test/kotlin/org/pkl/core/LanguageSnippetTestsEngine.kt +++ b/pkl-core/src/test/kotlin/org/pkl/core/LanguageSnippetTestsEngine.kt @@ -38,10 +38,15 @@ abstract class AbstractLanguageSnippetTestsEngine : InputOutputTestEngine() { internal val selection: String = "" protected val packageServer: PackageServer = PackageServer() - + override val includedTests: List = listOf(Regex(".*$selection\\.pkl")) - override val excludedTests: List = listOf(Regex(".*/native/.*")) + override val excludedTests: List = buildList { + add(Regex(".*/native/.*")) + if (IoUtils.isWindows()) { + addAll(windowsExcludedTests) + } + } override val inputDir: Path = snippetsDir.resolve("input") @@ -68,7 +73,12 @@ abstract class AbstractLanguageSnippetTestsEngine : InputOutputTestEngine() { packageServer.close() } - protected fun String.stripFilePaths() = replace(snippetsDir.toUri().toString(), "file:///\$snippetsDir/") + private val replacement by lazy { + if (snippetsDir.root.toString() != "/") "\$snippetsDir" else "/\$snippetsDir" + } + + protected fun String.stripFilePaths(): String = + replace(IoUtils.toNormalizedPathString(snippetsDir), replacement) protected fun String.stripLineNumbers() = replace(lineNumberRegex) { result -> // replace line number with equivalent number of 'x' characters to keep formatting intact @@ -80,7 +90,7 @@ abstract class AbstractLanguageSnippetTestsEngine : InputOutputTestEngine() { // can't think of a better solution right now protected fun String.stripVersionCheckErrorMessage() = replace("Pkl version is ${Release.current().version()}", "Pkl version is xxx") - + protected fun String.stripStdlibLocationSha(): String = replace("https://github.com/apple/pkl/blob/${Release.current().commitId()}/stdlib/", "https://github.com/apple/pkl/blob/\$commitId/stdlib/") @@ -261,7 +271,12 @@ class AlpineLanguageSnippetTestsEngine : AbstractNativeLanguageSnippetTestsEngin override val testClass: KClass<*> = AlpineLanguageSnippetTests::class } +// error message contains different file path on Windows +private val windowsExcludedTests get() = listOf(Regex(".*missingProjectDeps/bug\\.pkl")) + class WindowsLanguageSnippetTestsEngine : AbstractNativeLanguageSnippetTestsEngine() { override val pklExecutablePath: Path = PklExecutablePaths.windowsAmd64 override val testClass: KClass<*> = WindowsLanguageSnippetTests::class + override val excludedTests: List + get() = super.excludedTests + windowsExcludedTests } diff --git a/pkl-core/src/test/kotlin/org/pkl/core/project/ProjectTest.kt b/pkl-core/src/test/kotlin/org/pkl/core/project/ProjectTest.kt index 4fc5cf7d..da8c8521 100644 --- a/pkl-core/src/test/kotlin/org/pkl/core/project/ProjectTest.kt +++ b/pkl-core/src/test/kotlin/org/pkl/core/project/ProjectTest.kt @@ -1,16 +1,16 @@ package org.pkl.core.project -import org.pkl.commons.test.PackageServer -import org.pkl.commons.writeString -import org.pkl.core.* -import org.pkl.core.packages.PackageUri -import org.pkl.core.project.Project.EvaluatorSettings import org.assertj.core.api.Assertions.assertThat import org.assertj.core.api.Assertions.assertThatCode import org.junit.jupiter.api.Test import org.junit.jupiter.api.io.TempDir import org.pkl.commons.test.FileTestUtils +import org.pkl.commons.test.PackageServer +import org.pkl.commons.writeString +import org.pkl.core.* import org.pkl.core.http.HttpClient +import org.pkl.core.packages.PackageUri +import org.pkl.core.project.Project.EvaluatorSettings import java.net.URI import java.nio.file.Path import java.util.regex.Pattern diff --git a/pkl-core/src/test/resources/org/pkl/core/project/project5/PklProject b/pkl-core/src/test/resources/org/pkl/core/project/project5/PklProject new file mode 100644 index 00000000..771608ad --- /dev/null +++ b/pkl-core/src/test/resources/org/pkl/core/project/project5/PklProject @@ -0,0 +1,8 @@ +amends "pkl:Project" + +dependencies { + ["fruit"] { + uri = "package://localhost:0/fruit@1.0.5" + } + ["project4"] = import("../project4/PklProject") +} diff --git a/pkl-core/src/test/resources/org/pkl/core/project/project5/PklProject.deps.json b/pkl-core/src/test/resources/org/pkl/core/project/project5/PklProject.deps.json new file mode 100644 index 00000000..42f11d3d --- /dev/null +++ b/pkl-core/src/test/resources/org/pkl/core/project/project5/PklProject.deps.json @@ -0,0 +1,17 @@ +{ + "schemaVersion": 1, + "resolvedDependencies": { + "package://localhost:0/fruit@1": { + "type": "remote", + "uri": "projectpackage://localhost:0/fruit@1.0.5", + "checksums": { + "sha256": "$skipChecksumVerification" + } + }, + "package://localhost:0/project4@1": { + "type": "local", + "uri": "projectpackage://localhost:0/project4@1.0.0", + "path": "../project4" + } + } +} diff --git a/pkl-core/src/test/resources/org/pkl/core/project/project5/main.pkl b/pkl-core/src/test/resources/org/pkl/core/project/project5/main.pkl new file mode 100644 index 00000000..476e3cb4 --- /dev/null +++ b/pkl-core/src/test/resources/org/pkl/core/project/project5/main.pkl @@ -0,0 +1,5 @@ +import "@fruit/catalog/apple.pkl" +import "@project4/module1.pkl" + +prop1 = apple +prop2 = module1 diff --git a/pkl-core/src/test/resources/org/pkl/core/project/project6/PklProject b/pkl-core/src/test/resources/org/pkl/core/project/project6/PklProject new file mode 100644 index 00000000..d2eec0c1 --- /dev/null +++ b/pkl-core/src/test/resources/org/pkl/core/project/project6/PklProject @@ -0,0 +1,5 @@ +amends "pkl:Project" + +dependencies { + ["project7"] = import("../project7/PklProject") +} diff --git a/pkl-core/src/test/resources/org/pkl/core/project/project6/PklProject.deps.json b/pkl-core/src/test/resources/org/pkl/core/project/project6/PklProject.deps.json new file mode 100644 index 00000000..6fc3cc8c --- /dev/null +++ b/pkl-core/src/test/resources/org/pkl/core/project/project6/PklProject.deps.json @@ -0,0 +1,10 @@ +{ + "schemaVersion": 1, + "resolvedDependencies": { + "package://localhost:0/project7@1": { + "type": "local", + "uri": "projectpackage://localhost:0/project7@1.0.0", + "path": "../project7" + } + } +} diff --git a/pkl-core/src/test/resources/org/pkl/core/project/project6/globIntoDependency.pkl b/pkl-core/src/test/resources/org/pkl/core/project/project6/globIntoDependency.pkl new file mode 100644 index 00000000..a3cfa7dd --- /dev/null +++ b/pkl-core/src/test/resources/org/pkl/core/project/project6/globIntoDependency.pkl @@ -0,0 +1,3 @@ +import* "@project7/*.pkl" as proj7Files + +res = proj7Files diff --git a/pkl-core/src/test/resources/org/pkl/core/project/project6/globWithinDependency.pkl b/pkl-core/src/test/resources/org/pkl/core/project/project6/globWithinDependency.pkl new file mode 100644 index 00000000..df144528 --- /dev/null +++ b/pkl-core/src/test/resources/org/pkl/core/project/project6/globWithinDependency.pkl @@ -0,0 +1,3 @@ +import "@project7/main.pkl" + +res = main.res diff --git a/pkl-core/src/test/resources/org/pkl/core/project/project7/PklProject b/pkl-core/src/test/resources/org/pkl/core/project/project7/PklProject new file mode 100644 index 00000000..76e474a0 --- /dev/null +++ b/pkl-core/src/test/resources/org/pkl/core/project/project7/PklProject @@ -0,0 +1,8 @@ +amends "pkl:Project" + +package { + name = "project7" + version = "1.0.0" + packageZipUrl = "https://bogus.value" + baseUri = "package://localhost:0/project7" +} diff --git a/pkl-core/src/test/resources/org/pkl/core/project/project7/main.pkl b/pkl-core/src/test/resources/org/pkl/core/project/project7/main.pkl new file mode 100644 index 00000000..6c85a216 --- /dev/null +++ b/pkl-core/src/test/resources/org/pkl/core/project/project7/main.pkl @@ -0,0 +1 @@ +res = import*("*.pkl") diff --git a/pkl-core/src/test/resources/org/pkl/core/project/project7/moduleA.pkl b/pkl-core/src/test/resources/org/pkl/core/project/project7/moduleA.pkl new file mode 100644 index 00000000..e69de29b diff --git a/pkl-core/src/test/resources/org/pkl/core/project/project7/moduleB.pkl b/pkl-core/src/test/resources/org/pkl/core/project/project7/moduleB.pkl new file mode 100644 index 00000000..e69de29b diff --git a/pkl-executor/src/test/kotlin/org/pkl/executor/EmbeddedExecutorTest.kt b/pkl-executor/src/test/kotlin/org/pkl/executor/EmbeddedExecutorTest.kt index 7a8d44c2..54338fa2 100644 --- a/pkl-executor/src/test/kotlin/org/pkl/executor/EmbeddedExecutorTest.kt +++ b/pkl-executor/src/test/kotlin/org/pkl/executor/EmbeddedExecutorTest.kt @@ -478,7 +478,7 @@ class EmbeddedExecutorTest { ) val result = executor.evaluatePath(pklFile) { allowedModules("file:", "package:", "projectpackage:", "https:") - allowedResources("prop:", "package:", "projectpackage:", "https:") + allowedResources("file:", "prop:", "package:", "projectpackage:", "https:") moduleCacheDir(cacheDir) projectDir(projectDir) } diff --git a/stdlib/Project.pkl b/stdlib/Project.pkl index d7bb2684..1c5affa8 100644 --- a/stdlib/Project.pkl +++ b/stdlib/Project.pkl @@ -91,18 +91,17 @@ package: Package? /// ``` tests: Listing(isDistinct) -/// Tells if the project is a file-based module named `PklProject`. -local isLocalPklProject = (it: Project) -> - it.projectFileUri.startsWith("file:") && it.projectFileUri.endsWith("/PklProject") +/// Tells if the project is a local module named `PklProject`, is not self, and has a [package] section +local isValidLoadDependency = (it: Project) -> + isUriLocal(projectFileUri, it.projectFileUri) + && it.projectFileUri.endsWith("/PklProject") + && it != module + && it.package != null -/// A local dependency is another [Project] that is local to the file system. -/// -/// To declare, use `import("path/to/PklProject")` -typealias LocalDependency = Project( - isLocalPklProject, - this != module, - this.package != null -) +const local function isUriLocal(uri1: Uri, uri2: Uri): Boolean = + // This is an imperfect check; should also check that the URIs have the same authority. + // We should improve this if/when there is a URI library in the stdlib. + uri1.substring(0, uri1.indexOf(":")) == uri2.substring(0, uri2.indexOf(":")) /// The dependencies of this project. /// @@ -176,19 +175,31 @@ typealias LocalDependency = Project( /// 1. Gather all dependencies, both direct and transitive. /// 2. For each package's major version, determine the highest declared minor version. /// 3. Write each resolved dependency to sibling file `PklProject.deps.json`. -dependencies: Mapping +dependencies: Mapping + +local isFileBasedProject = projectFileUri.startsWith("file:") /// If set, controls the base evaluator settings when running the evaluator. /// /// These settings influence the behavior of the evaluator when running the `pkl eval`, `pkl test`, /// and `pkl repl` CLI commands. -/// Note that command line flags passed to the CLI will override any settings defined here. +/// Command line flags passed to the CLI will override any settings defined here. /// /// Other integrations can possibly ignore these evaluator settings. /// /// Evaluator settings do not get published as part of a package. /// It is not possible for a package dependency to influence the evaluator settings of a project. -evaluatorSettings: EvaluatorSettings +/// +/// The following values can only be set if this is a file-based project. +/// +/// - [modulePath][EvaluatorSettings.modulePath] +/// - [rootDir][EvaluatorSettings.rootDir] +/// - [moduleCacheDir][EvaluatorSettings.moduleCacheDir] +evaluatorSettings: EvaluatorSettings( + (modulePath != null).implies(isFileBasedProject), + (rootDir != null).implies(isFileBasedProject), + (moduleCacheDir != null).implies(isFileBasedProject) +) /// The URI of the PklProject file. ///