Support scheme-agnostic projects (#486)

This adds changes to support loading project dependencies in non-file based projects.

The design for this feature can be found in SPICE-0005: https://github.com/apple/pkl-evolution/pull/6

Changes:
* Consider all imports prefixed with `@` as dependency notation.
* Bugfix: fix resolution of glob expressions in a local dependency.
* Adjust pkl.Project:
  - Allow local dependencies from a scheme-local paths.
  - Disallow certain evaluator settings if not loaded as a file-based module.
* Breaking API change: `ProjectDependenciesManager` constructor now requires `ModuleResolver` and `SecurityManager`.
This commit is contained in:
Daniel Chao
2024-06-04 16:52:20 -07:00
committed by GitHub
parent c0a7080287
commit d5ba8fa736
49 changed files with 764 additions and 235 deletions

View File

@@ -105,7 +105,8 @@ public class EvaluatorImpl implements Evaluator {
packageResolver,
projectDependencies == null
? null
: new ProjectDependenciesManager(projectDependencies)));
: new ProjectDependenciesManager(
projectDependencies, moduleResolver, securityManager)));
});
this.timeout = timeout;
// NOTE: would probably make sense to share executor between evaluators

View File

@@ -1825,6 +1825,8 @@ public final class AstBuilder extends AbstractAstBuilder<Object> {
} catch (VmException e) {
throw exceptionBuilder()
.evalError(e.getMessage(), e.getMessageArguments())
.withCause(e.getCause())
.withHint(e.getHint())
.withSourceSection(createSourceSection(importUriCtx))
.build();
}

View File

@@ -22,6 +22,7 @@ import com.oracle.truffle.api.frame.FrameDescriptor;
import com.oracle.truffle.api.nodes.NodeInfo;
import com.oracle.truffle.api.source.SourceSection;
import java.io.IOException;
import java.net.URISyntaxException;
import org.graalvm.collections.EconomicMap;
import org.pkl.core.SecurityManagerException;
import org.pkl.core.ast.member.SharedMemberNode;
@@ -33,6 +34,7 @@ import org.pkl.core.runtime.VmMapping;
import org.pkl.core.runtime.VmObjectBuilder;
import org.pkl.core.util.GlobResolver;
import org.pkl.core.util.GlobResolver.InvalidGlobPatternException;
import org.pkl.core.util.IoUtils;
import org.pkl.core.util.LateInit;
@NodeInfo(shortName = "read*")
@@ -73,7 +75,7 @@ public abstract class ReadGlobNode extends AbstractReadNode {
var globUri = parseUri(globPattern);
var context = VmContext.get(this);
try {
var resolvedUri = currentModule.resolveUri(globUri);
var resolvedUri = IoUtils.resolve(context.getSecurityManager(), currentModule, globUri);
var reader = context.getResourceManager().getReader(resolvedUri, this);
if (!reader.isGlobbable()) {
throw exceptionBuilder().evalError("cannotGlobUri", globUri, globUri.getScheme()).build();
@@ -94,7 +96,7 @@ public abstract class ReadGlobNode extends AbstractReadNode {
return cachedResult;
} catch (IOException e) {
throw exceptionBuilder().evalError("ioErrorResolvingGlob", globPattern).withCause(e).build();
} catch (SecurityManagerException | HttpClientInitException e) {
} catch (SecurityManagerException | HttpClientInitException | URISyntaxException e) {
throw exceptionBuilder().withCause(e).build();
} catch (InvalidGlobPatternException e) {
throw exceptionBuilder()

View File

@@ -41,7 +41,6 @@ import org.pkl.core.util.ErrorMessages;
import org.pkl.core.util.HttpUtils;
import org.pkl.core.util.IoUtils;
import org.pkl.core.util.Nullable;
import org.pkl.core.util.Pair;
/** Utilities for creating and using {@link ModuleKey}s. */
public final class ModuleKeys {
@@ -290,14 +289,18 @@ public final class ModuleKeys {
}
}
private static class File extends DependencyAwareModuleKey {
private static class File implements ModuleKey {
final URI uri;
File(URI uri) {
super(uri);
this.uri = uri;
}
@Override
public URI getUri() {
return uri;
}
@Override
public boolean hasElement(SecurityManager securityManager, URI uri)
throws SecurityManagerException {
@@ -329,17 +332,18 @@ public final class ModuleKeys {
}
@Override
protected Map<String, ? extends Dependency> getDependencies() {
var projectDepsManager = VmContext.get(null).getProjectDependenciesManager();
if (projectDepsManager == null || !projectDepsManager.hasPath(Path.of(uri))) {
throw new PackageLoadError("cannotResolveDependencyNoProject");
}
return projectDepsManager.getDependencies();
public boolean isGlobbable() {
return true;
}
@Override
protected PackageLoadError cannotFindDependency(String name) {
return new PackageLoadError("cannotFindDependencyInProject", name);
public boolean isLocal() {
return true;
}
@Override
public boolean hasHierarchicalUris() {
return true;
}
}
@@ -547,40 +551,29 @@ public final class ModuleKeys {
}
}
/** Base implementation; knows how to resolve dependencies prefixed with <code>@</code>. */
private abstract static class DependencyAwareModuleKey implements ModuleKey {
private abstract static class AbstractPackage implements ModuleKey {
protected final URI uri;
protected final PackageAssetUri packageAssetUri;
DependencyAwareModuleKey(URI uri) {
this.uri = uri;
}
@Override
public URI getUri() {
return uri;
}
protected Pair<String, String> parseDependencyNotation(String importPath) {
var idx = importPath.indexOf('/');
if (idx == -1) {
// treat named dependency without a subpath as the root path.
// i.e. resolve to `@foo` to `package://example.com/foo@1.0.0#/`
return Pair.of(importPath.substring(1), "/");
}
return Pair.of(importPath.substring(1, idx), importPath.substring(idx));
AbstractPackage(PackageAssetUri packageAssetUri) {
this.packageAssetUri = packageAssetUri;
}
protected abstract Map<String, ? extends Dependency> getDependencies()
throws IOException, SecurityManagerException;
@Override
public boolean isLocal() {
public boolean hasHierarchicalUris() {
return true;
}
@Override
public boolean hasHierarchicalUris() {
public boolean hasFragmentPaths() {
return true;
}
@Override
public boolean isLocal() {
return true;
}
@@ -589,37 +582,36 @@ public final class ModuleKeys {
return true;
}
private URI resolveDependencyNotation(String notation)
throws IOException, SecurityManagerException {
var parsed = parseDependencyNotation(notation);
var name = parsed.getFirst();
var path = parsed.getSecond();
var dependency = getDependencies().get(name);
if (dependency == null) {
throw cannotFindDependency(name);
}
return dependency.getPackageUri().toPackageAssetUri(path).getUri();
@Override
public URI getUri() {
return packageAssetUri.getUri();
}
@Override
public URI resolveUri(URI baseUri, URI importUri) throws IOException, SecurityManagerException {
if (importUri.isAbsolute() || !importUri.getPath().startsWith("@")) {
var ssp = importUri.getSchemeSpecificPart();
if (importUri.isAbsolute() || !ssp.startsWith("@")) {
return ModuleKey.super.resolveUri(baseUri, importUri);
}
return resolveDependencyNotation(importUri.getPath());
var parsed = IoUtils.parseDependencyNotation(ssp);
var name = parsed.getFirst();
var path = parsed.getSecond();
var dependency = getDependencies().get(name);
if (dependency == null) {
throw new PackageLoadError(
"cannotFindDependencyInPackage",
name,
packageAssetUri.getPackageUri().getDisplayName());
}
return dependency.getPackageUri().toPackageAssetUri(path).getUri();
}
protected abstract PackageLoadError cannotFindDependency(String name);
}
/** Represents a module imported via the {@code package} scheme. */
private static class Package extends DependencyAwareModuleKey {
private final PackageAssetUri packageAssetUri;
private static class Package extends AbstractPackage {
Package(PackageAssetUri packageAssetUri) {
super(packageAssetUri.getUri());
this.packageAssetUri = packageAssetUri;
super(packageAssetUri);
}
private PackageResolver getPackageResolver() {
@@ -631,6 +623,7 @@ public final class ModuleKeys {
@Override
public ResolvedModuleKey resolve(SecurityManager securityManager)
throws IOException, SecurityManagerException {
var uri = packageAssetUri.getUri();
securityManager.checkResolveModule(uri);
var bytes =
getPackageResolver()
@@ -654,11 +647,6 @@ public final class ModuleKeys {
return getPackageResolver().hasElement(assetUri, assetUri.getPackageUri().getChecksums());
}
@Override
public boolean hasFragmentPaths() {
return true;
}
@Override
protected Map<String, ? extends Dependency> getDependencies()
throws IOException, SecurityManagerException {
@@ -667,12 +655,6 @@ public final class ModuleKeys {
packageAssetUri.getPackageUri(), packageAssetUri.getPackageUri().getChecksums())
.getDependencies();
}
@Override
protected PackageLoadError cannotFindDependency(String name) {
return new PackageLoadError(
"cannotFindDependencyInPackage", name, packageAssetUri.getPackageUri().getDisplayName());
}
}
/**
@@ -680,15 +662,11 @@ public final class ModuleKeys {
*
* <p>The {@code projectpackage} scheme is what project-local dependencies resolve to when
* imported using dependency notation (for example, {@code import "@foo/bar.pkl"}). This scheme is
* an internal implementation detail, and we do not expect a project to declare this.
* an internal implementation detail, and we do not expect a module to declare this.
*/
private static class ProjectPackage extends DependencyAwareModuleKey {
private final PackageAssetUri packageAssetUri;
public static class ProjectPackage extends AbstractPackage {
ProjectPackage(PackageAssetUri packageAssetUri) {
super(packageAssetUri.getUri());
this.packageAssetUri = packageAssetUri;
super(packageAssetUri);
}
private PackageResolver getPackageResolver() {
@@ -697,37 +675,36 @@ public final class ModuleKeys {
return packageResolver;
}
private ProjectDependenciesManager getProjectDepsResolver() {
private ProjectDependenciesManager getProjectDependenciesManager() {
var projectDepsManager = VmContext.get(null).getProjectDependenciesManager();
assert projectDepsManager != null;
return projectDepsManager;
}
private @Nullable Path getLocalPath(Dependency dependency, PackageAssetUri packageAssetUri) {
private @Nullable URI getLocalUri(Dependency dependency) {
return getLocalUri(dependency, packageAssetUri);
}
private @Nullable URI getLocalUri(Dependency dependency, PackageAssetUri assetUri) {
if (!(dependency instanceof LocalDependency localDependency)) {
return null;
}
return localDependency.resolveAssetPath(
getProjectDepsResolver().getProjectDir(), packageAssetUri);
}
private @Nullable Path getLocalPath(Dependency dependency) {
if (!(dependency instanceof LocalDependency)) {
return null;
}
return getLocalPath(dependency, packageAssetUri);
return localDependency.resolveAssetUri(
getProjectDependenciesManager().getProjectBaseUri(), assetUri);
}
@Override
public ResolvedModuleKey resolve(SecurityManager securityManager)
throws IOException, SecurityManagerException {
securityManager.checkResolveModule(packageAssetUri.getUri());
var uri = packageAssetUri.getUri();
securityManager.checkResolveModule(uri);
var dependency =
getProjectDepsResolver().getResolvedDependency(packageAssetUri.getPackageUri());
var path = getLocalPath(dependency);
if (path != null) {
securityManager.checkResolveModule(path.toUri());
return ResolvedModuleKeys.file(this, path.toUri(), path);
getProjectDependenciesManager().getResolvedDependency(packageAssetUri.getPackageUri());
var local = getLocalUri(dependency);
if (local != null) {
var resolved =
VmContext.get(null).getModuleResolver().resolve(local).resolve(securityManager);
return ResolvedModuleKeys.delegated(resolved, this);
}
var dep = (Dependency.RemoteDependency) dependency;
assert dep.getChecksums() != null;
@@ -741,11 +718,15 @@ public final class ModuleKeys {
securityManager.checkResolveModule(baseUri);
var packageAssetUri = PackageAssetUri.create(baseUri);
var dependency =
getProjectDepsResolver().getResolvedDependency(packageAssetUri.getPackageUri());
var path = getLocalPath(dependency, packageAssetUri);
if (path != null) {
securityManager.checkResolveModule(path.toUri());
return FileResolver.listElements(path);
getProjectDependenciesManager().getResolvedDependency(packageAssetUri.getPackageUri());
var local = getLocalUri(dependency, packageAssetUri);
if (local != null) {
var moduleKey = VmContext.get(null).getModuleResolver().resolve(local);
if (!moduleKey.isGlobbable()) {
throw new PackageLoadError(
"cannotResolveInLocalDependencyNotGlobbable", local.getScheme());
}
return moduleKey.listElements(securityManager, local);
}
var dep = (Dependency.RemoteDependency) dependency;
assert dep.getChecksums() != null;
@@ -758,42 +739,36 @@ public final class ModuleKeys {
securityManager.checkResolveModule(elementUri);
var packageAssetUri = PackageAssetUri.create(elementUri);
var dependency =
getProjectDepsResolver().getResolvedDependency(packageAssetUri.getPackageUri());
var path = getLocalPath(dependency, packageAssetUri);
if (path != null) {
securityManager.checkResolveModule(path.toUri());
return FileResolver.hasElement(path);
getProjectDependenciesManager().getResolvedDependency(packageAssetUri.getPackageUri());
var local = getLocalUri(dependency, packageAssetUri);
if (local != null) {
var moduleKey = VmContext.get(null).getModuleResolver().resolve(local);
if (!moduleKey.isGlobbable() && !moduleKey.isLocal()) {
throw new PackageLoadError(
"cannotResolveInLocalDependencyNotGlobbableNorLocal", local.getScheme());
}
return moduleKey.hasElement(securityManager, local);
}
var dep = (Dependency.RemoteDependency) dependency;
assert dep.getChecksums() != null;
return getPackageResolver().hasElement(packageAssetUri, dep.getChecksums());
}
@Override
public boolean hasFragmentPaths() {
return true;
}
@Override
protected Map<String, ? extends Dependency> getDependencies()
throws IOException, SecurityManagerException {
var packageUri = packageAssetUri.getPackageUri();
var projectResolver = getProjectDepsResolver();
var projectResolver = getProjectDependenciesManager();
if (projectResolver.isLocalPackage(packageUri)) {
return projectResolver.getLocalPackageDependencies(packageUri);
}
var dep =
(Dependency.RemoteDependency) getProjectDepsResolver().getResolvedDependency(packageUri);
(Dependency.RemoteDependency)
getProjectDependenciesManager().getResolvedDependency(packageUri);
assert dep.getChecksums() != null;
var dependencyMetadata =
getPackageResolver().getDependencyMetadata(packageUri, dep.getChecksums());
return projectResolver.getResolvedDependenciesForPackage(packageUri, dependencyMetadata);
}
@Override
protected PackageLoadError cannotFindDependency(String name) {
return new PackageLoadError(
"cannotFindDependencyInPackage", name, packageAssetUri.getPackageUri().getDisplayName());
}
}
}

View File

@@ -17,13 +17,14 @@ package org.pkl.core.module;
import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.HashMap;
import java.util.Map;
import java.util.Objects;
import javax.annotation.concurrent.GuardedBy;
import org.graalvm.collections.EconomicMap;
import org.pkl.core.PklBugException;
import org.pkl.core.SecurityManager;
import org.pkl.core.SecurityManagerException;
import org.pkl.core.packages.Dependency;
import org.pkl.core.packages.DependencyMetadata;
import org.pkl.core.packages.PackageLoadError;
@@ -31,8 +32,10 @@ import org.pkl.core.packages.PackageUri;
import org.pkl.core.project.CanonicalPackageUri;
import org.pkl.core.project.DeclaredDependencies;
import org.pkl.core.project.ProjectDeps;
import org.pkl.core.runtime.ModuleResolver;
import org.pkl.core.runtime.VmExceptionBuilder;
import org.pkl.core.util.EconomicMaps;
import org.pkl.core.util.IoUtils;
import org.pkl.core.util.json.Json.JsonParseException;
public final class ProjectDependenciesManager {
@@ -41,7 +44,9 @@ public final class ProjectDependenciesManager {
public static final String PKL_PROJECT_DEPS_FILENAME = "PklProject.deps.json";
private final DeclaredDependencies declaredDependencies;
private final Path projectDir;
private final URI projectBaseUri;
private final ModuleResolver moduleResolver;
private final SecurityManager securityManager;
@GuardedBy("lock")
private ProjectDeps projectDeps;
@@ -59,13 +64,21 @@ public final class ProjectDependenciesManager {
private final Object lock = new Object();
public ProjectDependenciesManager(DeclaredDependencies declaredDependencies) {
public ProjectDependenciesManager(
DeclaredDependencies declaredDependencies,
ModuleResolver moduleResolver,
SecurityManager securityManager) {
this.declaredDependencies = declaredDependencies;
this.projectDir = Path.of(declaredDependencies.getProjectFileUri()).getParent();
// new URI("scheme://host/a/b/c.txt").resolve(".") == new URI("scheme://host/a/b/")
this.projectBaseUri = IoUtils.resolve(declaredDependencies.getProjectFileUri(), ".");
this.moduleResolver = moduleResolver;
this.securityManager = securityManager;
}
public boolean hasPath(Path path) {
return path.startsWith(projectDir);
public boolean hasUri(URI uri) {
return projectBaseUri.getScheme().equals(uri.getScheme())
&& Objects.equals(projectBaseUri.getAuthority(), uri.getAuthority())
&& uri.getPath().startsWith(projectBaseUri.getPath());
}
private void ensureDependenciesInitialized() {
@@ -194,30 +207,39 @@ public final class ProjectDependenciesManager {
return dep;
}
public Path getProjectDir() {
return projectDir;
public URI getProjectBaseUri() {
return projectBaseUri;
}
public Path getProjectDepsFile() {
return projectDir.resolve(PKL_PROJECT_DEPS_FILENAME);
public URI getProjectDepsFileUri() {
return IoUtils.resolve(projectBaseUri, PKL_PROJECT_DEPS_FILENAME);
}
public URI getProjectFileUri() {
return declaredDependencies.getProjectFileUri();
}
private ProjectDeps getProjectDeps() {
synchronized (lock) {
if (projectDeps == null) {
var depsPath = getProjectDepsFile();
if (!Files.exists(depsPath)) {
throw new VmExceptionBuilder()
.evalError("missingProjectDepsJson", projectDir.toUri())
.build();
}
var depsUri = getProjectDepsFileUri();
var moduleKey = moduleResolver.resolve(depsUri);
try {
projectDeps = ProjectDeps.parse(depsPath);
} catch (IOException | URISyntaxException | JsonParseException e) {
// treat PklProject.deps.json as a module read, rather than introduce a new API.
var depsJson = moduleKey.resolve(securityManager).loadSource();
projectDeps = ProjectDeps.parse(depsJson);
} catch (IOException e) {
throw new VmExceptionBuilder()
.evalError("invalidProjectDepsJson", depsPath.toUri(), e.getMessage())
.evalError("cannotLoadProjectDepsJson", depsUri)
.withCause(e)
.withHint(e.getMessage() != null ? e.getMessage() : ("Encountered error: " + e))
.build();
} catch (JsonParseException e) {
throw new VmExceptionBuilder()
.evalError("invalidProjectDepsJson", depsUri, e.getMessage())
.build();
} catch (SecurityManagerException e) {
throw PklBugException.unreachableCode();
}
}
return projectDeps;

View File

@@ -54,6 +54,14 @@ public final class ResolvedModuleKeys {
return new Virtual(original, uri, sourceText, cached);
}
/**
* Creates a resolved module key that behaves like {@code delegate}, except with {@code original}
* as its original module key.
*/
public static ResolvedModuleKey delegated(ResolvedModuleKey delegate, ModuleKey original) {
return new Delegated(delegate, original);
}
private static class File implements ResolvedModuleKey {
final ModuleKey original;
final URI uri;
@@ -145,4 +153,30 @@ public final class ResolvedModuleKeys {
return sourceText;
}
}
private static class Delegated implements ResolvedModuleKey {
private final ResolvedModuleKey delegate;
private final ModuleKey original;
Delegated(ResolvedModuleKey delegate, ModuleKey original) {
this.delegate = delegate;
this.original = original;
}
@Override
public ModuleKey getOriginal() {
return original;
}
@Override
public URI getUri() {
return delegate.getUri();
}
@Override
public String loadSource() throws IOException {
return delegate.loadSource();
}
}
}

View File

@@ -15,9 +15,11 @@
*/
package org.pkl.core.packages;
import java.net.URI;
import java.nio.file.Path;
import java.util.Objects;
import org.pkl.core.Version;
import org.pkl.core.util.IoUtils;
import org.pkl.core.util.Nullable;
public abstract class Dependency {
@@ -48,10 +50,10 @@ public abstract class Dependency {
return path;
}
public Path resolveAssetPath(Path projectDir, PackageAssetUri packageAssetUri) {
public URI resolveAssetUri(URI projectBaseUri, PackageAssetUri packageAssetUri) {
// drop 1 to remove leading `/`
var assetPath = packageAssetUri.getAssetPath().substring(1);
return projectDir.resolve(path).resolve(assetPath);
return projectBaseUri.resolve(IoUtils.toNormalizedPathString(path.resolve(assetPath)));
}
@Override

View File

@@ -17,6 +17,7 @@ package org.pkl.core.project;
import java.net.URI;
import java.net.URISyntaxException;
import java.nio.file.FileSystemNotFoundException;
import java.nio.file.Path;
import java.util.HashMap;
import java.util.List;
@@ -27,6 +28,7 @@ import java.util.regex.Pattern;
import java.util.stream.Collectors;
import org.pkl.core.Composite;
import org.pkl.core.Duration;
import org.pkl.core.Evaluator;
import org.pkl.core.EvaluatorBuilder;
import org.pkl.core.ModuleSource;
import org.pkl.core.PClassInfo;
@@ -41,9 +43,11 @@ import org.pkl.core.Version;
import org.pkl.core.module.ModuleKeyFactories;
import org.pkl.core.packages.Checksums;
import org.pkl.core.packages.Dependency.RemoteDependency;
import org.pkl.core.packages.PackageLoadError;
import org.pkl.core.packages.PackageUri;
import org.pkl.core.packages.PackageUtils;
import org.pkl.core.resource.ResourceReaders;
import org.pkl.core.util.IoUtils;
import org.pkl.core.util.Nullable;
/** Java representation of module {@code pkl.Project}. */
@@ -52,8 +56,8 @@ public final class Project {
private final DeclaredDependencies dependencies;
private final EvaluatorSettings evaluatorSettings;
private final URI projectFileUri;
private final Path projectDir;
private final List<Path> tests;
private final URI projectBaseUri;
private final List<URI> tests;
private final Map<String, Project> localProjectDependencies;
/**
@@ -81,10 +85,7 @@ public final class Project {
.addEnvironmentVariables(envVars)
.setTimeout(timeout)
.build()) {
var output = evaluator.evaluateOutputValueAs(ModuleSource.path(path), PClassInfo.Project);
return Project.parseProject(output);
} catch (URISyntaxException e) {
throw new PklException(e.getMessage(), e);
return load(evaluator, ModuleSource.path(path));
}
}
@@ -103,6 +104,31 @@ public final class Project {
return loadFromPath(path, SecurityManagers.defaultManager, null);
}
/** Loads a project from the given source. */
public static Project load(ModuleSource moduleSource) {
try (var evaluator =
EvaluatorBuilder.unconfigured()
.setSecurityManager(SecurityManagers.defaultManager)
.setStackFrameTransformer(StackFrameTransformers.defaultTransformer)
.addModuleKeyFactory(ModuleKeyFactories.standardLibrary)
.addModuleKeyFactory(ModuleKeyFactories.file)
.addModuleKeyFactory(ModuleKeyFactories.classPath(Project.class.getClassLoader()))
.addResourceReader(ResourceReaders.environmentVariable())
.addResourceReader(ResourceReaders.file())
.build()) {
return load(evaluator, moduleSource);
}
}
public static Project load(Evaluator evaluator, ModuleSource moduleSource) {
try {
var output = evaluator.evaluateOutputValueAs(moduleSource, PClassInfo.Project);
return Project.parseProject(output);
} catch (URISyntaxException e) {
throw new PklException(e.getMessage(), e);
}
}
private static DeclaredDependencies parseDependencies(
PObject module, URI projectFileUri, @Nullable PackageUri packageUri)
throws URISyntaxException {
@@ -143,7 +169,7 @@ public final class Project {
var pkgObj = getNullableProperty(module, "package");
var projectFileUri = URI.create((String) module.getProperty("projectFileUri"));
var dependencies = parseDependencies(module, projectFileUri, null);
var projectDir = Path.of(projectFileUri).getParent();
var projectBaseUri = IoUtils.resolve(projectFileUri, ".");
Package pkg = null;
if (pkgObj != null) {
pkg = parsePackage((PObject) pkgObj);
@@ -152,12 +178,12 @@ public final class Project {
getProperty(
module,
"evaluatorSettings",
(settings) -> parseEvaluatorSettings(settings, projectDir));
(settings) -> parseEvaluatorSettings(settings, projectBaseUri));
@SuppressWarnings("unchecked")
var testPathStrs = (List<String>) getProperty(module, "tests");
var tests =
testPathStrs.stream()
.map((it) -> projectDir.resolve(it).normalize())
.map((it) -> projectBaseUri.resolve(it).normalize())
.collect(Collectors.toList());
var localProjectDependencies = parseLocalProjectDependencies(module);
return new Project(
@@ -165,7 +191,7 @@ public final class Project {
dependencies,
evaluatorSettings,
projectFileUri,
projectDir,
projectBaseUri,
tests,
localProjectDependencies);
}
@@ -185,7 +211,7 @@ public final class Project {
}
@SuppressWarnings("unchecked")
private static EvaluatorSettings parseEvaluatorSettings(Object settings, Path projectDir) {
private static EvaluatorSettings parseEvaluatorSettings(Object settings, URI projectBaseUri) {
var pSettings = (PObject) settings;
var externalProperties = getNullableProperty(pSettings, "externalProperties", Project::asMap);
var env = getNullableProperty(pSettings, "env", Project::asMap);
@@ -194,16 +220,18 @@ public final class Project {
getNullableProperty(pSettings, "allowedResources", Project::asPatternList);
var noCache = (Boolean) getNullableProperty(pSettings, "noCache");
var modulePathStrs = (List<String>) getNullableProperty(pSettings, "modulePath");
var timeout = (Duration) getNullableProperty(pSettings, "timeout");
List<Path> modulePath = null;
if (modulePathStrs != null) {
modulePath =
modulePathStrs.stream()
.map((it) -> projectDir.resolve(it).normalize())
.map((it) -> resolveNullablePath(it, projectBaseUri, "modulePath"))
.collect(Collectors.toList());
}
var timeout = (Duration) getNullableProperty(pSettings, "timeout");
var moduleCacheDir = getNullablePath(pSettings, "moduleCacheDir", projectDir);
var rootDir = getNullablePath(pSettings, "rootDir", projectDir);
var moduleCacheDir = getNullablePath(pSettings, "moduleCacheDir", projectBaseUri);
var rootDir = getNullablePath(pSettings, "rootDir", projectBaseUri);
return new EvaluatorSettings(
externalProperties,
env,
@@ -261,10 +289,28 @@ public final class Project {
return new URI((String) value);
}
/**
* Resolve a path string against projectBaseUri.
*
* @throws PackageLoadError if projectBaseUri is not a {@code file:} URI.
*/
private static @Nullable Path resolveNullablePath(
@Nullable String path, URI projectBaseUri, String propertyName) {
if (path == null) {
return null;
}
try {
return Path.of(projectBaseUri).resolve(path).normalize();
} catch (FileSystemNotFoundException e) {
throw new PackageLoadError(
"relativePathPropertyDefinedByProjectFromNonFileUri", projectBaseUri, propertyName);
}
}
private static @Nullable Path getNullablePath(
Composite object, String propertyName, Path projectDir) {
return getNullableProperty(
object, propertyName, (obj) -> projectDir.resolve((String) obj).normalize());
Composite object, String propertyName, URI projectBaseUri) {
return resolveNullablePath(
(String) getNullableProperty(object, propertyName), projectBaseUri, propertyName);
}
@SuppressWarnings("unchecked")
@@ -309,14 +355,14 @@ public final class Project {
DeclaredDependencies dependencies,
EvaluatorSettings evaluatorSettings,
URI projectFileUri,
Path projectDir,
List<Path> tests,
URI projectBaseUri,
List<URI> tests,
Map<String, Project> localProjectDependencies) {
this.pkg = pkg;
this.dependencies = dependencies;
this.evaluatorSettings = evaluatorSettings;
this.projectFileUri = projectFileUri;
this.projectDir = projectDir;
this.projectBaseUri = projectBaseUri;
this.tests = tests;
this.localProjectDependencies = localProjectDependencies;
}
@@ -334,7 +380,16 @@ public final class Project {
}
public List<Path> getTests() {
return tests;
return tests.stream()
.map(
(it) -> {
try {
return Path.of(it);
} catch (FileSystemNotFoundException e) {
throw new PackageLoadError("invalidUsageOfProjectFromNonFileUri");
}
})
.collect(Collectors.toList());
}
@Override
@@ -366,11 +421,17 @@ public final class Project {
return localProjectDependencies;
}
public URI getProjectBaseUri() {
return projectBaseUri;
}
public Path getProjectDir() {
return projectDir;
assert projectBaseUri.getScheme().equalsIgnoreCase("file");
return Path.of(projectBaseUri);
}
public static class EvaluatorSettings {
private final @Nullable Map<String, String> externalProperties;
private final @Nullable Map<String, String> env;
private final @Nullable List<Pattern> allowedModules;

View File

@@ -45,6 +45,7 @@ import org.pkl.core.SecurityManagerException;
import org.pkl.core.StackFrameTransformer;
import org.pkl.core.ast.builder.ImportsAndReadsParser;
import org.pkl.core.http.HttpClient;
import org.pkl.core.module.ModuleKeyFactories;
import org.pkl.core.module.ModuleKeys;
import org.pkl.core.module.ProjectDependenciesManager;
import org.pkl.core.module.ResolvedModuleKeys;
@@ -55,6 +56,7 @@ import org.pkl.core.packages.DependencyMetadata;
import org.pkl.core.packages.PackageLoadError;
import org.pkl.core.packages.PackageResolver;
import org.pkl.core.packages.PackageUri;
import org.pkl.core.runtime.ModuleResolver;
import org.pkl.core.runtime.VmExceptionBuilder;
import org.pkl.core.util.ByteArrayUtils;
import org.pkl.core.util.ErrorMessages;
@@ -98,6 +100,7 @@ public final class ProjectPackager {
private final Path workingDir;
private final String outputPathPattern;
private final StackFrameTransformer stackFrameTransformer;
private final SecurityManager securityManager;
private final PackageResolver packageResolver;
private final boolean skipPublishCheck;
private final Writer outputWriter;
@@ -115,6 +118,7 @@ public final class ProjectPackager {
this.workingDir = workingDir;
this.outputPathPattern = outputPathPattern;
this.stackFrameTransformer = stackFrameTransformer;
this.securityManager = securityManager;
// intentionally use InMemoryPackageResolver
this.packageResolver = PackageResolver.getInstance(securityManager, httpClient, null);
this.skipPublishCheck = skipPublishCheck;
@@ -226,7 +230,12 @@ public final class ProjectPackager {
new HashMap<String, RemoteDependency>(
project.getDependencies().getLocalDependencies().size()
+ project.getDependencies().getRemoteDependencies().size());
var projectDependenciesManager = new ProjectDependenciesManager(project.getDependencies());
// module resolver is only used for reading PklProject.deps.json, so provide one that reads
// files.
var moduleResolver = new ModuleResolver(List.of(ModuleKeyFactories.file));
var projectDependenciesManager =
new ProjectDependenciesManager(
project.getDependencies(), moduleResolver, this.securityManager);
for (var entry : project.getDependencies().getRemoteDependencies().entrySet()) {
var resolved =
(RemoteDependency)

View File

@@ -89,7 +89,9 @@ public class ReplServer implements AutoCloseable {
var languageRef = new MutableReference<VmLanguage>(null);
packageResolver = PackageResolver.getInstance(securityManager, httpClient, moduleCacheDir);
projectDependenciesManager =
projectDependencies == null ? null : new ProjectDependenciesManager(projectDependencies);
projectDependencies == null
? null
: new ProjectDependenciesManager(projectDependencies, moduleResolver, securityManager);
polyglotContext =
VmUtils.createContext(
() -> {

View File

@@ -22,7 +22,6 @@ import java.net.URISyntaxException;
import java.net.http.HttpRequest;
import java.net.http.HttpResponse.BodyHandlers;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
@@ -39,6 +38,7 @@ import org.pkl.core.packages.Dependency.LocalDependency;
import org.pkl.core.packages.PackageAssetUri;
import org.pkl.core.packages.PackageResolver;
import org.pkl.core.runtime.VmContext;
import org.pkl.core.runtime.VmExceptionBuilder;
import org.pkl.core.util.ErrorMessages;
import org.pkl.core.util.HttpUtils;
import org.pkl.core.util.IoUtils;
@@ -486,10 +486,9 @@ public final class ResourceReaders {
throws IOException, URISyntaxException, SecurityManagerException {
var assetUri = new PackageAssetUri(uri);
var dependency = getProjectDepsResolver().getResolvedDependency(assetUri.getPackageUri());
var path = getLocalPath(dependency, assetUri);
if (path != null) {
var bytes = Files.readAllBytes(path);
return Optional.of(new Resource(uri, bytes));
var local = getLocalUri(dependency, assetUri);
if (local != null) {
return VmContext.get(null).getResourceManager().read(local, null);
}
var remoteDep = (Dependency.RemoteDependency) dependency;
var bytes = getPackageResolver().getBytes(assetUri, true, remoteDep.getChecksums());
@@ -518,9 +517,15 @@ public final class ResourceReaders {
var packageAssetUri = PackageAssetUri.create(baseUri);
var dependency =
getProjectDepsResolver().getResolvedDependency(packageAssetUri.getPackageUri());
var path = getLocalPath(dependency, packageAssetUri);
if (path != null) {
return FileResolver.listElements(path);
var local = getLocalUri(dependency, packageAssetUri);
if (local != null) {
var reader = VmContext.get(null).getResourceManager().getResourceReader(local);
if (reader == null) {
throw new VmExceptionBuilder()
.evalError("noResourceReaderRegistered", local.getScheme())
.build();
}
return reader.listElements(securityManager, local);
}
var remoteDep = (Dependency.RemoteDependency) dependency;
return getPackageResolver()
@@ -534,9 +539,15 @@ public final class ResourceReaders {
var packageAssetUri = PackageAssetUri.create(elementUri);
var dependency =
getProjectDepsResolver().getResolvedDependency(packageAssetUri.getPackageUri());
var path = getLocalPath(dependency, packageAssetUri);
if (path != null) {
return FileResolver.hasElement(path);
var local = getLocalUri(dependency, packageAssetUri);
if (local != null) {
var reader = VmContext.get(null).getResourceManager().getResourceReader(local);
if (reader == null) {
throw new VmExceptionBuilder()
.evalError("noResourceReaderRegistered", local.getScheme())
.build();
}
return reader.hasElement(securityManager, local);
}
var remoteDep = (Dependency.RemoteDependency) dependency;
return getPackageResolver()
@@ -555,12 +566,12 @@ public final class ResourceReaders {
return projectDepsManager;
}
private @Nullable Path getLocalPath(Dependency dependency, PackageAssetUri packageAssetUri) {
private @Nullable URI getLocalUri(Dependency dependency, PackageAssetUri packageAssetUri) {
if (!(dependency instanceof LocalDependency localDependency)) {
return null;
}
return localDependency.resolveAssetPath(
getProjectDepsResolver().getProjectDir(), packageAssetUri);
return localDependency.resolveAssetUri(
getProjectDepsResolver().getProjectBaseUri(), packageAssetUri);
}
}

View File

@@ -31,6 +31,7 @@ import org.pkl.core.packages.PackageLoadError;
import org.pkl.core.resource.Resource;
import org.pkl.core.resource.ResourceReader;
import org.pkl.core.stdlib.VmObjectFactory;
import org.pkl.core.util.Nullable;
public final class ResourceManager {
private final Map<String, ResourceReader> resourceReaders = new HashMap<>();
@@ -67,17 +68,23 @@ public final class ResourceManager {
}
@TruffleBoundary
public Optional<Object> read(URI resourceUri, Node readNode) {
public Optional<Object> read(URI resourceUri, @Nullable Node readNode) {
return resources.computeIfAbsent(
resourceUri.normalize(),
uri -> {
try {
securityManager.checkReadResource(uri);
} catch (SecurityManagerException e) {
throw new VmExceptionBuilder().withCause(e).withLocation(readNode).build();
throw new VmExceptionBuilder().withCause(e).withOptionalLocation(readNode).build();
}
var reader = getReader(resourceUri, readNode);
var reader = resourceReaders.get(uri.getScheme());
if (reader == null) {
throw new VmExceptionBuilder()
.withOptionalLocation(readNode)
.evalError("noResourceReaderRegistered", resourceUri.getScheme())
.build();
}
Optional<Object> resource;
try {
@@ -86,16 +93,16 @@ public final class ResourceManager {
throw new VmExceptionBuilder()
.evalError("ioErrorReadingResource", uri)
.withCause(e)
.withLocation(readNode)
.withOptionalLocation(readNode)
.build();
} catch (URISyntaxException e) {
throw new VmExceptionBuilder()
.evalError("invalidResourceUri", resourceUri)
.withHint(e.getReason())
.withLocation(readNode)
.withOptionalLocation(readNode)
.build();
} catch (SecurityManagerException | PackageLoadError | HttpClientInitException e) {
throw new VmExceptionBuilder().withCause(e).withLocation(readNode).build();
throw new VmExceptionBuilder().withCause(e).withOptionalLocation(readNode).build();
}
if (resource.isEmpty()) return resource;
@@ -108,8 +115,16 @@ public final class ResourceManager {
throw new VmExceptionBuilder()
.evalError("unsupportedResourceType", reader.getClass().getName(), res.getClass())
.withLocation(readNode)
.withOptionalLocation(readNode)
.build();
});
}
/**
* Returns a {@link ResourceReader} registered to read the resource at {@code baseUri}, or {@code
* null} if there is none.
*/
public @Nullable ResourceReader getResourceReader(URI baseUri) {
return resourceReaders.get(baseUri.getScheme());
}
}

View File

@@ -324,7 +324,7 @@ public final class VmExceptionBuilder {
return this;
}
public VmExceptionBuilder withHint(String hint) {
public VmExceptionBuilder withHint(@Nullable String hint) {
this.hint = hint;
return this;
}

View File

@@ -28,6 +28,7 @@ import java.util.Map;
import java.util.WeakHashMap;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
import org.pkl.core.PklBugException;
import org.pkl.core.SecurityManager;
import org.pkl.core.SecurityManagerException;
import org.pkl.core.module.ModuleKey;
@@ -490,7 +491,14 @@ public final class GlobResolver {
}
return result;
}
var baseUri = enclosingModuleKey.resolveUri(enclosingUri, URI.create(basePath));
URI baseUri;
try {
baseUri = IoUtils.resolve(securityManager, enclosingModuleKey, URI.create(basePath));
} catch (URISyntaxException e) {
// assertion: this is only thrown if the pattern starts with a triple-dot import.
// the language will throw an error if glob imports is combined with triple-dots.
throw new PklBugException(e);
}
resolveHierarchicalGlob(
securityManager,

View File

@@ -39,7 +39,9 @@ import org.pkl.core.Platform;
import org.pkl.core.SecurityManager;
import org.pkl.core.SecurityManagerException;
import org.pkl.core.module.ModuleKey;
import org.pkl.core.packages.PackageLoadError;
import org.pkl.core.runtime.ReaderBase;
import org.pkl.core.runtime.VmContext;
import org.pkl.core.runtime.VmExceptionBuilder;
public final class IoUtils {
@@ -299,29 +301,15 @@ public final class IoUtils {
return resolve(baseUri, importUri);
}
/**
* Resolves {@code importUri} against the module key.
*
* <p>When {@code importUri} contains a triple-dot, it is resolved if the module key returns true
* for both {@link ModuleKey#isLocal()} and {@link ModuleKey#hasHierarchicalUris()}. Otherwise, an
* error is thrown.
*/
public static URI resolve(SecurityManager securityManager, ModuleKey moduleKey, URI importUri)
throws URISyntaxException, IOException, SecurityManagerException {
if (importUri.isAbsolute()) {
return moduleKey.resolveUri(importUri);
}
private static URI resolveTripleDotImport(
SecurityManager securityManager, ModuleKey moduleKey, String tripleDotPath)
throws IOException, SecurityManagerException {
var moduleKeyUri = moduleKey.getUri();
var tripleDotPath = parseTripleDotPath(importUri);
if (tripleDotPath == null) {
return moduleKey.resolveUri(importUri);
}
if (!moduleKey.isLocal() || !moduleKey.hasHierarchicalUris()) {
throw new VmExceptionBuilder()
.evalError("cannotResolveTripleDotImports", moduleKeyUri)
.build();
}
var currentPath =
moduleKey.hasFragmentPaths() ? moduleKeyUri.getFragment() : moduleKeyUri.getPath();
var effectiveImportPath =
@@ -351,6 +339,69 @@ public final class IoUtils {
throw new FileNotFoundException();
}
public static Pair<String, String> parseDependencyNotation(String importPath) {
var idx = importPath.indexOf('/');
if (idx == -1) {
// treat named dependency without a subpath as the root path.
// i.e. resolve to `@foo` to `package://example.com/foo@1.0.0#/`
return Pair.of(importPath.substring(1), "/");
}
return Pair.of(importPath.substring(1, idx), importPath.substring(idx));
}
private static URI resolveProjectDependency(ModuleKey moduleKey, String notation) {
var parsed = parseDependencyNotation(notation);
var name = parsed.getFirst();
var path = parsed.getSecond();
var projectDependenciesManager = VmContext.get(null).getProjectDependenciesManager();
if (!moduleKey.hasHierarchicalUris() && projectDependenciesManager != null) {
throw new PackageLoadError(
"cannotResolveDependencyWithoutHierarchicalUris",
projectDependenciesManager.getProjectFileUri());
}
if (projectDependenciesManager == null
|| !projectDependenciesManager.hasUri(moduleKey.getUri())) {
throw new PackageLoadError("cannotResolveDependencyNoProject");
}
var dependency = projectDependenciesManager.getDependencies().get(name);
if (dependency != null) {
return dependency.getPackageUri().toPackageAssetUri(path).getUri();
}
throw new PackageLoadError("cannotFindDependencyInProject", name);
}
/**
* Resolves {@code importUri} against the module key.
*
* <p>When {@code importUri} contains a triple-dot, it is resolved if the module key returns true
* for both {@link ModuleKey#isLocal()} and {@link ModuleKey#hasHierarchicalUris()}. Otherwise, an
* error is thrown.
*
* <p>When {@code importUri} starts with a {@code @}, it is resolved if the module key supports
* dependency notation ()
*/
public static URI resolve(SecurityManager securityManager, ModuleKey moduleKey, URI importUri)
throws URISyntaxException, IOException, SecurityManagerException {
if (importUri.isAbsolute()) {
return moduleKey.resolveUri(importUri);
}
var tripleDotPath = parseTripleDotPath(importUri);
if (tripleDotPath != null) {
return resolveTripleDotImport(securityManager, moduleKey, tripleDotPath);
}
var moduleScheme = moduleKey.getUri().getScheme();
var isPackage =
moduleScheme.equalsIgnoreCase("package") || moduleScheme.equalsIgnoreCase("projectpackage");
var relativePart = importUri.getSchemeSpecificPart();
// Special-case handling of project dependencies.
// We'll allow the Package and ProjectPackage module keys to resolve dependency notation on
// their own.
if (relativePart.startsWith("@") && !isPackage) {
return resolveProjectDependency(moduleKey, relativePart);
}
return moduleKey.resolveUri(importUri);
}
public static URI resolve(URI baseUri, URI newUri) {
if (newUri.isAbsolute()) return newUri;

View File

@@ -542,6 +542,12 @@ Cannot combine glob imports with triple-dot module URIs.
cannotGlobUri=\
Cannot expand glob pattern `{0}` because scheme `{1}` is not globbable.
cannotResolveInLocalDependencyNotGlobbable=\
Cannot resolve import in local dependency because scheme `{0}` is not globbable.
cannotResolveInLocalDependencyNotGlobbableNorLocal=\
Cannot resolve import in local dependency because scheme `{0}` is not globbable and is not local.
expectedAnnotationClass=\
Expected an annotation class.
@@ -680,6 +686,11 @@ Cannot resolve a triple-dot import from module URI `{0}`.\n\
\n\
Triple-dot imports may only be resolved by module schemes that are considered local, and have hierarchical URIs.
moduleDoesNotSupportDependencies=\
Module `{0}` does not support importing dependencies.\n\
\n\
Dependencies can only be imported in modules that belong to a project, or within a package.
cannotHaveRelativeImport=\
Module `{0}` cannot have a relative import URI.
@@ -819,6 +830,9 @@ Only type unions can have a default marker (*).
invalidModuleOutputValue=\
Expected `output.value` of module `{2}` to be of type `{0}`, but got type `{1}`.
cannotResolveDependencyWithoutHierarchicalUris=\
Cannot import dependency because project URI `{0}` does not have a hierarchical path.
cannotResolveDependencyNoProject=\
Cannot import dependency because there is no project found.\n\
\n\
@@ -830,6 +844,11 @@ Cannot find a dependency named `{0}`, because it is not declared in the current
\n\
To fix this, add it to the `dependencies` section of your `PklProject` file, and resolve your dependencies.
cannotResolveDependencyFromReaderWithOpaqueUris=\
Cannot resolve dependencies from module reader with opaque URIs.\n\
\n\
Module reader for scheme `{0}` does not support hierarchical URIs.
cannotFindDependencyInPackage=\
Cannot find dependency named `{0}`, because it was not declared in package `{1}`.
@@ -856,10 +875,10 @@ For example, `example.com` in URI `project://example.com/my/package@1.0.0`.
unexpectedChecksumInPackageUri=\
Did not expect to find a checksum component in this package URI.
missingProjectDepsJson=\
Cannot resolve dependency because file `PklProject.deps.json` is missing in project directory `{0}`.\n\
cannotLoadProjectDepsJson=\
Encountered an error when attempting to load `PklProject.deps.json` at `{0}`.\n\
\n\
Run `pkl project resolve` to create a new set of dependencies.
Try running `pkl project resolve` within the project directory to create a new set of dependencies.
invalidProjectDepsJson=\
Cannot resolve dependency because file `{0}` is malformed.\n\
@@ -978,6 +997,16 @@ No package was declared in project `{0}`.\n\
\n\
Add a `package` section to the PklProject file.
relativePathPropertyDefinedByProjectFromNonFileUri=\
Invalid property specified in project `{0}`\n\
\n\
Property `{1}` is only permitted in PklProject files loaded from `file:` URIs.
invalidUsageOfProjectFromNonFileUri=\
Invalid usage of project `{0}`\n\
\n\
This action can only be performed with PklProject files loaded from `file:` URIs.
packageTestsFailed=\
Failed to create package `{0}`, because its API tests are failing.

View File

@@ -11,4 +11,5 @@ dependencies {
uri = "package://localhost:0/badImportsWithinPackage@1.0.0"
}
["project2"] = import("../project2/PklProject")
["project6"] = import("../project6/PklProject")
}

View File

@@ -20,6 +20,11 @@
"uri": "projectpackage://localhost:0/project2@1.0.0",
"path": "../project2/"
},
"package://localhost:12110/project6@1": {
"type": "local",
"uri": "projectpackage://localhost:12110/project6@1.0.0",
"path": "../project6/"
},
"package://localhost:0/badImportsWithinPackage@1": {
"type": "remote",
"uri": "projectpackage://localhost:0/badImportsWithinPackage@1.0.0",

View File

@@ -34,4 +34,8 @@ examples {
["glob-read absolute package uri"] {
read*("package://localhost:0/birds@0.5.0#/catalog/*.pkl")
}
["glob-import behind local project import"] {
import("@project6/children.pkl")
}
}

View File

@@ -20,6 +20,11 @@
"uri": "projectpackage://localhost:0/project2@1.0.0",
"path": "../project2/"
},
"package://localhost:12110/project6@1": {
"type": "local",
"uri": "projectpackage://localhost:12110/project6@1.0.0",
"path": "../project6/"
},
"package://localhost:0/badImportsWithinPackage@1": {
"type": "remote",
"uri": "projectpackage://localhost:0/badImportsWithinPackage@1.0.0",

View File

@@ -0,0 +1,8 @@
amends "pkl:Project"
package {
name = "project6"
baseUri = "package://localhost:12110/project6"
version = "1.0.0"
packageZipUrl = "https://localhost:12110/project6/project6-\(version).zip"
}

View File

@@ -0,0 +1,4 @@
{
"schemaVersion": 1,
"resolvedDependencies": {}
}

View File

@@ -0,0 +1 @@
children = import*("children/*.pkl")

View File

@@ -0,0 +1 @@
name = "a"

View File

@@ -0,0 +1 @@
name = "b"

View File

@@ -0,0 +1 @@
name = "c"

View File

@@ -1,13 +1,13 @@
Pkl Error
Expected value of type `*RemoteDependency|LocalDependency`, but got a different `pkl.Project`.
Expected value of type `*RemoteDependency|Project(isValidLoadDependency)`, but got a different `pkl.Project`.
Value: new ModuleClass { package = null; tests {}; dependencies {}; evaluatorSetting...
xxx | dependencies: Mapping<String(!contains("/")), *RemoteDependency|LocalDependency>
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
xxx | dependencies: Mapping<String(!contains("/")), *RemoteDependency|Project(isValidLoadDependency)>
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
at pkl.Project#dependencies (pkl:Project)
* Value is not of type `LocalDependency` because:
Type constraint `this.package != null` violated.
* Value is not of type `Project(isValidLoadDependency)` because:
Type constraint `isValidLoadDependency` violated.
Value: new ModuleClass { package = null; tests {}; dependencies {}; evaluatorSetti...
x | dependencies {

View File

@@ -1,8 +1,9 @@
Pkl Error
Cannot resolve dependency because file `PklProject.deps.json` is missing in project directory `file:///$snippetsDir/input/projects/missingProjectDeps/`.
Encountered an error when attempting to load `PklProject.deps.json` at `file:///$snippetsDir/input/projects/missingProjectDeps/PklProject.deps.json`.
NoSuchFileException: /$snippetsDir/input/projects/missingProjectDeps/PklProject.deps.json
x | import "@birds/Bird.pkl"
^^^^^^^^^^^^^^^^^
at bug (file:///$snippetsDir/input/projects/missingProjectDeps/bug.pkl)
Run `pkl project resolve` to create a new set of dependencies.
Try running `pkl project resolve` within the project directory to create a new set of dependencies.

View File

@@ -262,4 +262,19 @@ examples {
}
}
}
["glob-import behind local project import"] {
new {
children {
["children/a.pkl"] {
name = "a"
}
["children/b.pkl"] {
name = "b"
}
["children/c.pkl"] {
name = "c"
}
}
}
}
}

View File

@@ -0,0 +1,11 @@
children {
["children/a.pkl"] {
name = "a"
}
["children/b.pkl"] {
name = "b"
}
["children/c.pkl"] {
name = "c"
}
}

View File

@@ -0,0 +1 @@
name = "a"

View File

@@ -0,0 +1 @@
name = "b"

View File

@@ -0,0 +1 @@
name = "c"

View File

@@ -6,6 +6,7 @@ import java.net.URI
import java.nio.file.Files
import java.nio.file.Path
import org.assertj.core.api.Assertions.assertThat
import org.assertj.core.api.Assertions.assertThatCode
import org.junit.jupiter.api.Test
import org.junit.jupiter.api.assertThrows
import org.junit.jupiter.api.io.TempDir
@@ -15,7 +16,16 @@ import org.pkl.commons.writeString
import org.pkl.core.ModuleSource.*
import org.pkl.core.util.IoUtils
import org.junit.jupiter.api.AfterAll
import org.pkl.commons.test.PackageServer
import org.pkl.core.module.ModuleKey
import org.pkl.core.module.ModuleKeyFactories
import org.pkl.core.module.ModuleKeyFactory
import org.pkl.core.module.ResolvedModuleKey
import org.pkl.core.project.Project
import java.nio.charset.StandardCharsets
import java.nio.file.FileSystems
import java.util.*
import java.util.regex.Pattern
import kotlin.io.path.writeText
class EvaluatorTest {
@@ -24,6 +34,28 @@ class EvaluatorTest {
private const val sourceText = "name = \"pigeon\"; age = 10 + 20"
private object CustomModuleKeyFactory : ModuleKeyFactory {
override fun create(uri: URI): Optional<ModuleKey> {
return if (uri.scheme == "custom") Optional.of(CustomModuleKey(uri))
else Optional.empty<ModuleKey>()
}
}
private class CustomModuleKey(private val uri: URI) : ModuleKey, ResolvedModuleKey {
override fun hasHierarchicalUris(): Boolean = true
override fun isGlobbable(): Boolean = false
override fun getOriginal(): ModuleKey = this
override fun getUri(): URI = uri
override fun loadSource(): String = javaClass.classLoader.getResourceAsStream(uri.path.drop(1))!!.use { it.readAllBytes().toString(
StandardCharsets.UTF_8) }
override fun resolve(securityManager: SecurityManager): ResolvedModuleKey = this
}
@AfterAll
@JvmStatic
fun afterAll() {
@@ -291,6 +323,132 @@ class EvaluatorTest {
assertThat(output["bar/../bark.yml"]?.text).isEqualTo("bark: bark bark")
}
@Test
fun `project set from modulepath`(@TempDir cacheDir: Path) {
PackageServer.populateCacheDir(cacheDir)
val evaluatorBuilder = EvaluatorBuilder.preconfigured().setModuleCacheDir(cacheDir)
val project = Project.load(modulePath("/org/pkl/core/project/project5/PklProject"))
val result = evaluatorBuilder.setProjectDependencies(project.dependencies).build().use { evaluator ->
evaluator.evaluateOutputText(modulePath("/org/pkl/core/project/project5/main.pkl"))
}
assertThat(result).isEqualTo("""
prop1 {
name = "Apple"
}
prop2 {
res = 1
}
""".trimIndent())
}
@Test
fun `project set from custom ModuleKeyFactory`(@TempDir cacheDir: Path) {
PackageServer.populateCacheDir(cacheDir)
val evaluatorBuilder = with(EvaluatorBuilder.preconfigured()) {
setAllowedModules(SecurityManagers.defaultAllowedModules + Pattern.compile("custom:"))
setAllowedResources(SecurityManagers.defaultAllowedResources + Pattern.compile("custom:"))
setModuleCacheDir(cacheDir)
setModuleKeyFactories(
listOf(
CustomModuleKeyFactory,
ModuleKeyFactories.standardLibrary,
ModuleKeyFactories.pkg,
ModuleKeyFactories.projectpackage,
ModuleKeyFactories.file
)
)
}
val project = evaluatorBuilder.build().use { Project.load(it, uri("custom:/org/pkl/core/project/project5/PklProject")) }
val evaluator = evaluatorBuilder.setProjectDependencies(project.dependencies).build()
val output = evaluator.use { it.evaluateOutputText(uri("custom:/org/pkl/core/project/project5/main.pkl")) }
assertThat(output)
.isEqualTo(
"""
prop1 {
name = "Apple"
}
prop2 {
res = 1
}
"""
.trimIndent()
)
}
@Test
fun `project base path set to non-hierarchical scheme`() {
class FooBarModuleKey(val moduleUri: URI) : ModuleKey, ResolvedModuleKey {
override fun hasHierarchicalUris(): Boolean = false
override fun isGlobbable(): Boolean = false
override fun getOriginal(): ModuleKey = this
override fun getUri(): URI = moduleUri
override fun loadSource(): String =
if (uri.schemeSpecificPart.endsWith("PklProject")) {
"""
amends "pkl:Project"
""".trimIndent()
} else """
birds = import("@birds/catalog/Ostritch.pkl")
""".trimIndent()
override fun resolve(securityManager: SecurityManager): ResolvedModuleKey {
return this
}
}
val fooBayModuleKeyFactory = ModuleKeyFactory { uri ->
if (uri.scheme == "foobar") Optional.of(FooBarModuleKey(uri))
else Optional.empty()
}
val evaluatorBuilder = with(EvaluatorBuilder.preconfigured()) {
setAllowedModules(SecurityManagers.defaultAllowedModules + Pattern.compile("foobar:"))
setAllowedResources(SecurityManagers.defaultAllowedResources + Pattern.compile("foobar:"))
setModuleKeyFactories(
listOf(
fooBayModuleKeyFactory,
ModuleKeyFactories.standardLibrary,
ModuleKeyFactories.pkg,
ModuleKeyFactories.projectpackage,
ModuleKeyFactories.file
)
)
}
val project = evaluatorBuilder.build().use { Project.load(it, uri("foobar:foo/PklProject")) }
val evaluator = evaluatorBuilder.setProjectDependencies(project.dependencies).build()
assertThatCode { evaluator.use { it.evaluateOutputText(uri("foobar:baz")) } }
.hasMessageContaining("Cannot import dependency because project URI `foobar:foo/PklProject` does not have a hierarchical path.")
}
@Test
fun `cannot glob import in local dependency from modulepath`(@TempDir cacheDir: Path) {
PackageServer.populateCacheDir(cacheDir)
val evaluatorBuilder = EvaluatorBuilder.preconfigured().setModuleCacheDir(cacheDir)
val project = Project.load(modulePath("/org/pkl/core/project/project6/PklProject"))
evaluatorBuilder.setProjectDependencies(project.dependencies).build().use { evaluator ->
assertThatCode {
evaluator.evaluateOutputText(modulePath("/org/pkl/core/project/project6/globWithinDependency.pkl"))
}.hasMessageContaining("""
Cannot resolve import in local dependency because scheme `modulepath` is not globbable.
1 | res = import*("*.pkl")
^^^^^^^^^^^^^^^^
""".trimIndent())
assertThatCode {
evaluator.evaluateOutputText(modulePath("/org/pkl/core/project/project6/globIntoDependency.pkl"))
}.hasMessageContaining("""
Pkl Error
Cannot resolve import in local dependency because scheme `modulepath` is not globbable.
1 | import* "@project7/*.pkl" as proj7Files
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
""".trimIndent())
}
}
private fun checkModule(module: PModule) {
assertThat(module.properties.size).isEqualTo(2)
assertThat(module.getProperty("name")).isEqualTo("pigeon")

View File

@@ -38,10 +38,15 @@ abstract class AbstractLanguageSnippetTestsEngine : InputOutputTestEngine() {
internal val selection: String = ""
protected val packageServer: PackageServer = PackageServer()
override val includedTests: List<Regex> = listOf(Regex(".*$selection\\.pkl"))
override val excludedTests: List<Regex> = listOf(Regex(".*/native/.*"))
override val excludedTests: List<Regex> = buildList {
add(Regex(".*/native/.*"))
if (IoUtils.isWindows()) {
addAll(windowsExcludedTests)
}
}
override val inputDir: Path = snippetsDir.resolve("input")
@@ -68,7 +73,12 @@ abstract class AbstractLanguageSnippetTestsEngine : InputOutputTestEngine() {
packageServer.close()
}
protected fun String.stripFilePaths() = replace(snippetsDir.toUri().toString(), "file:///\$snippetsDir/")
private val replacement by lazy {
if (snippetsDir.root.toString() != "/") "\$snippetsDir" else "/\$snippetsDir"
}
protected fun String.stripFilePaths(): String =
replace(IoUtils.toNormalizedPathString(snippetsDir), replacement)
protected fun String.stripLineNumbers() = replace(lineNumberRegex) { result ->
// replace line number with equivalent number of 'x' characters to keep formatting intact
@@ -80,7 +90,7 @@ abstract class AbstractLanguageSnippetTestsEngine : InputOutputTestEngine() {
// can't think of a better solution right now
protected fun String.stripVersionCheckErrorMessage() =
replace("Pkl version is ${Release.current().version()}", "Pkl version is xxx")
protected fun String.stripStdlibLocationSha(): String =
replace("https://github.com/apple/pkl/blob/${Release.current().commitId()}/stdlib/", "https://github.com/apple/pkl/blob/\$commitId/stdlib/")
@@ -261,7 +271,12 @@ class AlpineLanguageSnippetTestsEngine : AbstractNativeLanguageSnippetTestsEngin
override val testClass: KClass<*> = AlpineLanguageSnippetTests::class
}
// error message contains different file path on Windows
private val windowsExcludedTests get() = listOf(Regex(".*missingProjectDeps/bug\\.pkl"))
class WindowsLanguageSnippetTestsEngine : AbstractNativeLanguageSnippetTestsEngine() {
override val pklExecutablePath: Path = PklExecutablePaths.windowsAmd64
override val testClass: KClass<*> = WindowsLanguageSnippetTests::class
override val excludedTests: List<Regex>
get() = super.excludedTests + windowsExcludedTests
}

View File

@@ -1,16 +1,16 @@
package org.pkl.core.project
import org.pkl.commons.test.PackageServer
import org.pkl.commons.writeString
import org.pkl.core.*
import org.pkl.core.packages.PackageUri
import org.pkl.core.project.Project.EvaluatorSettings
import org.assertj.core.api.Assertions.assertThat
import org.assertj.core.api.Assertions.assertThatCode
import org.junit.jupiter.api.Test
import org.junit.jupiter.api.io.TempDir
import org.pkl.commons.test.FileTestUtils
import org.pkl.commons.test.PackageServer
import org.pkl.commons.writeString
import org.pkl.core.*
import org.pkl.core.http.HttpClient
import org.pkl.core.packages.PackageUri
import org.pkl.core.project.Project.EvaluatorSettings
import java.net.URI
import java.nio.file.Path
import java.util.regex.Pattern

View File

@@ -0,0 +1,8 @@
amends "pkl:Project"
dependencies {
["fruit"] {
uri = "package://localhost:0/fruit@1.0.5"
}
["project4"] = import("../project4/PklProject")
}

View File

@@ -0,0 +1,17 @@
{
"schemaVersion": 1,
"resolvedDependencies": {
"package://localhost:0/fruit@1": {
"type": "remote",
"uri": "projectpackage://localhost:0/fruit@1.0.5",
"checksums": {
"sha256": "$skipChecksumVerification"
}
},
"package://localhost:0/project4@1": {
"type": "local",
"uri": "projectpackage://localhost:0/project4@1.0.0",
"path": "../project4"
}
}
}

View File

@@ -0,0 +1,5 @@
import "@fruit/catalog/apple.pkl"
import "@project4/module1.pkl"
prop1 = apple
prop2 = module1

View File

@@ -0,0 +1,5 @@
amends "pkl:Project"
dependencies {
["project7"] = import("../project7/PklProject")
}

View File

@@ -0,0 +1,10 @@
{
"schemaVersion": 1,
"resolvedDependencies": {
"package://localhost:0/project7@1": {
"type": "local",
"uri": "projectpackage://localhost:0/project7@1.0.0",
"path": "../project7"
}
}
}

View File

@@ -0,0 +1,3 @@
import* "@project7/*.pkl" as proj7Files
res = proj7Files

View File

@@ -0,0 +1,3 @@
import "@project7/main.pkl"
res = main.res

View File

@@ -0,0 +1,8 @@
amends "pkl:Project"
package {
name = "project7"
version = "1.0.0"
packageZipUrl = "https://bogus.value"
baseUri = "package://localhost:0/project7"
}

View File

@@ -0,0 +1 @@
res = import*("*.pkl")