From 207d0c78f0cb8ddb7332672883e8cccf16c6a3a2 Mon Sep 17 00:00:00 2001 From: translatenix <119817707+translatenix@users.noreply.github.com> Date: Mon, 3 Jun 2024 17:10:56 -0700 Subject: [PATCH] Fix globbed read/import bugs (#449) * Split MemberNode into (Regular/Shared)MemberNode SharedMemberNode enables generating non-constant object members at run time without generating an unbounded number of Truffle root nodes. * Invert shouldRunTypeCheck to match its name * Introduce VmObjectBuilder Introduce VmObjectBuilder, a uniform way to build `VmObject`s whose `ObjectMember`s are determined at run time. Replace some manual object building code with VmObjectBuilder. Add some assertions to fix IntelliJ warnings. * Improve implementation of globbed read/import nodes - Leverage SharedMemberNode to have a single Truffle root node per globbed read/import expression instead of one root node per resolved glob element. - Remove caching in ReadGlobNode because it only works correctly if glob pattern is a string *constant*. - Remove caching in StaticReadNode (now ReadGlobElementNode/ImportGlobElementNode) because it seems unnecessary and the implementation doesn't look quite right. * Simplify code * Fix ClassCastException when reflecting on globbed import * Fix caching of globbed reads Problem: The result of a globbed read depends not only on the glob pattern but also on the current module URI. However, ResourceManager does not take this into account when caching globbed reads, causing wrong results. Changes: - Cache globbed reads per read expression. This is also how globbed imports are cached, except that import URIs are constant. - Make ReadGlobNode and ImportGlobNode code as similar as possible. - Reduce code duplication by inheriting from AbstractReadNode. - Add some tests. --- .../java/org/pkl/core/ast/MemberNode.java | 30 +--- .../org/pkl/core/ast/builder/AstBuilder.java | 6 +- .../expression/unary/AbstractImportNode.java | 7 +- .../expression/unary/AbstractReadNode.java | 25 ++-- .../unary/ImportGlobMemberBodyNode.java | 67 +++++++++ .../ast/expression/unary/ImportGlobNode.java | 121 +++++++--------- .../core/ast/expression/unary/ImportNode.java | 4 +- .../unary/ReadGlobMemberBodyNode.java | 52 +++++++ .../ast/expression/unary/ReadGlobNode.java | 135 +++++++++--------- .../ast/expression/unary/StaticReadNode.java | 50 ------- .../org/pkl/core/ast/member/FunctionNode.java | 3 +- .../ast/member/LocalTypedPropertyNode.java | 3 +- .../pkl/core/ast/member/ObjectMethodNode.java | 3 +- .../core/ast/member/RegularMemberNode.java | 52 +++++++ .../pkl/core/ast/member/SharedMemberNode.java | 64 +++++++++ .../ast/member/TypeCheckedPropertyNode.java | 7 +- .../core/ast/member/TypedPropertyNode.java | 5 +- .../ast/member/UntypedObjectMemberNode.java | 3 +- .../org/pkl/core/runtime/ModuleCache.java | 7 +- .../java/org/pkl/core/runtime/ModuleInfo.java | 10 +- .../org/pkl/core/runtime/ResourceManager.java | 78 ++-------- .../java/org/pkl/core/runtime/VmLanguage.java | 3 +- .../java/org/pkl/core/runtime/VmList.java | 14 ++ .../main/java/org/pkl/core/runtime/VmMap.java | 22 +++ .../org/pkl/core/runtime/VmObjectBuilder.java | 86 +++++++++++ .../main/java/org/pkl/core/runtime/VmSet.java | 14 ++ .../java/org/pkl/core/runtime/VmUtils.java | 7 +- .../org/pkl/core/stdlib/base/ListNodes.java | 26 +--- .../org/pkl/core/stdlib/base/MapNodes.java | 39 +---- .../org/pkl/core/stdlib/base/SetNodes.java | 28 +--- .../java/org/pkl/core/util/GlobResolver.java | 18 +-- .../input-helper/basic/read/child/module2.pkl | 2 + .../basic/read/child/resource.txt | 1 + .../input-helper/basic/read/module1.pkl | 2 + .../input-helper/basic/read/resource.txt | 1 + .../LanguageSnippetTests/input/basic/read.pkl | 22 ++- .../input/basic/readGlob.pkl | 14 ++ .../output/basic/read.pcf | 134 ++++++++--------- .../output/basic/readGlob.pcf | 62 ++++++++ 39 files changed, 724 insertions(+), 503 deletions(-) create mode 100644 pkl-core/src/main/java/org/pkl/core/ast/expression/unary/ImportGlobMemberBodyNode.java create mode 100644 pkl-core/src/main/java/org/pkl/core/ast/expression/unary/ReadGlobMemberBodyNode.java delete mode 100644 pkl-core/src/main/java/org/pkl/core/ast/expression/unary/StaticReadNode.java create mode 100644 pkl-core/src/main/java/org/pkl/core/ast/member/RegularMemberNode.java create mode 100644 pkl-core/src/main/java/org/pkl/core/ast/member/SharedMemberNode.java create mode 100644 pkl-core/src/main/java/org/pkl/core/runtime/VmObjectBuilder.java create mode 100644 pkl-core/src/test/files/LanguageSnippetTests/input-helper/basic/read/child/module2.pkl create mode 100644 pkl-core/src/test/files/LanguageSnippetTests/input-helper/basic/read/child/resource.txt create mode 100644 pkl-core/src/test/files/LanguageSnippetTests/input-helper/basic/read/module1.pkl create mode 100644 pkl-core/src/test/files/LanguageSnippetTests/input-helper/basic/read/resource.txt diff --git a/pkl-core/src/main/java/org/pkl/core/ast/MemberNode.java b/pkl-core/src/main/java/org/pkl/core/ast/MemberNode.java index a13d48e5..548b542c 100644 --- a/pkl-core/src/main/java/org/pkl/core/ast/MemberNode.java +++ b/pkl-core/src/main/java/org/pkl/core/ast/MemberNode.java @@ -20,35 +20,22 @@ import com.oracle.truffle.api.frame.VirtualFrame; import com.oracle.truffle.api.source.SourceSection; import java.util.function.Function; import org.pkl.core.ast.member.DefaultPropertyBodyNode; -import org.pkl.core.ast.member.Member; import org.pkl.core.runtime.VmExceptionBuilder; import org.pkl.core.runtime.VmLanguage; import org.pkl.core.runtime.VmUtils; import org.pkl.core.util.Nullable; public abstract class MemberNode extends PklRootNode { - protected final Member member; @Child protected ExpressionNode bodyNode; protected MemberNode( - @Nullable VmLanguage language, - FrameDescriptor descriptor, - Member member, - ExpressionNode bodyNode) { + @Nullable VmLanguage language, FrameDescriptor descriptor, ExpressionNode bodyNode) { super(language, descriptor); - this.member = member; this.bodyNode = bodyNode; } - @Override - public final SourceSection getSourceSection() { - return member.getSourceSection(); - } - - public final SourceSection getHeaderSection() { - return member.getHeaderSection(); - } + public abstract SourceSection getHeaderSection(); public final SourceSection getBodySection() { return bodyNode.getSourceSection(); @@ -58,11 +45,6 @@ public abstract class MemberNode extends PklRootNode { return bodyNode; } - @Override - public final String getName() { - return member.getQualifiedName(); - } - public final void replaceBody(Function replacer) { bodyNode = insert(replacer.apply(bodyNode)); } @@ -72,7 +54,7 @@ public abstract class MemberNode extends PklRootNode { } protected final VmExceptionBuilder exceptionBuilder() { - return new VmExceptionBuilder().withSourceSection(member.getHeaderSection()); + return new VmExceptionBuilder().withSourceSection(getHeaderSection()); } /** @@ -83,9 +65,9 @@ public abstract class MemberNode extends PklRootNode { * org.pkl.core.runtime.VmUtils#SKIP_TYPECHECK_MARKER}. IDEA: might be more appropriate to only * skip constraints check */ - protected final boolean shouldRunTypecheck(VirtualFrame frame) { - return frame.getArguments().length == 4 - && frame.getArguments()[3] == VmUtils.SKIP_TYPECHECK_MARKER; + protected final boolean shouldRunTypeCheck(VirtualFrame frame) { + return frame.getArguments().length != 4 + || frame.getArguments()[3] != VmUtils.SKIP_TYPECHECK_MARKER; } public boolean isUndefined() { diff --git a/pkl-core/src/main/java/org/pkl/core/ast/builder/AstBuilder.java b/pkl-core/src/main/java/org/pkl/core/ast/builder/AstBuilder.java index bf44cf44..c9a6b6e1 100644 --- a/pkl-core/src/main/java/org/pkl/core/ast/builder/AstBuilder.java +++ b/pkl-core/src/main/java/org/pkl/core/ast/builder/AstBuilder.java @@ -2218,8 +2218,7 @@ public final class AstBuilder extends AbstractAstBuilder { return ReadOrNullNodeGen.create(createSourceSection(ctx), moduleKey, visitExpr(exprCtx)); } assert tokenType == PklLexer.READ_GLOB; - return ReadGlobNodeGen.create( - language, createSourceSection(ctx), moduleKey, visitExpr(exprCtx)); + return ReadGlobNodeGen.create(createSourceSection(ctx), moduleKey, visitExpr(exprCtx)); } @Override @@ -2660,8 +2659,7 @@ public final class AstBuilder extends AbstractAstBuilder { } var resolvedUri = resolveImport(importUri, importUriCtx); if (isGlobImport) { - return new ImportGlobNode( - language, section, moduleInfo.getResolvedModuleKey(), resolvedUri, importUri); + return new ImportGlobNode(section, moduleInfo.getResolvedModuleKey(), resolvedUri, importUri); } return new ImportNode(language, section, moduleInfo.getResolvedModuleKey(), resolvedUri); } diff --git a/pkl-core/src/main/java/org/pkl/core/ast/expression/unary/AbstractImportNode.java b/pkl-core/src/main/java/org/pkl/core/ast/expression/unary/AbstractImportNode.java index 054382b5..ae3fc06f 100644 --- a/pkl-core/src/main/java/org/pkl/core/ast/expression/unary/AbstractImportNode.java +++ b/pkl-core/src/main/java/org/pkl/core/ast/expression/unary/AbstractImportNode.java @@ -18,16 +18,19 @@ package org.pkl.core.ast.expression.unary; import com.oracle.truffle.api.source.SourceSection; import java.net.URI; import org.pkl.core.ast.ExpressionNode; +import org.pkl.core.module.ResolvedModuleKey; public abstract class AbstractImportNode extends ExpressionNode { + protected final ResolvedModuleKey currentModule; protected final URI importUri; - AbstractImportNode(SourceSection sourceSection, URI importUri) { + AbstractImportNode(SourceSection sourceSection, ResolvedModuleKey currentModule, URI importUri) { super(sourceSection); + this.currentModule = currentModule; this.importUri = importUri; } - public URI getImportUri() { + public final URI getImportUri() { return importUri; } } diff --git a/pkl-core/src/main/java/org/pkl/core/ast/expression/unary/AbstractReadNode.java b/pkl-core/src/main/java/org/pkl/core/ast/expression/unary/AbstractReadNode.java index 0e5c891d..9d6a1ce4 100644 --- a/pkl-core/src/main/java/org/pkl/core/ast/expression/unary/AbstractReadNode.java +++ b/pkl-core/src/main/java/org/pkl/core/ast/expression/unary/AbstractReadNode.java @@ -30,30 +30,33 @@ import org.pkl.core.util.IoUtils; import org.pkl.core.util.Nullable; public abstract class AbstractReadNode extends UnaryExpressionNode { - private final ModuleKey moduleKey; + protected final ModuleKey currentModule; - protected AbstractReadNode(SourceSection sourceSection, ModuleKey moduleKey) { + protected AbstractReadNode(SourceSection sourceSection, ModuleKey currentModule) { super(sourceSection); - this.moduleKey = moduleKey; + this.currentModule = currentModule; } @TruffleBoundary - protected @Nullable Object doRead(String resourceUri, VmContext context, Node readNode) { - var resolvedUri = resolveResource(moduleKey, resourceUri); - return context.getResourceManager().read(resolvedUri, readNode).orElse(null); - } - - private URI resolveResource(ModuleKey moduleKey, String resourceUri) { - URI parsedUri; + protected final URI parseUri(String resourceUri) { try { - parsedUri = IoUtils.toUri(resourceUri); + return IoUtils.toUri(resourceUri); } catch (URISyntaxException e) { throw exceptionBuilder() .evalError("invalidResourceUri", resourceUri) .withHint(e.getReason()) .build(); } + } + @TruffleBoundary + protected final @Nullable Object doRead(String resourceUri, VmContext context, Node readNode) { + var resolvedUri = resolveResource(currentModule, resourceUri); + return context.getResourceManager().read(resolvedUri, readNode).orElse(null); + } + + private URI resolveResource(ModuleKey moduleKey, String resourceUri) { + var parsedUri = parseUri(resourceUri); var context = VmContext.get(this); URI resolvedUri; try { diff --git a/pkl-core/src/main/java/org/pkl/core/ast/expression/unary/ImportGlobMemberBodyNode.java b/pkl-core/src/main/java/org/pkl/core/ast/expression/unary/ImportGlobMemberBodyNode.java new file mode 100644 index 00000000..fde8c329 --- /dev/null +++ b/pkl-core/src/main/java/org/pkl/core/ast/expression/unary/ImportGlobMemberBodyNode.java @@ -0,0 +1,67 @@ +/** + * Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.pkl.core.ast.expression.unary; + +import com.oracle.truffle.api.CompilerDirectives.TruffleBoundary; +import com.oracle.truffle.api.frame.VirtualFrame; +import com.oracle.truffle.api.source.SourceSection; +import java.util.Map; +import org.pkl.core.SecurityManagerException; +import org.pkl.core.ast.ExpressionNode; +import org.pkl.core.http.HttpClientInitException; +import org.pkl.core.module.ResolvedModuleKey; +import org.pkl.core.packages.PackageLoadError; +import org.pkl.core.runtime.VmContext; +import org.pkl.core.runtime.VmLanguage; +import org.pkl.core.runtime.VmObjectLike; +import org.pkl.core.runtime.VmTyped; +import org.pkl.core.runtime.VmUtils; +import org.pkl.core.util.GlobResolver.ResolvedGlobElement; + +/** Used by {@link ReadGlobNode}. */ +public final class ImportGlobMemberBodyNode extends ExpressionNode { + private final VmLanguage language; + private final ResolvedModuleKey currentModule; + + public ImportGlobMemberBodyNode( + SourceSection sourceSection, VmLanguage language, ResolvedModuleKey currentModule) { + super(sourceSection); + this.language = language; + this.currentModule = currentModule; + } + + @Override + public Object executeGeneric(VirtualFrame frame) { + var mapping = VmUtils.getObjectReceiver(frame); + var path = (String) VmUtils.getMemberKey(frame); + return importModule(mapping, path); + } + + @TruffleBoundary + private VmTyped importModule(VmObjectLike mapping, String path) { + @SuppressWarnings("unchecked") + var globElements = (Map) mapping.getExtraStorage(); + var importUri = globElements.get(path).getUri(); + var context = VmContext.get(this); + try { + context.getSecurityManager().checkImportModule(currentModule.getUri(), importUri); + var moduleToImport = context.getModuleResolver().resolve(importUri, this); + return language.loadModule(moduleToImport, this); + } catch (SecurityManagerException | PackageLoadError | HttpClientInitException e) { + throw exceptionBuilder().withCause(e).build(); + } + } +} diff --git a/pkl-core/src/main/java/org/pkl/core/ast/expression/unary/ImportGlobNode.java b/pkl-core/src/main/java/org/pkl/core/ast/expression/unary/ImportGlobNode.java index 3d4cd2b6..b22b32aa 100644 --- a/pkl-core/src/main/java/org/pkl/core/ast/expression/unary/ImportGlobNode.java +++ b/pkl-core/src/main/java/org/pkl/core/ast/expression/unary/ImportGlobNode.java @@ -17,114 +17,93 @@ package org.pkl.core.ast.expression.unary; import com.oracle.truffle.api.CompilerDirectives; import com.oracle.truffle.api.CompilerDirectives.CompilationFinal; -import com.oracle.truffle.api.CompilerDirectives.TruffleBoundary; import com.oracle.truffle.api.frame.FrameDescriptor; import com.oracle.truffle.api.frame.VirtualFrame; import com.oracle.truffle.api.nodes.NodeInfo; import com.oracle.truffle.api.source.SourceSection; import java.io.IOException; import java.net.URI; -import java.util.List; -import org.graalvm.collections.EconomicMap; import org.pkl.core.SecurityManagerException; -import org.pkl.core.ast.VmModifier; -import org.pkl.core.ast.member.ObjectMember; -import org.pkl.core.ast.member.UntypedObjectMemberNode; +import org.pkl.core.ast.member.SharedMemberNode; import org.pkl.core.http.HttpClientInitException; import org.pkl.core.module.ResolvedModuleKey; import org.pkl.core.packages.PackageLoadError; -import org.pkl.core.runtime.BaseModule; import org.pkl.core.runtime.VmContext; import org.pkl.core.runtime.VmLanguage; import org.pkl.core.runtime.VmMapping; -import org.pkl.core.runtime.VmUtils; -import org.pkl.core.util.EconomicMaps; +import org.pkl.core.runtime.VmObjectBuilder; import org.pkl.core.util.GlobResolver; import org.pkl.core.util.GlobResolver.InvalidGlobPatternException; -import org.pkl.core.util.GlobResolver.ResolvedGlobElement; import org.pkl.core.util.LateInit; @NodeInfo(shortName = "import*") public class ImportGlobNode extends AbstractImportNode { - private final VmLanguage language; - - private final ResolvedModuleKey currentModule; - private final String globPattern; - - @CompilationFinal @LateInit private VmMapping importedMapping; + @Child @LateInit private SharedMemberNode memberNode; + @CompilationFinal @LateInit private VmMapping cachedResult; public ImportGlobNode( - VmLanguage language, SourceSection sourceSection, ResolvedModuleKey currentModule, URI importUri, String globPattern) { - super(sourceSection, importUri); - this.language = language; - this.currentModule = currentModule; + super(sourceSection, currentModule, importUri); this.globPattern = globPattern; } - @TruffleBoundary - private EconomicMap buildMembers( - FrameDescriptor frameDescriptor, List uris) { - var members = EconomicMaps.create(); - for (var entry : uris) { - var readNode = - new ImportNode( - language, VmUtils.unavailableSourceSection(), currentModule, entry.getUri()); - var member = - new ObjectMember( - VmUtils.unavailableSourceSection(), - VmUtils.unavailableSourceSection(), - VmModifier.ENTRY, - null, - ""); - var memberNode = new UntypedObjectMemberNode(language, frameDescriptor, member, readNode); - member.initMemberNode(memberNode); - EconomicMaps.put(members, entry.getPath(), member); + private SharedMemberNode getMemberNode() { + if (memberNode == null) { + CompilerDirectives.transferToInterpreterAndInvalidate(); + var language = VmLanguage.get(this); + memberNode = + new SharedMemberNode( + sourceSection, + sourceSection, + "", + language, + new FrameDescriptor(), + new ImportGlobMemberBodyNode(sourceSection, language, currentModule)); } - return members; + return memberNode; } @Override public Object executeGeneric(VirtualFrame frame) { - if (importedMapping == null) { - CompilerDirectives.transferToInterpreterAndInvalidate(); - var context = VmContext.get(this); - try { - var moduleKey = context.getModuleResolver().resolve(importUri); - var securityManager = VmContext.get(this).getSecurityManager(); - if (!moduleKey.isGlobbable()) { - throw exceptionBuilder() - .evalError("cannotGlobUri", importUri, importUri.getScheme()) - .build(); - } - var uris = - GlobResolver.resolveGlob( - securityManager, - moduleKey, - currentModule.getOriginal(), - currentModule.getUri(), - globPattern); - var members = buildMembers(frame.getFrameDescriptor(), uris); - importedMapping = - new VmMapping( - frame.materialize(), BaseModule.getMappingClass().getPrototype(), members); - } catch (IOException e) { - throw exceptionBuilder().evalError("ioErrorResolvingGlob", importUri).withCause(e).build(); - } catch (SecurityManagerException | HttpClientInitException e) { - throw exceptionBuilder().withCause(e).build(); - } catch (PackageLoadError e) { - throw exceptionBuilder().adhocEvalError(e.getMessage()).build(); - } catch (InvalidGlobPatternException e) { + if (cachedResult != null) return cachedResult; + + CompilerDirectives.transferToInterpreterAndInvalidate(); + var context = VmContext.get(this); + try { + var moduleKey = context.getModuleResolver().resolve(importUri); + if (!moduleKey.isGlobbable()) { throw exceptionBuilder() - .evalError("invalidGlobPattern", globPattern) - .withHint(e.getMessage()) + .evalError("cannotGlobUri", importUri, importUri.getScheme()) .build(); } + var resolvedElements = + GlobResolver.resolveGlob( + context.getSecurityManager(), + moduleKey, + currentModule.getOriginal(), + currentModule.getUri(), + globPattern); + var builder = new VmObjectBuilder(resolvedElements.size()); + for (var entry : resolvedElements.entrySet()) { + builder.addEntry(entry.getKey(), getMemberNode()); + } + cachedResult = builder.toMapping(resolvedElements); + return cachedResult; + } catch (IOException e) { + throw exceptionBuilder().evalError("ioErrorResolvingGlob", importUri).withCause(e).build(); + } catch (SecurityManagerException | HttpClientInitException e) { + throw exceptionBuilder().withCause(e).build(); + } catch (PackageLoadError e) { + throw exceptionBuilder().adhocEvalError(e.getMessage()).build(); + } catch (InvalidGlobPatternException e) { + throw exceptionBuilder() + .evalError("invalidGlobPattern", globPattern) + .withHint(e.getMessage()) + .build(); } - return importedMapping; } } diff --git a/pkl-core/src/main/java/org/pkl/core/ast/expression/unary/ImportNode.java b/pkl-core/src/main/java/org/pkl/core/ast/expression/unary/ImportNode.java index f8fdd4a8..94ce45b0 100644 --- a/pkl-core/src/main/java/org/pkl/core/ast/expression/unary/ImportNode.java +++ b/pkl-core/src/main/java/org/pkl/core/ast/expression/unary/ImportNode.java @@ -33,7 +33,6 @@ import org.pkl.core.util.LateInit; @NodeInfo(shortName = "import") public final class ImportNode extends AbstractImportNode { private final VmLanguage language; - private final ResolvedModuleKey currentModule; @CompilationFinal @LateInit private VmTyped importedModule; @@ -42,9 +41,8 @@ public final class ImportNode extends AbstractImportNode { SourceSection sourceSection, ResolvedModuleKey currentModule, URI importUri) { - super(sourceSection, importUri); + super(sourceSection, currentModule, importUri); this.language = language; - this.currentModule = currentModule; assert importUri.isAbsolute(); } diff --git a/pkl-core/src/main/java/org/pkl/core/ast/expression/unary/ReadGlobMemberBodyNode.java b/pkl-core/src/main/java/org/pkl/core/ast/expression/unary/ReadGlobMemberBodyNode.java new file mode 100644 index 00000000..838122ca --- /dev/null +++ b/pkl-core/src/main/java/org/pkl/core/ast/expression/unary/ReadGlobMemberBodyNode.java @@ -0,0 +1,52 @@ +/** + * Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.pkl.core.ast.expression.unary; + +import com.oracle.truffle.api.CompilerDirectives; +import com.oracle.truffle.api.frame.VirtualFrame; +import com.oracle.truffle.api.source.SourceSection; +import java.util.Map; +import org.pkl.core.ast.ExpressionNode; +import org.pkl.core.runtime.VmContext; +import org.pkl.core.runtime.VmObjectLike; +import org.pkl.core.runtime.VmUtils; +import org.pkl.core.util.GlobResolver.ResolvedGlobElement; + +/** Used by {@link ReadGlobNode}. */ +public class ReadGlobMemberBodyNode extends ExpressionNode { + public ReadGlobMemberBodyNode(SourceSection sourceSection) { + super(sourceSection); + } + + @Override + public Object executeGeneric(VirtualFrame frame) { + var mapping = VmUtils.getObjectReceiver(frame); + var path = (String) VmUtils.getMemberKey(frame); + return readResource(mapping, path); + } + + private Object readResource(VmObjectLike mapping, String path) { + @SuppressWarnings("unchecked") + var globElements = (Map) mapping.getExtraStorage(); + var resourceUri = VmUtils.getMapValue(globElements, path).getUri(); + var resource = VmContext.get(this).getResourceManager().read(resourceUri, this).orElse(null); + if (resource == null) { + CompilerDirectives.transferToInterpreter(); + throw exceptionBuilder().evalError("cannotFindResource", resourceUri).build(); + } + return resource; + } +} diff --git a/pkl-core/src/main/java/org/pkl/core/ast/expression/unary/ReadGlobNode.java b/pkl-core/src/main/java/org/pkl/core/ast/expression/unary/ReadGlobNode.java index 9161083a..1a58ed82 100644 --- a/pkl-core/src/main/java/org/pkl/core/ast/expression/unary/ReadGlobNode.java +++ b/pkl-core/src/main/java/org/pkl/core/ast/expression/unary/ReadGlobNode.java @@ -16,96 +16,91 @@ package org.pkl.core.ast.expression.unary; import com.oracle.truffle.api.CompilerDirectives; -import com.oracle.truffle.api.CompilerDirectives.CompilationFinal; import com.oracle.truffle.api.CompilerDirectives.TruffleBoundary; import com.oracle.truffle.api.dsl.Specialization; import com.oracle.truffle.api.frame.FrameDescriptor; -import com.oracle.truffle.api.frame.VirtualFrame; import com.oracle.truffle.api.nodes.NodeInfo; import com.oracle.truffle.api.source.SourceSection; -import java.net.URI; -import java.net.URISyntaxException; -import java.util.List; +import java.io.IOException; import org.graalvm.collections.EconomicMap; -import org.pkl.core.ast.VmModifier; -import org.pkl.core.ast.member.ObjectMember; -import org.pkl.core.ast.member.UntypedObjectMemberNode; +import org.pkl.core.SecurityManagerException; +import org.pkl.core.ast.member.SharedMemberNode; +import org.pkl.core.http.HttpClientInitException; import org.pkl.core.module.ModuleKey; -import org.pkl.core.runtime.BaseModule; import org.pkl.core.runtime.VmContext; import org.pkl.core.runtime.VmLanguage; import org.pkl.core.runtime.VmMapping; -import org.pkl.core.runtime.VmUtils; -import org.pkl.core.util.EconomicMaps; -import org.pkl.core.util.GlobResolver.ResolvedGlobElement; -import org.pkl.core.util.IoUtils; +import org.pkl.core.runtime.VmObjectBuilder; +import org.pkl.core.util.GlobResolver; +import org.pkl.core.util.GlobResolver.InvalidGlobPatternException; import org.pkl.core.util.LateInit; @NodeInfo(shortName = "read*") -public abstract class ReadGlobNode extends UnaryExpressionNode { - private final VmLanguage language; - private final ModuleKey currentModule; +public abstract class ReadGlobNode extends AbstractReadNode { + private final EconomicMap cachedResults = EconomicMap.create(); + @Child @LateInit private SharedMemberNode memberNode; - @CompilationFinal @LateInit VmMapping readResult; - - protected ReadGlobNode( - VmLanguage language, SourceSection sourceSection, ModuleKey currentModule) { - super(sourceSection); - this.currentModule = currentModule; - this.language = language; + protected ReadGlobNode(SourceSection sourceSection, ModuleKey currentModule) { + super(sourceSection, currentModule); } - @TruffleBoundary - private URI doResolveUri(String globExpression) { - try { - var globUri = IoUtils.toUri(globExpression); - var tripleDotImport = IoUtils.parseTripleDotPath(globUri); - if (tripleDotImport != null) { - throw exceptionBuilder().evalError("cannotGlobTripleDots").build(); - } - return globUri; - } catch (URISyntaxException e) { - throw exceptionBuilder() - .evalError("invalidResourceUri", globExpression) - .withHint(e.getReason()) - .build(); + private SharedMemberNode getMemberNode() { + if (memberNode == null) { + CompilerDirectives.transferToInterpreterAndInvalidate(); + var language = VmLanguage.get(this); + memberNode = + new SharedMemberNode( + sourceSection, + sourceSection, + "", + language, + new FrameDescriptor(), + new ReadGlobMemberBodyNode(sourceSection)); } - } - - @TruffleBoundary - private EconomicMap buildMembers( - FrameDescriptor frameDescriptor, List uris) { - var members = EconomicMaps.create(); - for (var entry : uris) { - var readNode = new StaticReadNode(entry.getUri()); - var member = - new ObjectMember( - VmUtils.unavailableSourceSection(), - VmUtils.unavailableSourceSection(), - VmModifier.ENTRY, - null, - ""); - var memberNode = new UntypedObjectMemberNode(language, frameDescriptor, member, readNode); - member.initMemberNode(memberNode); - EconomicMaps.put(members, entry.getPath(), member); - } - return members; + return memberNode; } @Specialization - public Object read(VirtualFrame frame, String globPattern) { - if (readResult == null) { - CompilerDirectives.transferToInterpreterAndInvalidate(); - var context = VmContext.get(this); - var resolvedUri = doResolveUri(globPattern); - var uris = - context - .getResourceManager() - .resolveGlob(resolvedUri, currentModule.getUri(), currentModule, this, globPattern); - var members = buildMembers(frame.getFrameDescriptor(), uris); - readResult = - new VmMapping(frame.materialize(), BaseModule.getMappingClass().getPrototype(), members); + @TruffleBoundary + public Object read(String globPattern) { + var cachedResult = cachedResults.get(globPattern); + if (cachedResult != null) return cachedResult; + + // use same check as for globbed imports (see AstBuilder) + if (globPattern.startsWith("...")) { + throw exceptionBuilder().evalError("cannotGlobTripleDots").build(); + } + var globUri = parseUri(globPattern); + var context = VmContext.get(this); + try { + var resolvedUri = currentModule.resolveUri(globUri); + var reader = context.getResourceManager().getReader(resolvedUri, this); + if (!reader.isGlobbable()) { + throw exceptionBuilder().evalError("cannotGlobUri", globUri, globUri.getScheme()).build(); + } + var resolvedElements = + GlobResolver.resolveGlob( + context.getSecurityManager(), + reader, + currentModule, + currentModule.getUri(), + globPattern); + var builder = new VmObjectBuilder(resolvedElements.size()); + for (var entry : resolvedElements.entrySet()) { + builder.addEntry(entry.getKey(), getMemberNode()); + } + cachedResult = builder.toMapping(resolvedElements); + cachedResults.put(globPattern, cachedResult); + return cachedResult; + } catch (IOException e) { + throw exceptionBuilder().evalError("ioErrorResolvingGlob", globPattern).withCause(e).build(); + } catch (SecurityManagerException | HttpClientInitException e) { + throw exceptionBuilder().withCause(e).build(); + } catch (InvalidGlobPatternException e) { + throw exceptionBuilder() + .evalError("invalidGlobPattern", globPattern) + .withHint(e.getMessage()) + .build(); } - return readResult; } } diff --git a/pkl-core/src/main/java/org/pkl/core/ast/expression/unary/StaticReadNode.java b/pkl-core/src/main/java/org/pkl/core/ast/expression/unary/StaticReadNode.java deleted file mode 100644 index eb98934d..00000000 --- a/pkl-core/src/main/java/org/pkl/core/ast/expression/unary/StaticReadNode.java +++ /dev/null @@ -1,50 +0,0 @@ -/** - * Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.pkl.core.ast.expression.unary; - -import com.oracle.truffle.api.CompilerDirectives; -import com.oracle.truffle.api.CompilerDirectives.CompilationFinal; -import com.oracle.truffle.api.frame.VirtualFrame; -import java.net.URI; -import org.pkl.core.runtime.VmContext; -import org.pkl.core.runtime.VmUtils; -import org.pkl.core.util.LateInit; - -/** Used by {@link ReadGlobNode}. */ -public class StaticReadNode extends UnaryExpressionNode { - private final URI resourceUri; - - @CompilationFinal @LateInit private Object readResult; - - public StaticReadNode(URI resourceUri) { - super(VmUtils.unavailableSourceSection()); - assert resourceUri.isAbsolute(); - this.resourceUri = resourceUri; - } - - @Override - public Object executeGeneric(VirtualFrame frame) { - if (readResult == null) { - CompilerDirectives.transferToInterpreterAndInvalidate(); - var context = VmContext.get(this); - readResult = context.getResourceManager().read(resourceUri, this).orElse(null); - if (readResult == null) { - throw exceptionBuilder().evalError("cannotFindResource", resourceUri).build(); - } - } - return readResult; - } -} diff --git a/pkl-core/src/main/java/org/pkl/core/ast/member/FunctionNode.java b/pkl-core/src/main/java/org/pkl/core/ast/member/FunctionNode.java index 31c7d879..ef42cda0 100644 --- a/pkl-core/src/main/java/org/pkl/core/ast/member/FunctionNode.java +++ b/pkl-core/src/main/java/org/pkl/core/ast/member/FunctionNode.java @@ -27,7 +27,6 @@ import org.pkl.core.PClass; import org.pkl.core.PType; import org.pkl.core.TypeParameter; import org.pkl.core.ast.ExpressionNode; -import org.pkl.core.ast.MemberNode; import org.pkl.core.ast.VmModifier; import org.pkl.core.ast.type.TypeNode; import org.pkl.core.ast.type.VmTypeMismatchException; @@ -36,7 +35,7 @@ import org.pkl.core.util.CollectionUtils; import org.pkl.core.util.Nullable; import org.pkl.core.util.Pair; -public final class FunctionNode extends MemberNode { +public final class FunctionNode extends RegularMemberNode { // Every function (and property) call passes two implicit arguments at positions // frame.getArguments()[0] and [1]: // - the receiver (target) of the call, of type Object (see VmUtils.getReceiver()) diff --git a/pkl-core/src/main/java/org/pkl/core/ast/member/LocalTypedPropertyNode.java b/pkl-core/src/main/java/org/pkl/core/ast/member/LocalTypedPropertyNode.java index 39a564e7..87784d7e 100644 --- a/pkl-core/src/main/java/org/pkl/core/ast/member/LocalTypedPropertyNode.java +++ b/pkl-core/src/main/java/org/pkl/core/ast/member/LocalTypedPropertyNode.java @@ -19,7 +19,6 @@ import com.oracle.truffle.api.CompilerDirectives; import com.oracle.truffle.api.frame.FrameDescriptor; import com.oracle.truffle.api.frame.VirtualFrame; import org.pkl.core.ast.ExpressionNode; -import org.pkl.core.ast.MemberNode; import org.pkl.core.ast.type.TypeNode; import org.pkl.core.ast.type.UnresolvedTypeNode; import org.pkl.core.ast.type.VmTypeMismatchException; @@ -28,7 +27,7 @@ import org.pkl.core.runtime.VmLanguage; import org.pkl.core.util.LateInit; import org.pkl.core.util.Nullable; -public final class LocalTypedPropertyNode extends MemberNode { +public final class LocalTypedPropertyNode extends RegularMemberNode { private final VmLanguage language; @Child private UnresolvedTypeNode unresolvedTypeNode; @Child @LateInit private TypeNode typeNode; diff --git a/pkl-core/src/main/java/org/pkl/core/ast/member/ObjectMethodNode.java b/pkl-core/src/main/java/org/pkl/core/ast/member/ObjectMethodNode.java index 5bae1241..35a9740e 100644 --- a/pkl-core/src/main/java/org/pkl/core/ast/member/ObjectMethodNode.java +++ b/pkl-core/src/main/java/org/pkl/core/ast/member/ObjectMethodNode.java @@ -21,14 +21,13 @@ import com.oracle.truffle.api.CompilerDirectives.CompilationFinal; import com.oracle.truffle.api.frame.FrameDescriptor; import com.oracle.truffle.api.frame.VirtualFrame; import org.pkl.core.ast.ExpressionNode; -import org.pkl.core.ast.MemberNode; import org.pkl.core.ast.type.TypeNode; import org.pkl.core.ast.type.UnresolvedTypeNode; import org.pkl.core.runtime.*; import org.pkl.core.util.LateInit; import org.pkl.core.util.Nullable; -public final class ObjectMethodNode extends MemberNode { +public final class ObjectMethodNode extends RegularMemberNode { private final VmLanguage language; private final int parameterCount; @Children private final @Nullable UnresolvedTypeNode[] unresolvedParameterTypeNodes; diff --git a/pkl-core/src/main/java/org/pkl/core/ast/member/RegularMemberNode.java b/pkl-core/src/main/java/org/pkl/core/ast/member/RegularMemberNode.java new file mode 100644 index 00000000..6c3a122c --- /dev/null +++ b/pkl-core/src/main/java/org/pkl/core/ast/member/RegularMemberNode.java @@ -0,0 +1,52 @@ +/** + * Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.pkl.core.ast.member; + +import com.oracle.truffle.api.frame.FrameDescriptor; +import com.oracle.truffle.api.source.SourceSection; +import org.pkl.core.ast.ExpressionNode; +import org.pkl.core.ast.MemberNode; +import org.pkl.core.runtime.VmLanguage; +import org.pkl.core.util.Nullable; + +/** A {@code MemberNode} that belongs to a single {@link Member}. */ +public abstract class RegularMemberNode extends MemberNode { + protected final Member member; + + protected RegularMemberNode( + @Nullable VmLanguage language, + FrameDescriptor descriptor, + Member member, + ExpressionNode bodyNode) { + + super(language, descriptor, bodyNode); + this.member = member; + } + + @Override + public final SourceSection getSourceSection() { + return member.getSourceSection(); + } + + public final SourceSection getHeaderSection() { + return member.getHeaderSection(); + } + + @Override + public final String getName() { + return member.getQualifiedName(); + } +} diff --git a/pkl-core/src/main/java/org/pkl/core/ast/member/SharedMemberNode.java b/pkl-core/src/main/java/org/pkl/core/ast/member/SharedMemberNode.java new file mode 100644 index 00000000..827e9e04 --- /dev/null +++ b/pkl-core/src/main/java/org/pkl/core/ast/member/SharedMemberNode.java @@ -0,0 +1,64 @@ +/** + * Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.pkl.core.ast.member; + +import com.oracle.truffle.api.frame.FrameDescriptor; +import com.oracle.truffle.api.frame.VirtualFrame; +import com.oracle.truffle.api.source.SourceSection; +import org.pkl.core.ast.ExpressionNode; +import org.pkl.core.ast.MemberNode; +import org.pkl.core.runtime.VmLanguage; +import org.pkl.core.util.Nullable; + +/** A {@code MemberNode} that is shared between multiple {@linkplain Member members}. */ +public class SharedMemberNode extends MemberNode { + private final SourceSection sourceSection; + private final SourceSection headerSection; + private final @Nullable String qualifiedName; + + public SharedMemberNode( + SourceSection sourceSection, + SourceSection headerSection, + @Nullable String qualifiedName, + @Nullable VmLanguage language, + FrameDescriptor descriptor, + ExpressionNode bodyNode) { + + super(language, descriptor, bodyNode); + this.sourceSection = sourceSection; + this.headerSection = headerSection; + this.qualifiedName = qualifiedName; + } + + @Override + public SourceSection getSourceSection() { + return sourceSection; + } + + public SourceSection getHeaderSection() { + return headerSection; + } + + @Override + public @Nullable String getName() { + return qualifiedName; + } + + @Override + public Object execute(VirtualFrame frame) { + return executeBody(frame); + } +} diff --git a/pkl-core/src/main/java/org/pkl/core/ast/member/TypeCheckedPropertyNode.java b/pkl-core/src/main/java/org/pkl/core/ast/member/TypeCheckedPropertyNode.java index c3014d90..0b0038fa 100644 --- a/pkl-core/src/main/java/org/pkl/core/ast/member/TypeCheckedPropertyNode.java +++ b/pkl-core/src/main/java/org/pkl/core/ast/member/TypeCheckedPropertyNode.java @@ -23,13 +23,12 @@ import com.oracle.truffle.api.frame.VirtualFrame; import com.oracle.truffle.api.nodes.DirectCallNode; import com.oracle.truffle.api.nodes.IndirectCallNode; import org.pkl.core.ast.ExpressionNode; -import org.pkl.core.ast.MemberNode; import org.pkl.core.ast.expression.primary.GetOwnerNode; import org.pkl.core.runtime.*; import org.pkl.core.util.Nullable; /** A property definition that does not have a type annotation but should be type-checked. */ -public abstract class TypeCheckedPropertyNode extends MemberNode { +public abstract class TypeCheckedPropertyNode extends RegularMemberNode { @Child @Executed protected ExpressionNode ownerNode = new GetOwnerNode(); protected TypeCheckedPropertyNode( @@ -55,7 +54,7 @@ public abstract class TypeCheckedPropertyNode extends MemberNode { var result = executeBody(frame); // TODO: propagate SUPER_CALL_MARKER to disable constraint (but not type) check - if (callNode != null && !shouldRunTypecheck(frame)) { + if (callNode != null && shouldRunTypeCheck(frame)) { callNode.call(VmUtils.getReceiverOrNull(frame), property.getOwner(), result); } @@ -68,7 +67,7 @@ public abstract class TypeCheckedPropertyNode extends MemberNode { var result = executeBody(frame); - if (!shouldRunTypecheck(frame)) { + if (shouldRunTypeCheck(frame)) { var property = getProperty(owner.getVmClass()); var typeAnnNode = property.getTypeNode(); if (typeAnnNode != null) { diff --git a/pkl-core/src/main/java/org/pkl/core/ast/member/TypedPropertyNode.java b/pkl-core/src/main/java/org/pkl/core/ast/member/TypedPropertyNode.java index 495844a2..1a139da7 100644 --- a/pkl-core/src/main/java/org/pkl/core/ast/member/TypedPropertyNode.java +++ b/pkl-core/src/main/java/org/pkl/core/ast/member/TypedPropertyNode.java @@ -20,12 +20,11 @@ import com.oracle.truffle.api.frame.FrameDescriptor; import com.oracle.truffle.api.frame.VirtualFrame; import com.oracle.truffle.api.nodes.DirectCallNode; import org.pkl.core.ast.ExpressionNode; -import org.pkl.core.ast.MemberNode; import org.pkl.core.runtime.VmLanguage; import org.pkl.core.runtime.VmUtils; /** A property definition that has a type annotation. */ -public final class TypedPropertyNode extends MemberNode { +public final class TypedPropertyNode extends RegularMemberNode { @Child private DirectCallNode typeCheckCallNode; @TruffleBoundary @@ -46,7 +45,7 @@ public final class TypedPropertyNode extends MemberNode { @Override public Object execute(VirtualFrame frame) { var propertyValue = executeBody(frame); - if (!shouldRunTypecheck(frame)) { + if (shouldRunTypeCheck(frame)) { typeCheckCallNode.call(VmUtils.getReceiver(frame), VmUtils.getOwner(frame), propertyValue); } return propertyValue; diff --git a/pkl-core/src/main/java/org/pkl/core/ast/member/UntypedObjectMemberNode.java b/pkl-core/src/main/java/org/pkl/core/ast/member/UntypedObjectMemberNode.java index 6aeed879..ea599eab 100644 --- a/pkl-core/src/main/java/org/pkl/core/ast/member/UntypedObjectMemberNode.java +++ b/pkl-core/src/main/java/org/pkl/core/ast/member/UntypedObjectMemberNode.java @@ -18,11 +18,10 @@ package org.pkl.core.ast.member; import com.oracle.truffle.api.frame.FrameDescriptor; import com.oracle.truffle.api.frame.VirtualFrame; import org.pkl.core.ast.ExpressionNode; -import org.pkl.core.ast.MemberNode; import org.pkl.core.runtime.VmLanguage; import org.pkl.core.util.Nullable; -public final class UntypedObjectMemberNode extends MemberNode { +public final class UntypedObjectMemberNode extends RegularMemberNode { public UntypedObjectMemberNode( @Nullable VmLanguage language, FrameDescriptor descriptor, diff --git a/pkl-core/src/main/java/org/pkl/core/runtime/ModuleCache.java b/pkl-core/src/main/java/org/pkl/core/runtime/ModuleCache.java index 5558246b..f0add3f9 100644 --- a/pkl-core/src/main/java/org/pkl/core/runtime/ModuleCache.java +++ b/pkl-core/src/main/java/org/pkl/core/runtime/ModuleCache.java @@ -30,7 +30,6 @@ import java.util.stream.Collectors; import org.pkl.core.Release; import org.pkl.core.SecurityManager; import org.pkl.core.SecurityManagerException; -import org.pkl.core.ast.expression.unary.ImportNode; import org.pkl.core.module.ModuleKey; import org.pkl.core.module.ModuleKeys; import org.pkl.core.module.ResolvedModuleKey; @@ -77,7 +76,7 @@ public final class ModuleCache { ModuleResolver moduleResolver, Supplier moduleInstantiator, ModuleInitializer moduleInitializer, - @Nullable ImportNode importNode) { + @Nullable Node importNode) { if (ModuleKeys.isStdLibModule(moduleKey)) { var moduleName = moduleKey.getUri().getSchemeSpecificPart(); @@ -162,7 +161,7 @@ public final class ModuleCache { ModuleResolver moduleResolver, Supplier moduleInstantiator, ModuleInitializer moduleInitializer, - @Nullable ImportNode importNode) { + @Nullable Node importNode) { VmTyped module = moduleInstantiator.get(); @@ -189,7 +188,7 @@ public final class ModuleCache { } private ResolvedModuleKey resolve( - ModuleKey module, SecurityManager securityManager, @Nullable ImportNode importNode) { + ModuleKey module, SecurityManager securityManager, @Nullable Node importNode) { try { return module.resolve(securityManager); } catch (SecurityManagerException | PackageLoadError e) { diff --git a/pkl-core/src/main/java/org/pkl/core/runtime/ModuleInfo.java b/pkl-core/src/main/java/org/pkl/core/runtime/ModuleInfo.java index f452e03e..ce1c4be9 100644 --- a/pkl-core/src/main/java/org/pkl/core/runtime/ModuleInfo.java +++ b/pkl-core/src/main/java/org/pkl/core/runtime/ModuleInfo.java @@ -22,8 +22,7 @@ import org.pkl.core.ModuleSchema; import org.pkl.core.PClass; import org.pkl.core.TypeAlias; import org.pkl.core.ast.MemberNode; -import org.pkl.core.ast.expression.unary.ImportGlobNode; -import org.pkl.core.ast.expression.unary.ImportNode; +import org.pkl.core.ast.expression.unary.AbstractImportNode; import org.pkl.core.module.ModuleKey; import org.pkl.core.module.ResolvedModuleKey; import org.pkl.core.util.EconomicMaps; @@ -132,11 +131,8 @@ public final class ModuleInfo { if (propertyDef.isImport()) { MemberNode memberNode = propertyDef.getMemberNode(); assert memberNode != null; // import is never a constant - var importNode = memberNode.getBodyNode(); - var importUri = - importNode instanceof ImportNode - ? ((ImportNode) importNode).getImportUri() - : ((ImportGlobNode) importNode).getImportUri(); + var importNode = (AbstractImportNode) memberNode.getBodyNode(); + var importUri = importNode.getImportUri(); imports.put(propertyDef.getName().toString(), importUri); continue; } diff --git a/pkl-core/src/main/java/org/pkl/core/runtime/ResourceManager.java b/pkl-core/src/main/java/org/pkl/core/runtime/ResourceManager.java index 62b7c262..7c5e64b8 100644 --- a/pkl-core/src/main/java/org/pkl/core/runtime/ResourceManager.java +++ b/pkl-core/src/main/java/org/pkl/core/runtime/ResourceManager.java @@ -22,20 +22,15 @@ import java.net.URI; import java.net.URISyntaxException; import java.util.Collection; import java.util.HashMap; -import java.util.List; import java.util.Map; import java.util.Optional; import org.pkl.core.SecurityManager; import org.pkl.core.SecurityManagerException; import org.pkl.core.http.HttpClientInitException; -import org.pkl.core.module.ModuleKey; import org.pkl.core.packages.PackageLoadError; import org.pkl.core.resource.Resource; import org.pkl.core.resource.ResourceReader; import org.pkl.core.stdlib.VmObjectFactory; -import org.pkl.core.util.GlobResolver; -import org.pkl.core.util.GlobResolver.InvalidGlobPatternException; -import org.pkl.core.util.GlobResolver.ResolvedGlobElement; public final class ResourceManager { private final Map resourceReaders = new HashMap<>(); @@ -45,8 +40,6 @@ public final class ResourceManager { // cache resources indefinitely to make resource reads deterministic private final Map> resources = new HashMap<>(); - private final Map> globExpressions = new HashMap<>(); - public ResourceManager(SecurityManager securityManager, Collection readers) { this.securityManager = securityManager; @@ -61,61 +54,16 @@ public final class ResourceManager { .addProperty("base64", Resource::getBase64); } - /** - * Resolves the glob URI into a set of URIs. - * - *

The glob URI must be absolute. For example: {@code "file:///foo/bar/*.pkl"}. - */ @TruffleBoundary - public List resolveGlob( - URI globUri, - URI enclosingUri, - ModuleKey enclosingModuleKey, - Node readNode, - String globExpression) { - return globExpressions.computeIfAbsent( - globUri.normalize(), - uri -> { - var scheme = uri.getScheme(); - URI resolvedUri; - try { - resolvedUri = enclosingModuleKey.resolveUri(globUri); - } catch (SecurityManagerException | IOException e) { - throw new VmExceptionBuilder().withLocation(readNode).withCause(e).build(); - } - try { - var reader = resourceReaders.get(resolvedUri.getScheme()); - if (reader == null) { - throw new VmExceptionBuilder() - .withLocation(readNode) - .evalError("noResourceReaderRegistered", scheme) - .build(); - } - if (!reader.isGlobbable()) { - throw new VmExceptionBuilder() - .evalError("cannotGlobUri", uri, scheme) - .withLocation(readNode) - .build(); - } - var securityManager = VmContext.get(readNode).getSecurityManager(); - return GlobResolver.resolveGlob( - securityManager, reader, enclosingModuleKey, enclosingUri, globExpression); - } catch (InvalidGlobPatternException e) { - throw new VmExceptionBuilder() - .evalError("invalidGlobPattern", globExpression) - .withHint(e.getMessage()) - .withLocation(readNode) - .build(); - } catch (SecurityManagerException | HttpClientInitException e) { - throw new VmExceptionBuilder().withCause(e).withLocation(readNode).build(); - } catch (IOException e) { - throw new VmExceptionBuilder() - .evalError("ioErrorResolvingGlob", globExpression) - .withCause(e) - .withLocation(readNode) - .build(); - } - }); + public ResourceReader getReader(URI resourceUri, Node readNode) { + var reader = resourceReaders.get(resourceUri.getScheme()); + if (reader == null) { + throw new VmExceptionBuilder() + .withLocation(readNode) + .evalError("noResourceReaderRegistered", resourceUri.getScheme()) + .build(); + } + return reader; } @TruffleBoundary @@ -129,13 +77,7 @@ public final class ResourceManager { throw new VmExceptionBuilder().withCause(e).withLocation(readNode).build(); } - var reader = resourceReaders.get(uri.getScheme()); - if (reader == null) { - throw new VmExceptionBuilder() - .withLocation(readNode) - .evalError("noResourceReaderRegistered", resourceUri.getScheme()) - .build(); - } + var reader = getReader(resourceUri, readNode); Optional resource; try { diff --git a/pkl-core/src/main/java/org/pkl/core/runtime/VmLanguage.java b/pkl-core/src/main/java/org/pkl/core/runtime/VmLanguage.java index 1a1ce9a2..bbf21b14 100644 --- a/pkl-core/src/main/java/org/pkl/core/runtime/VmLanguage.java +++ b/pkl-core/src/main/java/org/pkl/core/runtime/VmLanguage.java @@ -22,7 +22,6 @@ import com.oracle.truffle.api.TruffleLanguage.ContextPolicy; import com.oracle.truffle.api.nodes.Node; import com.oracle.truffle.api.source.Source; import org.pkl.core.ast.builder.AstBuilder; -import org.pkl.core.ast.expression.unary.ImportNode; import org.pkl.core.module.ModuleKey; import org.pkl.core.module.ResolvedModuleKey; import org.pkl.core.parser.LexParseException; @@ -73,7 +72,7 @@ public final class VmLanguage extends TruffleLanguage { } @TruffleBoundary - public VmTyped loadModule(ModuleKey moduleKey, ImportNode importNode) { + public VmTyped loadModule(ModuleKey moduleKey, @Nullable Node importNode) { var context = VmContext.get(null); return context diff --git a/pkl-core/src/main/java/org/pkl/core/runtime/VmList.java b/pkl-core/src/main/java/org/pkl/core/runtime/VmList.java index 8c397798..9cfbe2c4 100644 --- a/pkl-core/src/main/java/org/pkl/core/runtime/VmList.java +++ b/pkl-core/src/main/java/org/pkl/core/runtime/VmList.java @@ -382,6 +382,20 @@ public final class VmList extends VmCollection { return VmSet.create(rrbt); } + @TruffleBoundary + public VmListing toListing() { + var builder = new VmObjectBuilder(rrbt.size()); + for (var elem : rrbt) builder.addElement(elem); + return builder.toListing(); + } + + @TruffleBoundary + public VmDynamic toDynamic() { + var builder = new VmObjectBuilder(rrbt.size()); + for (var elem : rrbt) builder.addElement(elem); + return builder.toDynamic(); + } + @Override @TruffleBoundary public void force(boolean allowUndefinedValues) { diff --git a/pkl-core/src/main/java/org/pkl/core/runtime/VmMap.java b/pkl-core/src/main/java/org/pkl/core/runtime/VmMap.java index e2fa42e7..dfe3abfd 100644 --- a/pkl-core/src/main/java/org/pkl/core/runtime/VmMap.java +++ b/pkl-core/src/main/java/org/pkl/core/runtime/VmMap.java @@ -225,6 +225,28 @@ public final class VmMap extends VmValue implements Iterable export() { diff --git a/pkl-core/src/main/java/org/pkl/core/runtime/VmObjectBuilder.java b/pkl-core/src/main/java/org/pkl/core/runtime/VmObjectBuilder.java new file mode 100644 index 00000000..867d33fd --- /dev/null +++ b/pkl-core/src/main/java/org/pkl/core/runtime/VmObjectBuilder.java @@ -0,0 +1,86 @@ +/** + * Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.pkl.core.runtime; + +import org.graalvm.collections.EconomicMap; +import org.pkl.core.ast.VmModifier; +import org.pkl.core.ast.member.ObjectMember; +import org.pkl.core.ast.member.SharedMemberNode; +import org.pkl.core.util.EconomicMaps; + +/** A builder for {@link VmObject}s whose {@link ObjectMember}s are determined at run time. */ +public final class VmObjectBuilder { + private final EconomicMap members; + private int elementCount = 0; + + public VmObjectBuilder(int initialSize) { + members = EconomicMaps.create(initialSize); + } + + public VmObjectBuilder addProperty(Identifier name, Object value) { + EconomicMaps.put(members, name, VmUtils.createSyntheticObjectProperty(name, "", value)); + return this; + } + + public VmObjectBuilder addElement(Object value) { + EconomicMaps.put( + members, (long) elementCount++, VmUtils.createSyntheticObjectElement("", value)); + return this; + } + + public VmObjectBuilder addEntry(Object key, Object value) { + EconomicMaps.put(members, key, VmUtils.createSyntheticObjectEntry("", value)); + return this; + } + + public VmObjectBuilder addEntry(Object key, SharedMemberNode valueNode) { + var entry = + new ObjectMember( + valueNode.getSourceSection(), valueNode.getHeaderSection(), VmModifier.ENTRY, null, ""); + entry.initMemberNode(valueNode); + EconomicMaps.put(members, key, entry); + return this; + } + + public VmListing toListing() { + return new VmListing( + VmUtils.createEmptyMaterializedFrame(), + BaseModule.getListingClass().getPrototype(), + members, + elementCount); + } + + public VmMapping toMapping() { + return new VmMapping( + VmUtils.createEmptyMaterializedFrame(), + BaseModule.getMappingClass().getPrototype(), + members); + } + + public VmMapping toMapping(Object extraStorage) { + var result = toMapping(); + result.setExtraStorage(extraStorage); + return result; + } + + public VmDynamic toDynamic() { + return new VmDynamic( + VmUtils.createEmptyMaterializedFrame(), + BaseModule.getDynamicClass().getPrototype(), + members, + elementCount); + } +} diff --git a/pkl-core/src/main/java/org/pkl/core/runtime/VmSet.java b/pkl-core/src/main/java/org/pkl/core/runtime/VmSet.java index e00b7b32..89d79373 100644 --- a/pkl-core/src/main/java/org/pkl/core/runtime/VmSet.java +++ b/pkl-core/src/main/java/org/pkl/core/runtime/VmSet.java @@ -282,6 +282,20 @@ public final class VmSet extends VmCollection { return this; } + @TruffleBoundary + public VmListing toListing() { + var builder = new VmObjectBuilder(elementOrder.size()); + for (var elem : elementOrder) builder.addElement(elem); + return builder.toListing(); + } + + @TruffleBoundary + public VmDynamic toDynamic() { + var builder = new VmObjectBuilder(elementOrder.size()); + for (var elem : elementOrder) builder.addElement(elem); + return builder.toDynamic(); + } + @Override @TruffleBoundary public void force(boolean allowUndefinedValues) { diff --git a/pkl-core/src/main/java/org/pkl/core/runtime/VmUtils.java b/pkl-core/src/main/java/org/pkl/core/runtime/VmUtils.java index a367198a..6fcbc305 100644 --- a/pkl-core/src/main/java/org/pkl/core/runtime/VmUtils.java +++ b/pkl-core/src/main/java/org/pkl/core/runtime/VmUtils.java @@ -62,7 +62,7 @@ import org.pkl.core.util.EconomicMaps; import org.pkl.core.util.Nullable; public final class VmUtils { - /** See {@link MemberNode#shouldRunTypecheck(VirtualFrame)}. */ + /** See {@link MemberNode#shouldRunTypeCheck(VirtualFrame)}. */ @SuppressWarnings("JavadocReference") public static final Object SKIP_TYPECHECK_MARKER = new Object(); @@ -844,4 +844,9 @@ public final class VmUtils { public static int findAuxiliarySlot(VirtualFrame frame, Object identifier) { return frame.getFrameDescriptor().getAuxiliarySlots().getOrDefault(identifier, -1); } + + @TruffleBoundary + public static V getMapValue(Map map, K key) { + return map.get(key); + } } diff --git a/pkl-core/src/main/java/org/pkl/core/stdlib/base/ListNodes.java b/pkl-core/src/main/java/org/pkl/core/stdlib/base/ListNodes.java index 11b1a199..1eab8ddd 100644 --- a/pkl-core/src/main/java/org/pkl/core/stdlib/base/ListNodes.java +++ b/pkl-core/src/main/java/org/pkl/core/stdlib/base/ListNodes.java @@ -19,18 +19,15 @@ import com.oracle.truffle.api.CompilerDirectives; import com.oracle.truffle.api.CompilerDirectives.TruffleBoundary; import com.oracle.truffle.api.dsl.Specialization; import com.oracle.truffle.api.nodes.LoopNode; -import org.graalvm.collections.EconomicMap; import org.pkl.core.ast.expression.binary.*; import org.pkl.core.ast.internal.IsInstanceOfNode; import org.pkl.core.ast.internal.IsInstanceOfNodeGen; import org.pkl.core.ast.lambda.*; -import org.pkl.core.ast.member.ObjectMember; import org.pkl.core.runtime.*; import org.pkl.core.stdlib.*; import org.pkl.core.stdlib.base.CollectionNodes.CompareByNode; import org.pkl.core.stdlib.base.CollectionNodes.CompareNode; import org.pkl.core.stdlib.base.CollectionNodes.CompareWithNode; -import org.pkl.core.util.EconomicMaps; import org.pkl.core.util.EconomicSets; // duplication between ListNodes and SetNodes is "intentional" @@ -1295,11 +1292,7 @@ public final class ListNodes { @Specialization @TruffleBoundary protected VmListing eval(VmList self) { - return new VmListing( - VmUtils.createEmptyMaterializedFrame(), - BaseModule.getListingClass().getPrototype(), - toObjectMembers(self), - self.getLength()); + return self.toListing(); } } @@ -1307,22 +1300,7 @@ public final class ListNodes { @Specialization @TruffleBoundary protected VmDynamic eval(VmList self) { - return new VmDynamic( - VmUtils.createEmptyMaterializedFrame(), - BaseModule.getDynamicClass().getPrototype(), - toObjectMembers(self), - self.getLength()); + return self.toDynamic(); } } - - private static EconomicMap toObjectMembers(VmList self) { - var result = EconomicMaps.create(self.getLength()); - - for (long idx = 0; idx < self.getLength(); idx++) { - EconomicMaps.put( - result, idx, VmUtils.createSyntheticObjectElement(String.valueOf(idx), self.get(idx))); - } - - return result; - } } diff --git a/pkl-core/src/main/java/org/pkl/core/stdlib/base/MapNodes.java b/pkl-core/src/main/java/org/pkl/core/stdlib/base/MapNodes.java index 5bbb129e..f722a77b 100644 --- a/pkl-core/src/main/java/org/pkl/core/stdlib/base/MapNodes.java +++ b/pkl-core/src/main/java/org/pkl/core/stdlib/base/MapNodes.java @@ -21,13 +21,11 @@ import com.oracle.truffle.api.dsl.Specialization; import com.oracle.truffle.api.nodes.LoopNode; import java.util.Map; import org.pkl.core.ast.lambda.*; -import org.pkl.core.ast.member.ObjectMember; import org.pkl.core.runtime.*; import org.pkl.core.stdlib.ExternalMethod0Node; import org.pkl.core.stdlib.ExternalMethod1Node; import org.pkl.core.stdlib.ExternalMethod2Node; import org.pkl.core.stdlib.ExternalPropertyNode; -import org.pkl.core.util.EconomicMaps; import org.pkl.core.util.MutableReference; public final class MapNodes { @@ -238,28 +236,7 @@ public final class MapNodes { public abstract static class toDynamic extends ExternalMethod0Node { @Specialization protected VmDynamic eval(VmMap self) { - var members = EconomicMaps.create(self.getLength()); - - for (var entry : self) { - var key = VmUtils.getKey(entry); - - if (key instanceof String string) { - var name = Identifier.get(string); - EconomicMaps.put( - members, - name, - VmUtils.createSyntheticObjectProperty(name, "", VmUtils.getValue(entry))); - } else { - EconomicMaps.put( - members, key, VmUtils.createSyntheticObjectEntry("", VmUtils.getValue(entry))); - } - } - - return new VmDynamic( - VmUtils.createEmptyMaterializedFrame(), - BaseModule.getDynamicClass().getPrototype(), - members, - 0); + return self.toDynamic(); } } @@ -287,19 +264,7 @@ public final class MapNodes { public abstract static class toMapping extends ExternalMethod0Node { @Specialization protected VmMapping eval(VmMap self) { - var members = EconomicMaps.create(self.getLength()); - - for (var entry : self) { - EconomicMaps.put( - members, - VmUtils.getKey(entry), - VmUtils.createSyntheticObjectEntry("", VmUtils.getValue(entry))); - } - - return new VmMapping( - VmUtils.createEmptyMaterializedFrame(), - BaseModule.getMappingClass().getPrototype(), - members); + return self.toMapping(); } } } diff --git a/pkl-core/src/main/java/org/pkl/core/stdlib/base/SetNodes.java b/pkl-core/src/main/java/org/pkl/core/stdlib/base/SetNodes.java index 95d3d86b..0be7f1c1 100644 --- a/pkl-core/src/main/java/org/pkl/core/stdlib/base/SetNodes.java +++ b/pkl-core/src/main/java/org/pkl/core/stdlib/base/SetNodes.java @@ -19,7 +19,6 @@ import com.oracle.truffle.api.CompilerDirectives; import com.oracle.truffle.api.CompilerDirectives.TruffleBoundary; import com.oracle.truffle.api.dsl.Specialization; import com.oracle.truffle.api.nodes.LoopNode; -import org.graalvm.collections.EconomicMap; import org.pkl.core.ast.expression.binary.GreaterThanNode; import org.pkl.core.ast.expression.binary.GreaterThanNodeGen; import org.pkl.core.ast.expression.binary.LessThanNode; @@ -27,13 +26,11 @@ import org.pkl.core.ast.expression.binary.LessThanNodeGen; import org.pkl.core.ast.internal.IsInstanceOfNode; import org.pkl.core.ast.internal.IsInstanceOfNodeGen; import org.pkl.core.ast.lambda.*; -import org.pkl.core.ast.member.ObjectMember; import org.pkl.core.runtime.*; import org.pkl.core.stdlib.*; import org.pkl.core.stdlib.base.CollectionNodes.CompareByNode; import org.pkl.core.stdlib.base.CollectionNodes.CompareNode; import org.pkl.core.stdlib.base.CollectionNodes.CompareWithNode; -import org.pkl.core.util.EconomicMaps; // duplication between ListNodes and SetNodes is "intentional" // (sharing nodes between VmCollection subtypes results in @@ -1052,11 +1049,7 @@ public final class SetNodes { @Specialization @TruffleBoundary protected VmListing eval(VmSet self) { - return new VmListing( - VmUtils.createEmptyMaterializedFrame(), - BaseModule.getListingClass().getPrototype(), - toObjectMembers(self), - self.getLength()); + return self.toListing(); } } @@ -1064,11 +1057,7 @@ public final class SetNodes { @Specialization @TruffleBoundary protected VmDynamic eval(VmSet self) { - return new VmDynamic( - VmUtils.createEmptyMaterializedFrame(), - BaseModule.getDynamicClass().getPrototype(), - toObjectMembers(self), - self.getLength()); + return self.toDynamic(); } } @@ -1110,17 +1099,4 @@ public final class SetNodes { return builder.build(); } } - - private static EconomicMap toObjectMembers(VmSet self) { - var result = EconomicMaps.create(self.getLength()); - - long idx = 0; - for (var element : self) { - EconomicMaps.put( - result, idx, VmUtils.createSyntheticObjectElement(String.valueOf(idx), element)); - idx += 1; - } - - return result; - } } diff --git a/pkl-core/src/main/java/org/pkl/core/util/GlobResolver.java b/pkl-core/src/main/java/org/pkl/core/util/GlobResolver.java index 52b9b4b0..d64712b9 100644 --- a/pkl-core/src/main/java/org/pkl/core/util/GlobResolver.java +++ b/pkl-core/src/main/java/org/pkl/core/util/GlobResolver.java @@ -22,6 +22,7 @@ import java.net.URISyntaxException; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.WeakHashMap; @@ -250,7 +251,7 @@ public final class GlobResolver { ReaderBase reader, URI globUri, Pattern pattern, - ArrayList result) + Map result) throws IOException, SecurityManagerException { var elements = reader.listElements(securityManager, globUri); for (var elem : sorted(elements)) { @@ -261,7 +262,8 @@ public final class GlobResolver { throw new IllegalArgumentException(e.getMessage(), e); } if (pattern.matcher(resolvedUri.toString()).matches()) { - result.add(new ResolvedGlobElement(resolvedUri.toString(), resolvedUri, false)); + var path = resolvedUri.toString(); + result.put(path, new ResolvedGlobElement(path, resolvedUri, false)); } } } @@ -372,7 +374,7 @@ public final class GlobResolver { URI baseUri, String expandedGlobPatternSoFar, boolean hasAbsoluteGlob, - List result, + Map result, MutableLong listElementCallCount) throws IOException, SecurityManagerException, InvalidGlobPatternException { var isLeaf = idx == globPatternParts.length - 1; @@ -384,7 +386,7 @@ public final class GlobResolver { if (reader.hasElement(securityManager, newBaseUri)) { // Note: isDirectory is not meaningful here. Setting it to false is a way to skip setting // it. - result.add(new ResolvedGlobElement(newPath, newBaseUri, false)); + result.put(newPath, new ResolvedGlobElement(newPath, newBaseUri, false)); } } else { var newBaseUri = IoUtils.resolve(reader, baseUri, patternPart + "/"); @@ -414,7 +416,7 @@ public final class GlobResolver { listElementCallCount); for (var element : matchedElements) { if (isLeaf) { - result.add(element); + result.put(element.getPath(), element); } else if (element.isDirectory()) { resolveHierarchicalGlob( securityManager, @@ -465,7 +467,7 @@ public final class GlobResolver { *

Each pair is the expanded form of the glob pattern, paired with its resolved absolute URI. */ @TruffleBoundary - public static List resolveGlob( + public static Map resolveGlob( SecurityManager securityManager, ReaderBase reader, ModuleKey enclosingModuleKey, @@ -473,7 +475,7 @@ public final class GlobResolver { String globPattern) throws IOException, SecurityManagerException, InvalidGlobPatternException { - var result = new ArrayList(); + var result = new LinkedHashMap(); var hasAbsoluteGlob = globPattern.matches("\\w+:.*"); if (reader.hasHierarchicalUris()) { @@ -484,7 +486,7 @@ public final class GlobResolver { if (globParts.length == 0) { var resolvedUri = IoUtils.resolve(reader, enclosingUri, globPattern); if (reader.hasElement(securityManager, resolvedUri)) { - result.add(new ResolvedGlobElement(globPattern, resolvedUri, true)); + result.put(globPattern, new ResolvedGlobElement(globPattern, resolvedUri, true)); } return result; } diff --git a/pkl-core/src/test/files/LanguageSnippetTests/input-helper/basic/read/child/module2.pkl b/pkl-core/src/test/files/LanguageSnippetTests/input-helper/basic/read/child/module2.pkl new file mode 100644 index 00000000..141cb953 --- /dev/null +++ b/pkl-core/src/test/files/LanguageSnippetTests/input-helper/basic/read/child/module2.pkl @@ -0,0 +1,2 @@ +normalRead = read("resource.txt") +globbedRead = read*("*.txt") diff --git a/pkl-core/src/test/files/LanguageSnippetTests/input-helper/basic/read/child/resource.txt b/pkl-core/src/test/files/LanguageSnippetTests/input-helper/basic/read/child/resource.txt new file mode 100644 index 00000000..cdf56820 --- /dev/null +++ b/pkl-core/src/test/files/LanguageSnippetTests/input-helper/basic/read/child/resource.txt @@ -0,0 +1 @@ +child resource diff --git a/pkl-core/src/test/files/LanguageSnippetTests/input-helper/basic/read/module1.pkl b/pkl-core/src/test/files/LanguageSnippetTests/input-helper/basic/read/module1.pkl new file mode 100644 index 00000000..141cb953 --- /dev/null +++ b/pkl-core/src/test/files/LanguageSnippetTests/input-helper/basic/read/module1.pkl @@ -0,0 +1,2 @@ +normalRead = read("resource.txt") +globbedRead = read*("*.txt") diff --git a/pkl-core/src/test/files/LanguageSnippetTests/input-helper/basic/read/resource.txt b/pkl-core/src/test/files/LanguageSnippetTests/input-helper/basic/read/resource.txt new file mode 100644 index 00000000..91e75c67 --- /dev/null +++ b/pkl-core/src/test/files/LanguageSnippetTests/input-helper/basic/read/resource.txt @@ -0,0 +1 @@ +resource diff --git a/pkl-core/src/test/files/LanguageSnippetTests/input/basic/read.pkl b/pkl-core/src/test/files/LanguageSnippetTests/input/basic/read.pkl index dd0da33f..8a691813 100644 --- a/pkl-core/src/test/files/LanguageSnippetTests/input/basic/read.pkl +++ b/pkl-core/src/test/files/LanguageSnippetTests/input/basic/read.pkl @@ -22,14 +22,32 @@ examples { } ["read file"] { - read("read.pkl") + read("globtest/file1.txt") module.catch(() -> read("other.txt")) - read?("read.pkl") + read?("globtest/file1.txt") read?("other.txt") } + + ["read triple-dot file"] { + read(".../input-helper/basic/read/resource.txt") + read?(".../input-helper/basic/read/resource.txt") + } ["read non-allowed resource"] { module.catch(() -> read("forbidden:resource")) module.catch(() -> read?("forbidden:resource")) } + + ["use read expression with non-constant resource URI"] { + local function doRead(uri) = read(uri) + doRead("globtest/file1.txt") + doRead("globtest/file2.txt") + } + + ["read different resources with same relative resource URI"] { + local module1 = import(".../input-helper/basic/read/module1.pkl") + local module2 = import(".../input-helper/basic/read/child/module2.pkl") + module1.normalRead + module2.normalRead + } } diff --git a/pkl-core/src/test/files/LanguageSnippetTests/input/basic/readGlob.pkl b/pkl-core/src/test/files/LanguageSnippetTests/input/basic/readGlob.pkl index a7b2b18e..8f7b8adb 100644 --- a/pkl-core/src/test/files/LanguageSnippetTests/input/basic/readGlob.pkl +++ b/pkl-core/src/test/files/LanguageSnippetTests/input/basic/readGlob.pkl @@ -30,4 +30,18 @@ examples { ["package:"] { read*("package://localhost:0/birds@0.5.0#/**.pkl") } + + ["use read expression with non-constant glob pattern"] { + local function doRead(pattern) = read*(pattern) + doRead("globtest/file*.txt") + doRead("globtest/file1.txt") + doRead("globtest/file2.txt") + } + + ["read different resources with same glob pattern"] { + local module1 = import(".../input-helper/basic/read/module1.pkl") + local module2 = import(".../input-helper/basic/read/child/module2.pkl") + module1.globbedRead + module2.globbedRead + } } diff --git a/pkl-core/src/test/files/LanguageSnippetTests/output/basic/read.pcf b/pkl-core/src/test/files/LanguageSnippetTests/output/basic/read.pcf index 590d34bf..733f2e77 100644 --- a/pkl-core/src/test/files/LanguageSnippetTests/output/basic/read.pcf +++ b/pkl-core/src/test/files/LanguageSnippetTests/output/basic/read.pcf @@ -17,94 +17,80 @@ examples { } ["read file"] { new { - uri = "file:///$snippetsDir/input/basic/read.pkl" + uri = "file:///$snippetsDir/input/basic/globtest/file1.txt" text = """ - amends "../snippetTest.pkl" - - examples { - ["read env variable"] { - read("env:NAME1") - read("env:NAME2") - module.catch(() -> read("env:OTHER")) - - read?("env:NAME1") - read?("env:NAME2") - read?("env:OTHER") - } - - ["read external property"] { - read("prop:name1") - read("prop:name2") - module.catch(() -> read("prop:other")) - - read?("prop:name1") - read?("prop:name2") - read?("prop:other") - } - - ["read file"] { - read("read.pkl") - module.catch(() -> read("other.txt")) - read?("read.pkl") - read?("other.txt") - } - - ["read non-allowed resource"] { - module.catch(() -> read("forbidden:resource")) - module.catch(() -> read?("forbidden:resource")) - } - } + file1 """ - base64 = "YW1lbmRzICIuLi9zbmlwcGV0VGVzdC5wa2wiCgpleGFtcGxlcyB7CiAgWyJyZWFkIGVudiB2YXJpYWJsZSJdIHsKICAgIHJlYWQoImVudjpOQU1FMSIpCiAgICByZWFkKCJlbnY6TkFNRTIiKQogICAgbW9kdWxlLmNhdGNoKCgpIC0+IHJlYWQoImVudjpPVEhFUiIpKQoKICAgIHJlYWQ/KCJlbnY6TkFNRTEiKQogICAgcmVhZD8oImVudjpOQU1FMiIpCiAgICByZWFkPygiZW52Ok9USEVSIikKICB9CgogIFsicmVhZCBleHRlcm5hbCBwcm9wZXJ0eSJdIHsKICAgIHJlYWQoInByb3A6bmFtZTEiKQogICAgcmVhZCgicHJvcDpuYW1lMiIpCiAgICBtb2R1bGUuY2F0Y2goKCkgLT4gcmVhZCgicHJvcDpvdGhlciIpKQoKICAgIHJlYWQ/KCJwcm9wOm5hbWUxIikKICAgIHJlYWQ/KCJwcm9wOm5hbWUyIikKICAgIHJlYWQ/KCJwcm9wOm90aGVyIikKICB9CgogIFsicmVhZCBmaWxlIl0gewogICAgcmVhZCgicmVhZC5wa2wiKQogICAgbW9kdWxlLmNhdGNoKCgpIC0+IHJlYWQoIm90aGVyLnR4dCIpKQogICAgcmVhZD8oInJlYWQucGtsIikKICAgIHJlYWQ/KCJvdGhlci50eHQiKQogIH0KCiAgWyJyZWFkIG5vbi1hbGxvd2VkIHJlc291cmNlIl0gewogICAgbW9kdWxlLmNhdGNoKCgpIC0+IHJlYWQoImZvcmJpZGRlbjpyZXNvdXJjZSIpKQogICAgbW9kdWxlLmNhdGNoKCgpIC0+IHJlYWQ/KCJmb3JiaWRkZW46cmVzb3VyY2UiKSkKICB9Cn0K" + base64 = "ZmlsZTEK" } "Cannot find resource `other.txt`." new { - uri = "file:///$snippetsDir/input/basic/read.pkl" + uri = "file:///$snippetsDir/input/basic/globtest/file1.txt" text = """ - amends "../snippetTest.pkl" - - examples { - ["read env variable"] { - read("env:NAME1") - read("env:NAME2") - module.catch(() -> read("env:OTHER")) - - read?("env:NAME1") - read?("env:NAME2") - read?("env:OTHER") - } - - ["read external property"] { - read("prop:name1") - read("prop:name2") - module.catch(() -> read("prop:other")) - - read?("prop:name1") - read?("prop:name2") - read?("prop:other") - } - - ["read file"] { - read("read.pkl") - module.catch(() -> read("other.txt")) - read?("read.pkl") - read?("other.txt") - } - - ["read non-allowed resource"] { - module.catch(() -> read("forbidden:resource")) - module.catch(() -> read?("forbidden:resource")) - } - } + file1 """ - base64 = "YW1lbmRzICIuLi9zbmlwcGV0VGVzdC5wa2wiCgpleGFtcGxlcyB7CiAgWyJyZWFkIGVudiB2YXJpYWJsZSJdIHsKICAgIHJlYWQoImVudjpOQU1FMSIpCiAgICByZWFkKCJlbnY6TkFNRTIiKQogICAgbW9kdWxlLmNhdGNoKCgpIC0+IHJlYWQoImVudjpPVEhFUiIpKQoKICAgIHJlYWQ/KCJlbnY6TkFNRTEiKQogICAgcmVhZD8oImVudjpOQU1FMiIpCiAgICByZWFkPygiZW52Ok9USEVSIikKICB9CgogIFsicmVhZCBleHRlcm5hbCBwcm9wZXJ0eSJdIHsKICAgIHJlYWQoInByb3A6bmFtZTEiKQogICAgcmVhZCgicHJvcDpuYW1lMiIpCiAgICBtb2R1bGUuY2F0Y2goKCkgLT4gcmVhZCgicHJvcDpvdGhlciIpKQoKICAgIHJlYWQ/KCJwcm9wOm5hbWUxIikKICAgIHJlYWQ/KCJwcm9wOm5hbWUyIikKICAgIHJlYWQ/KCJwcm9wOm90aGVyIikKICB9CgogIFsicmVhZCBmaWxlIl0gewogICAgcmVhZCgicmVhZC5wa2wiKQogICAgbW9kdWxlLmNhdGNoKCgpIC0+IHJlYWQoIm90aGVyLnR4dCIpKQogICAgcmVhZD8oInJlYWQucGtsIikKICAgIHJlYWQ/KCJvdGhlci50eHQiKQogIH0KCiAgWyJyZWFkIG5vbi1hbGxvd2VkIHJlc291cmNlIl0gewogICAgbW9kdWxlLmNhdGNoKCgpIC0+IHJlYWQoImZvcmJpZGRlbjpyZXNvdXJjZSIpKQogICAgbW9kdWxlLmNhdGNoKCgpIC0+IHJlYWQ/KCJmb3JiaWRkZW46cmVzb3VyY2UiKSkKICB9Cn0K" + base64 = "ZmlsZTEK" } null } + ["read triple-dot file"] { + new { + uri = "file:///$snippetsDir/input-helper/basic/read/resource.txt" + text = """ + resource + + """ + base64 = "cmVzb3VyY2UK" + } + new { + uri = "file:///$snippetsDir/input-helper/basic/read/resource.txt" + text = """ + resource + + """ + base64 = "cmVzb3VyY2UK" + } + } ["read non-allowed resource"] { "Refusing to read resource `forbidden:resource` because it does not match any entry in the resource allowlist (`--allowed-resources`)." "Refusing to read resource `forbidden:resource` because it does not match any entry in the resource allowlist (`--allowed-resources`)." } + ["use read expression with non-constant resource URI"] { + new { + uri = "file:///$snippetsDir/input/basic/globtest/file1.txt" + text = """ + file1 + + """ + base64 = "ZmlsZTEK" + } + new { + uri = "file:///$snippetsDir/input/basic/globtest/file2.txt" + text = """ + file2 + + """ + base64 = "ZmlsZTIK" + } + } + ["read different resources with same relative resource URI"] { + new { + uri = "file:///$snippetsDir/input-helper/basic/read/resource.txt" + text = """ + resource + + """ + base64 = "cmVzb3VyY2UK" + } + new { + uri = "file:///$snippetsDir/input-helper/basic/read/child/resource.txt" + text = """ + child resource + + """ + base64 = "Y2hpbGQgcmVzb3VyY2UK" + } + } } diff --git a/pkl-core/src/test/files/LanguageSnippetTests/output/basic/readGlob.pcf b/pkl-core/src/test/files/LanguageSnippetTests/output/basic/readGlob.pcf index 3df2be34..6fedc704 100644 --- a/pkl-core/src/test/files/LanguageSnippetTests/output/basic/readGlob.pcf +++ b/pkl-core/src/test/files/LanguageSnippetTests/output/basic/readGlob.pcf @@ -176,4 +176,66 @@ examples { } } } + ["use read expression with non-constant glob pattern"] { + new { + ["globtest/file1.txt"] { + uri = "file:///$snippetsDir/input/basic/globtest/file1.txt" + text = """ + file1 + + """ + base64 = "ZmlsZTEK" + } + ["globtest/file2.txt"] { + uri = "file:///$snippetsDir/input/basic/globtest/file2.txt" + text = """ + file2 + + """ + base64 = "ZmlsZTIK" + } + } + new { + ["globtest/file1.txt"] { + uri = "file:///$snippetsDir/input/basic/globtest/file1.txt" + text = """ + file1 + + """ + base64 = "ZmlsZTEK" + } + } + new { + ["globtest/file2.txt"] { + uri = "file:///$snippetsDir/input/basic/globtest/file2.txt" + text = """ + file2 + + """ + base64 = "ZmlsZTIK" + } + } + } + ["read different resources with same glob pattern"] { + new { + ["resource.txt"] { + uri = "file:///$snippetsDir/input-helper/basic/read/resource.txt" + text = """ + resource + + """ + base64 = "cmVzb3VyY2UK" + } + } + new { + ["resource.txt"] { + uri = "file:///$snippetsDir/input-helper/basic/read/child/resource.txt" + text = """ + child resource + + """ + base64 = "Y2hpbGQgcmVzb3VyY2UK" + } + } + } }