From 939e844c032ab9b2acb972ff770978cc611674f2 Mon Sep 17 00:00:00 2001 From: Islon Scherer Date: Fri, 23 Oct 2026 15:51:22 +0200 Subject: [PATCH] Add pkl:syntax stdlib module (#1821) --- .../kotlin/org/pkl/cli/CliFormatterCommand.kt | 6 +- pkl-core/pkl-core.gradle.kts | 1 + .../org/pkl/core/ast/builder/AstBuilder.java | 8 +- .../java/org/pkl/core/runtime/Identifier.java | 12 + .../org/pkl/core/runtime/MirrorFactories.java | 2 +- .../org/pkl/core/runtime/ModuleCache.java | 2 + .../org/pkl/core/runtime/ReflectModule.java | 8 - .../org/pkl/core/runtime/SyntaxModule.java | 61 +++ .../java/org/pkl/core/runtime/VmUtils.java | 32 +- .../org/pkl/core/stdlib/VmObjectFactory.java | 9 + .../syntax/FormattingRendererNodes.java | 55 +++ .../org/pkl/core/stdlib/syntax/NodeNodes.java | 410 +++++++++++++++++ .../pkl/core/stdlib/syntax/ParserNodes.java | 146 ++++++ .../pkl/core/stdlib/syntax/SyntaxNodes.java | 305 +++++++++++++ .../pkl/core/stdlib/syntax/package-info.java | 4 + .../org/pkl/core/errorMessages.properties | 18 + .../input-helper/syntax/spans/sample.pkl | 5 + .../input/errors/syntaxParseExpression.pkl | 5 + .../input/errors/syntaxParseModule.pkl | 12 + .../errors/syntaxParseModuleLexerError.pkl | 8 + .../input/syntax/query.pkl | 257 +++++++++++ .../input/syntax/render.pkl | 413 +++++++++++++++++ .../input/syntax/spans.pkl | 80 ++++ .../input/syntax/traversal.pkl | 101 +++++ .../output/errors/cannotFindStdLibModule.err | 1 + .../output/errors/syntaxParseExpression.err | 16 + .../output/errors/syntaxParseModule.err | 16 + .../errors/syntaxParseModuleLexerError.err | 16 + .../output/syntax/query.pcf | 115 +++++ .../output/syntax/render.pcf | 146 ++++++ .../output/syntax/spans.pcf | 91 ++++ .../output/syntax/traversal.pcf | 30 ++ .../main/java/org/pkl/formatter/Builder.java | 159 ++++--- .../java/org/pkl/formatter/Formatter.java | 37 +- .../kotlin/org/pkl/formatter/FormatterTest.kt | 19 + .../org/pkl/parser/BaseParserVisitor.java | 4 +- .../java/org/pkl/parser/GenericParser.java | 4 + .../org/pkl/parser/GenericParserImpl.java | 165 +++++-- .../main/java/org/pkl/parser/ParserImpl.java | 4 +- .../java/org/pkl/parser/ParserVisitor.java | 2 +- .../main/java/org/pkl/parser/syntax/Type.java | 6 +- .../pkl/parser/syntax/generic/FullSpan.java | 5 +- .../org/pkl/parser/syntax/generic/Node.java | 4 + .../pkl/parser/syntax/generic/NodeType.java | 75 +++- .../org/pkl/parser/GenericSexpRenderer.kt | 45 +- .../kotlin/org/pkl/parser/SexpRenderer.kt | 6 +- stdlib/reflect.pkl | 15 +- stdlib/syntax.pkl | 422 ++++++++++++++++++ 48 files changed, 3149 insertions(+), 214 deletions(-) create mode 100644 pkl-core/src/main/java/org/pkl/core/runtime/SyntaxModule.java create mode 100644 pkl-core/src/main/java/org/pkl/core/stdlib/syntax/FormattingRendererNodes.java create mode 100644 pkl-core/src/main/java/org/pkl/core/stdlib/syntax/NodeNodes.java create mode 100644 pkl-core/src/main/java/org/pkl/core/stdlib/syntax/ParserNodes.java create mode 100644 pkl-core/src/main/java/org/pkl/core/stdlib/syntax/SyntaxNodes.java create mode 100644 pkl-core/src/main/java/org/pkl/core/stdlib/syntax/package-info.java create mode 100644 pkl-core/src/test/files/LanguageSnippetTests/input-helper/syntax/spans/sample.pkl create mode 100644 pkl-core/src/test/files/LanguageSnippetTests/input/errors/syntaxParseExpression.pkl create mode 100644 pkl-core/src/test/files/LanguageSnippetTests/input/errors/syntaxParseModule.pkl create mode 100644 pkl-core/src/test/files/LanguageSnippetTests/input/errors/syntaxParseModuleLexerError.pkl create mode 100644 pkl-core/src/test/files/LanguageSnippetTests/input/syntax/query.pkl create mode 100644 pkl-core/src/test/files/LanguageSnippetTests/input/syntax/render.pkl create mode 100644 pkl-core/src/test/files/LanguageSnippetTests/input/syntax/spans.pkl create mode 100644 pkl-core/src/test/files/LanguageSnippetTests/input/syntax/traversal.pkl create mode 100644 pkl-core/src/test/files/LanguageSnippetTests/output/errors/syntaxParseExpression.err create mode 100644 pkl-core/src/test/files/LanguageSnippetTests/output/errors/syntaxParseModule.err create mode 100644 pkl-core/src/test/files/LanguageSnippetTests/output/errors/syntaxParseModuleLexerError.err create mode 100644 pkl-core/src/test/files/LanguageSnippetTests/output/syntax/query.pcf create mode 100644 pkl-core/src/test/files/LanguageSnippetTests/output/syntax/render.pcf create mode 100644 pkl-core/src/test/files/LanguageSnippetTests/output/syntax/spans.pcf create mode 100644 pkl-core/src/test/files/LanguageSnippetTests/output/syntax/traversal.pcf create mode 100644 stdlib/syntax.pkl diff --git a/pkl-cli/src/main/kotlin/org/pkl/cli/CliFormatterCommand.kt b/pkl-cli/src/main/kotlin/org/pkl/cli/CliFormatterCommand.kt index 8fccd10a9..0058cef81 100644 --- a/pkl-cli/src/main/kotlin/org/pkl/cli/CliFormatterCommand.kt +++ b/pkl-cli/src/main/kotlin/org/pkl/cli/CliFormatterCommand.kt @@ -33,7 +33,6 @@ import org.pkl.core.util.IoUtils import org.pkl.formatter.Formatter import org.pkl.formatter.GrammarVersion import org.pkl.parser.GenericParserError -import org.pkl.parser.ParserError class CliFormatterCommand @JvmOverloads @@ -128,10 +127,7 @@ constructor( consoleWriter.write(formatted) consoleWriter.flush() } - } catch (pe: ParserError) { // thrown by the lexer - writeErrLine("Could not format `$pathStr`: $pe") - status.update(ERROR) - } catch (pe: GenericParserError) { // thrown by the generic parser + } catch (pe: GenericParserError) { writeErrLine("Could not format `$pathStr`: $pe") status.update(ERROR) } catch (e: IOException) { diff --git a/pkl-core/pkl-core.gradle.kts b/pkl-core/pkl-core.gradle.kts index cf9fd332d..b9784263f 100644 --- a/pkl-core/pkl-core.gradle.kts +++ b/pkl-core/pkl-core.gradle.kts @@ -61,6 +61,7 @@ dependencies { compileOnly(projects.pklExecutor) implementation(projects.pklParser) + implementation(projects.pklFormatter) implementation(libs.msgpack) implementation(libs.truffleApi) implementation(libs.graalSdk) diff --git a/pkl-core/src/main/java/org/pkl/core/ast/builder/AstBuilder.java b/pkl-core/src/main/java/org/pkl/core/ast/builder/AstBuilder.java index 7be6b32c5..661702fcd 100644 --- a/pkl-core/src/main/java/org/pkl/core/ast/builder/AstBuilder.java +++ b/pkl-core/src/main/java/org/pkl/core/ast/builder/AstBuilder.java @@ -279,7 +279,7 @@ import org.pkl.parser.syntax.Type.ModuleType; import org.pkl.parser.syntax.Type.NothingType; import org.pkl.parser.syntax.Type.NullableType; import org.pkl.parser.syntax.Type.ParenthesizedType; -import org.pkl.parser.syntax.Type.StringConstantType; +import org.pkl.parser.syntax.Type.StringLiteralType; import org.pkl.parser.syntax.Type.ThisType; import org.pkl.parser.syntax.Type.UnionType; import org.pkl.parser.syntax.Type.UnknownType; @@ -474,7 +474,7 @@ public class AstBuilder extends AbstractAstBuilder { } @Override - public UnresolvedTypeNode visitStringConstantType(StringConstantType type) { + public UnresolvedTypeNode visitStringLiteralType(StringLiteralType type) { return new UnresolvedTypeNode.StringLiteral( createSourceSection(type), type.getStr().getString()); } @@ -565,7 +565,7 @@ public class AstBuilder extends AbstractAstBuilder { boolean isUnionOfStringLiterals = true; for (var typ : elementTypes) { - if (!(typ instanceof StringConstantType)) { + if (!(typ instanceof StringLiteralType)) { isUnionOfStringLiterals = false; break; } @@ -576,7 +576,7 @@ public class AstBuilder extends AbstractAstBuilder { return new UnresolvedTypeNode.UnionOfStringLiterals( createSourceSection(type), defaultIndex, - elementTypes.stream().map(it -> ((StringConstantType) it).getStr().getString()).toList()); + elementTypes.stream().map(it -> ((StringLiteralType) it).getStr().getString()).toList()); } var elements = new UnresolvedTypeNode[elementTypes.size()]; diff --git a/pkl-core/src/main/java/org/pkl/core/runtime/Identifier.java b/pkl-core/src/main/java/org/pkl/core/runtime/Identifier.java index d311b2b70..1816c6dea 100644 --- a/pkl-core/src/main/java/org/pkl/core/runtime/Identifier.java +++ b/pkl-core/src/main/java/org/pkl/core/runtime/Identifier.java @@ -181,6 +181,18 @@ public final class Identifier implements Comparable { // common in lambdas etc public static final Identifier IT = get("it"); + // members of pkl.syntax#Node, pkl.syntax#Span and pkl.syntax#SourceLocation + public static final Identifier TYPE = get("type"); + public static final Identifier CHILDREN = get("children"); + public static final Identifier SPAN = get("span"); + public static final Identifier START = get("start"); + public static final Identifier END = get("end"); + public static final Identifier LINE = get("line"); + public static final Identifier COLUMN = get("column"); + + // members of pkl.syntax#FormattingRenderer + public static final Identifier GRAMMAR_VERSION = get("grammarVersion"); + private final String name; private Identifier(String name) { diff --git a/pkl-core/src/main/java/org/pkl/core/runtime/MirrorFactories.java b/pkl-core/src/main/java/org/pkl/core/runtime/MirrorFactories.java index cee20dc53..8b322b26e 100644 --- a/pkl-core/src/main/java/org/pkl/core/runtime/MirrorFactories.java +++ b/pkl-core/src/main/java/org/pkl/core/runtime/MirrorFactories.java @@ -107,7 +107,7 @@ public final class MirrorFactories { new VmObjectFactory<>(ReflectModule::getNothingTypeClass); public static final VmObjectFactory sourceLocationFactory = - new VmObjectFactory<>(ReflectModule::getSourceLocationClass); + new VmObjectFactory<>(SyntaxModule::getSourceLocationClass); static { moduleFactory diff --git a/pkl-core/src/main/java/org/pkl/core/runtime/ModuleCache.java b/pkl-core/src/main/java/org/pkl/core/runtime/ModuleCache.java index 48a1b165a..a88133adc 100644 --- a/pkl-core/src/main/java/org/pkl/core/runtime/ModuleCache.java +++ b/pkl-core/src/main/java/org/pkl/core/runtime/ModuleCache.java @@ -117,6 +117,8 @@ public final class ModuleCache { case "settings": // always needed if ~/.config/pkl/settings.pkl is present return SettingsModule.getModule(); + case "syntax": + return SyntaxModule.getModule(); case "test": return TestModule.getModule(); case "xml": diff --git a/pkl-core/src/main/java/org/pkl/core/runtime/ReflectModule.java b/pkl-core/src/main/java/org/pkl/core/runtime/ReflectModule.java index d95d816f3..2e01831eb 100644 --- a/pkl-core/src/main/java/org/pkl/core/runtime/ReflectModule.java +++ b/pkl-core/src/main/java/org/pkl/core/runtime/ReflectModule.java @@ -104,10 +104,6 @@ public final class ReflectModule extends StdLibModule { return TypeVariableClass.instance; } - public static VmClass getSourceLocationClass() { - return SourceLocationClass.instance; - } - private static final class ModuleClass { static final VmClass instance = loadClass("Module"); } @@ -176,10 +172,6 @@ public final class ReflectModule extends StdLibModule { static final VmClass instance = loadClass("TypeVariable"); } - private static final class SourceLocationClass { - static final VmClass instance = loadClass("SourceLocation"); - } - @TruffleBoundary private static VmClass loadClass(String className) { var theModule = getModule(); diff --git a/pkl-core/src/main/java/org/pkl/core/runtime/SyntaxModule.java b/pkl-core/src/main/java/org/pkl/core/runtime/SyntaxModule.java new file mode 100644 index 000000000..e4303770b --- /dev/null +++ b/pkl-core/src/main/java/org/pkl/core/runtime/SyntaxModule.java @@ -0,0 +1,61 @@ +/* + * Copyright © 2025-2026 Apple Inc. and the Pkl project authors. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.pkl.core.runtime; + +import com.oracle.truffle.api.CompilerDirectives.TruffleBoundary; +import java.net.URI; + +public final class SyntaxModule extends StdLibModule { + private static final VmTyped instance = VmUtils.createEmptyModule(); + + static { + loadModule(URI.create("pkl:syntax"), instance); + } + + public static VmTyped getModule() { + return instance; + } + + public static VmClass getNodeClass() { + return NodeClass.instance; + } + + public static VmClass getSpanClass() { + return SpanClass.instance; + } + + public static VmClass getSourceLocationClass() { + return SourceLocationClass.instance; + } + + private static final class NodeClass { + static final VmClass instance = loadClass("Node"); + } + + private static final class SpanClass { + static final VmClass instance = loadClass("Span"); + } + + private static final class SourceLocationClass { + static final VmClass instance = loadClass("SourceLocation"); + } + + @TruffleBoundary + private static VmClass loadClass(String className) { + var theModule = getModule(); + return (VmClass) VmUtils.readMember(theModule, Identifier.get(className)); + } +} diff --git a/pkl-core/src/main/java/org/pkl/core/runtime/VmUtils.java b/pkl-core/src/main/java/org/pkl/core/runtime/VmUtils.java index 4b5d01f76..232339404 100644 --- a/pkl-core/src/main/java/org/pkl/core/runtime/VmUtils.java +++ b/pkl-core/src/main/java/org/pkl/core/runtime/VmUtils.java @@ -947,22 +947,28 @@ public final class VmUtils { } public static String getDisplayUri(SourceSection section, StackFrameTransformer transformer) { - var sourceUri = section.getSource().getURI(); - var frame = - new StackFrame( - sourceUri.toString(), - null, - List.of(), - section.getStartLine(), - section.getStartColumn(), - section.getEndLine(), - section.getEndColumn()); - StackFrame transformed = transformer.apply(frame); - return transformed.getModuleUri(); + return getDisplayUri( + section.getSource().getURI().toString(), + section.getStartLine(), + section.getStartColumn(), + section.getEndLine(), + section.getEndColumn(), + transformer); } public static String getDisplayUri(URI moduleUri, StackFrameTransformer transformer) { - var frame = new StackFrame(moduleUri.toString(), null, List.of(), 1, 1, 1, 1); + return getDisplayUri(moduleUri.toString(), 1, 1, 1, 1, transformer); + } + + public static String getDisplayUri( + String moduleUri, + int startLine, + int startColumn, + int endLine, + int endColumn, + StackFrameTransformer transformer) { + var frame = + new StackFrame(moduleUri, null, List.of(), startLine, startColumn, endLine, endColumn); StackFrame transformed = transformer.apply(frame); return transformed.getModuleUri(); } diff --git a/pkl-core/src/main/java/org/pkl/core/stdlib/VmObjectFactory.java b/pkl-core/src/main/java/org/pkl/core/stdlib/VmObjectFactory.java index 377a86709..256133596 100644 --- a/pkl-core/src/main/java/org/pkl/core/stdlib/VmObjectFactory.java +++ b/pkl-core/src/main/java/org/pkl/core/stdlib/VmObjectFactory.java @@ -118,6 +118,15 @@ public final class VmObjectFactory { return result; } + @TruffleBoundary + public VmTyped create(@Nullable E extraStorage, VmTyped parent) { + var clazz = classSupplier.get(); + assert parent.getVmClass() == clazz; + var result = new VmTyped(VmUtils.createEmptyMaterializedFrame(), parent, clazz, members); + result.setExtraStorage(extraStorage); + return result; + } + @FunctionalInterface public interface IntProperty { long evaluate(E extraStorage); diff --git a/pkl-core/src/main/java/org/pkl/core/stdlib/syntax/FormattingRendererNodes.java b/pkl-core/src/main/java/org/pkl/core/stdlib/syntax/FormattingRendererNodes.java new file mode 100644 index 000000000..7b36edae0 --- /dev/null +++ b/pkl-core/src/main/java/org/pkl/core/stdlib/syntax/FormattingRendererNodes.java @@ -0,0 +1,55 @@ +/* + * Copyright © 2026 Apple Inc. and the Pkl project authors. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.pkl.core.stdlib.syntax; + +import com.oracle.truffle.api.CompilerDirectives; +import com.oracle.truffle.api.CompilerDirectives.TruffleBoundary; +import com.oracle.truffle.api.dsl.Cached; +import com.oracle.truffle.api.dsl.Specialization; +import com.oracle.truffle.api.nodes.IndirectCallNode; +import org.pkl.core.runtime.Identifier; +import org.pkl.core.runtime.VmTyped; +import org.pkl.core.runtime.VmUtils; +import org.pkl.core.stdlib.ExternalMethod1Node; +import org.pkl.formatter.Formatter; +import org.pkl.formatter.GrammarVersion; +import org.pkl.parser.syntax.generic.Node; + +public final class FormattingRendererNodes { + private FormattingRendererNodes() {} + + public abstract static class render extends ExternalMethod1Node { + @Specialization + protected String eval( + VmTyped self, VmTyped nodeVm, @Cached("create()") IndirectCallNode callNode) { + var grammarVersion = (String) VmUtils.readMember(self, Identifier.GRAMMAR_VERSION, callNode); + var node = SyntaxNodes.convertVmToNode(nodeVm, SyntaxNodes.ZERO_SPAN, callNode); + try { + return format(grammarVersion, node); + } catch (RuntimeException e) { + CompilerDirectives.transferToInterpreter(); + throw exceptionBuilder() + .evalError("cannotRenderSyntaxNode", VmUtils.readMember(nodeVm, Identifier.TYPE)) + .build(); + } + } + + @TruffleBoundary + private static String format(String grammarVersion, Node node) { + return new Formatter(GrammarVersion.valueOf(grammarVersion)).format(node); + } + } +} diff --git a/pkl-core/src/main/java/org/pkl/core/stdlib/syntax/NodeNodes.java b/pkl-core/src/main/java/org/pkl/core/stdlib/syntax/NodeNodes.java new file mode 100644 index 000000000..1ed19a5a6 --- /dev/null +++ b/pkl-core/src/main/java/org/pkl/core/stdlib/syntax/NodeNodes.java @@ -0,0 +1,410 @@ +/* + * Copyright © 2026 Apple Inc. and the Pkl project authors. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.pkl.core.stdlib.syntax; + +import com.oracle.truffle.api.CompilerDirectives; +import com.oracle.truffle.api.CompilerDirectives.TruffleBoundary; +import com.oracle.truffle.api.dsl.Specialization; +import com.oracle.truffle.api.nodes.IndirectCallNode; +import com.oracle.truffle.api.nodes.LoopNode; +import com.oracle.truffle.api.nodes.Node; +import java.util.ArrayDeque; +import java.util.ArrayList; +import java.util.List; +import org.jspecify.annotations.Nullable; +import org.pkl.core.ast.lambda.ApplyVmFunction1Node; +import org.pkl.core.ast.lambda.ApplyVmFunction2Node; +import org.pkl.core.ast.lambda.ApplyVmFunction2NodeGen; +import org.pkl.core.runtime.Identifier; +import org.pkl.core.runtime.VmFunction; +import org.pkl.core.runtime.VmList; +import org.pkl.core.runtime.VmNull; +import org.pkl.core.runtime.VmTyped; +import org.pkl.core.runtime.VmUtils; +import org.pkl.core.stdlib.ExternalMethod1Node; +import org.pkl.core.stdlib.ExternalMethod2Node; +import org.pkl.core.stdlib.syntax.SyntaxNodes.Rebuild; + +/** Backs the methods of {@code pkl.syntax#Node}. */ +public final class NodeNodes { + private NodeNodes() {} + + public abstract static class fold extends ExternalMethod2Node { + @Child private ApplyVmFunction2Node applyAccumulate = ApplyVmFunction2NodeGen.create(); + @Child private IndirectCallNode callNode = IndirectCallNode.create(); + + @Specialization + protected Object eval(VmTyped self, Object initial, VmFunction operator) { + var pending = new ArrayDeque(); + pending.push(self); + var result = initial; + var visited = 0; + while (!pending.isEmpty()) { + var node = pending.pop(); + result = applyAccumulate.execute(operator, result, node); + var children = (VmList) VmUtils.readMember(node, Identifier.CHILDREN, callNode); + for (var i = children.getLength() - 1; i >= 0; i--) { + pending.push((VmTyped) children.get(i)); + } + visited += 1; + } + LoopNode.reportLoopCount(this, visited); + return result; + } + } + + public abstract static class findChild extends ExternalMethod1Node { + @Child private ApplyVmFunction1Node applyPredicate = ApplyVmFunction1Node.create(); + @Child private IndirectCallNode callNode = IndirectCallNode.create(); + + @Specialization + protected VmTyped eval(VmTyped self, VmFunction predicate) { + var result = findFirstChild(self, new PredicateMatcher(predicate, applyPredicate), callNode); + if (result == null) { + CompilerDirectives.transferToInterpreter(); + throw exceptionBuilder().evalError("cannotFindMatchingChildNode").build(); + } + return result; + } + } + + public abstract static class findChildOrNull extends ExternalMethod1Node { + @Child private ApplyVmFunction1Node applyPredicate = ApplyVmFunction1Node.create(); + @Child private IndirectCallNode callNode = IndirectCallNode.create(); + + @Specialization + protected Object eval(VmTyped self, VmFunction predicate) { + return VmNull.lift( + findFirstChild(self, new PredicateMatcher(predicate, applyPredicate), callNode)); + } + } + + public abstract static class findChildren extends ExternalMethod1Node { + @Child private ApplyVmFunction1Node applyPredicate = ApplyVmFunction1Node.create(); + @Child private IndirectCallNode callNode = IndirectCallNode.create(); + + @Specialization + protected VmList eval(VmTyped self, VmFunction predicate) { + return VmList.create( + findMatchingChildren( + this, self, new PredicateMatcher(predicate, applyPredicate), callNode)); + } + } + + public abstract static class findChildOfType extends ExternalMethod1Node { + @Child private IndirectCallNode callNode = IndirectCallNode.create(); + + @Specialization + protected VmTyped eval(VmTyped self, String type) { + var result = findFirstChild(self, new TypeMatcher(type), callNode); + if (result == null) { + CompilerDirectives.transferToInterpreter(); + throw exceptionBuilder().evalError("cannotFindChildNodeOfType", type).build(); + } + return result; + } + } + + public abstract static class findChildOfTypeOrNull extends ExternalMethod1Node { + @Child private IndirectCallNode callNode = IndirectCallNode.create(); + + @Specialization + protected Object eval(VmTyped self, String type) { + return VmNull.lift(findFirstChild(self, new TypeMatcher(type), callNode)); + } + } + + public abstract static class findChildrenOfType extends ExternalMethod1Node { + @Child private IndirectCallNode callNode = IndirectCallNode.create(); + + @Specialization + protected VmList eval(VmTyped self, String type) { + return VmList.create(findMatchingChildren(this, self, new TypeMatcher(type), callNode)); + } + } + + public abstract static class replaceChild extends ExternalMethod2Node { + @Child private ApplyVmFunction1Node applyPredicate = ApplyVmFunction1Node.create(); + @Child private IndirectCallNode callNode = IndirectCallNode.create(); + + @Specialization + protected VmTyped eval(VmTyped self, VmFunction predicate, VmFunction replacer) { + var matcher = new PredicateMatcher(predicate, applyPredicate); + return replaceMatchingChildren(self, matcher, true, replacer, callNode); + } + } + + public abstract static class replaceChildren extends ExternalMethod2Node { + @Child private ApplyVmFunction1Node applyPredicate = ApplyVmFunction1Node.create(); + @Child private IndirectCallNode callNode = IndirectCallNode.create(); + + @Specialization + protected VmTyped eval(VmTyped self, VmFunction predicate, VmFunction replacer) { + var matcher = new PredicateMatcher(predicate, applyPredicate); + return replaceMatchingChildren(self, matcher, false, replacer, callNode); + } + } + + public abstract static class replaceChildOfType extends ExternalMethod2Node { + @Child private IndirectCallNode callNode = IndirectCallNode.create(); + + @Specialization + protected VmTyped eval(VmTyped self, String type, VmFunction replacer) { + return replaceMatchingChildren(self, new TypeMatcher(type), true, replacer, callNode); + } + } + + public abstract static class replaceChildrenOfType extends ExternalMethod2Node { + @Child private IndirectCallNode callNode = IndirectCallNode.create(); + + @Specialization + protected VmTyped eval(VmTyped self, String type, VmFunction replacer) { + return replaceMatchingChildren(self, new TypeMatcher(type), false, replacer, callNode); + } + } + + public abstract static class findParent extends ExternalMethod1Node { + @Child private ApplyVmFunction1Node applyPredicate = ApplyVmFunction1Node.create(); + + @Specialization + protected VmTyped eval(VmTyped self, VmFunction predicate) { + var result = findFirstParent(this, self, new PredicateMatcher(predicate, applyPredicate)); + if (result == null) { + CompilerDirectives.transferToInterpreter(); + throw exceptionBuilder().evalError("cannotFindMatchingParentNode").build(); + } + return result; + } + } + + public abstract static class findParentOrNull extends ExternalMethod1Node { + @Child private ApplyVmFunction1Node applyPredicate = ApplyVmFunction1Node.create(); + + @Specialization + protected Object eval(VmTyped self, VmFunction predicate) { + return VmNull.lift( + findFirstParent(this, self, new PredicateMatcher(predicate, applyPredicate))); + } + } + + public abstract static class findParentOfType extends ExternalMethod1Node { + @Specialization + protected VmTyped eval(VmTyped self, String type) { + var result = findFirstParent(this, self, new TypeMatcher(type)); + if (result == null) { + CompilerDirectives.transferToInterpreter(); + throw exceptionBuilder().evalError("cannotFindParentNodeOfType", type).build(); + } + return result; + } + } + + public abstract static class findParentOfTypeOrNull extends ExternalMethod1Node { + @Specialization + protected Object eval(VmTyped self, String type) { + return VmNull.lift(findFirstParent(this, self, new TypeMatcher(type))); + } + } + + public abstract static class findParents extends ExternalMethod1Node { + @Child private ApplyVmFunction1Node applyPredicate = ApplyVmFunction1Node.create(); + + @Specialization + protected VmList eval(VmTyped self, VmFunction predicate) { + var matcher = new PredicateMatcher(predicate, applyPredicate); + var matches = VmList.EMPTY.builder(); + var visited = 0; + for (var node = parentOf(self); node != null; node = parentOf(node)) { + visited += 1; + if (matcher.matches(node)) { + matches.add(node); + } + } + LoopNode.reportLoopCount(this, visited); + return matches.build(); + } + } + + public abstract static class hasParent extends ExternalMethod1Node { + @Child private ApplyVmFunction1Node applyPredicate = ApplyVmFunction1Node.create(); + + @Specialization + protected boolean eval(VmTyped self, VmFunction predicate) { + return findFirstParent(this, self, new PredicateMatcher(predicate, applyPredicate)) != null; + } + } + + public abstract static class hasParentOfType extends ExternalMethod1Node { + @Specialization + protected boolean eval(VmTyped self, String type) { + return findFirstParent(this, self, new TypeMatcher(type)) != null; + } + } + + /** Decides whether a node is a match for a search. */ + private interface NodeMatcher { + boolean matches(VmTyped node); + } + + /** Matches the nodes a Pkl predicate accepts. */ + private record PredicateMatcher(VmFunction predicate, ApplyVmFunction1Node applyPredicate) + implements NodeMatcher { + + @Override + public boolean matches(VmTyped node) { + return applyPredicate.executeBoolean(predicate, node); + } + } + + /** Matches the nodes of a given type. */ + private record TypeMatcher(String type) implements NodeMatcher { + @Override + public boolean matches(VmTyped node) { + return type.equals(VmUtils.readMember(node, Identifier.TYPE)); + } + } + + private static @Nullable VmTyped findFirstChild( + VmTyped self, NodeMatcher matcher, IndirectCallNode callNode) { + var pending = new ArrayDeque(); + pushChildren(pending, self, callNode); + while (!pending.isEmpty()) { + var node = pending.pop(); + if (matcher.matches(node)) { + return node; + } + pushChildren(pending, node, callNode); + } + return null; + } + + /** + * Collect the descendants of {@code self} matching {@code matcher}, searching depth-first in + * pre-order. + * + *

A match is searched for further matches, so a match may contain another. + */ + private static List findMatchingChildren( + Node owner, VmTyped self, NodeMatcher matcher, IndirectCallNode callNode) { + + List matches = new ArrayList<>(); + var pending = new ArrayDeque(); + pushChildren(pending, self, callNode); + var visited = 0; + while (!pending.isEmpty()) { + var node = pending.pop(); + visited += 1; + if (matcher.matches(node)) { + matches.add(node); + } + pushChildren(pending, node, callNode); + } + LoopNode.reportLoopCount(owner, visited); + return matches; + } + + private static void pushChildren( + ArrayDeque pending, VmTyped node, IndirectCallNode callNode) { + var children = (VmList) VmUtils.readMember(node, Identifier.CHILDREN, callNode); + for (var i = children.getLength() - 1; i >= 0; i--) { + pending.push((VmTyped) children.get(i)); + } + } + + /** + * Returns {@code self} rebuilt with the descendants matching {@code matcher} replaced by {@code + * replacer}'s results, searching depth-first in pre-order and stopping at the first match if + * {@code firstOnly}. + * + *

A match is not searched for further matches. + */ + private static VmTyped replaceMatchingChildren( + VmTyped self, + NodeMatcher matcher, + boolean firstOnly, + VmFunction replacer, + IndirectCallNode callNode) { + + var search = new ReplaceSearch(matcher, firstOnly, replacer, callNode); + return SyntaxNodes.rebuild(self, search.replaceBelow(self)); + } + + /** Finds and replaces the matches of a {@code Node.replaceChild*} call. */ + private static final class ReplaceSearch { + private final NodeMatcher matcher; + private final boolean firstOnly; + private final VmFunction replacer; + private final IndirectCallNode callNode; + + private boolean done; + + private ReplaceSearch( + NodeMatcher matcher, boolean firstOnly, VmFunction replacer, IndirectCallNode callNode) { + this.matcher = matcher; + this.firstOnly = firstOnly; + this.replacer = replacer; + this.callNode = callNode; + } + + /** + * Returns the children of {@code node} with the matches below it replaced, or {@code null} if + * nothing below it matches. + * + *

Each element is either a {@link Rebuild} or a node to keep as it is. + */ + @TruffleBoundary + private Object @Nullable [] replaceBelow(VmTyped node) { + var children = (VmList) VmUtils.readMember(node, Identifier.CHILDREN, callNode); + Object @Nullable [] result = null; + for (var i = 0; i < children.getLength() && !done; i++) { + var child = (VmTyped) children.get(i); + Object replacement; + if (matcher.matches(child)) { + replacement = replacer.apply(child); + done = firstOnly; + } else { + var grandchildren = replaceBelow(child); + replacement = grandchildren == null ? null : new Rebuild(child, grandchildren); + } + if (replacement != null) { + if (result == null) { + result = children.toArray(); + } + result[i] = replacement; + } + } + return result; + } + } + + private static @Nullable VmTyped findFirstParent(Node owner, VmTyped self, NodeMatcher matcher) { + VmTyped result = null; + var visited = 0; + for (var node = parentOf(self); node != null; node = parentOf(node)) { + visited += 1; + if (matcher.matches(node)) { + result = node; + break; + } + } + LoopNode.reportLoopCount(owner, visited); + return result; + } + + private static @Nullable VmTyped parentOf(VmTyped node) { + return (VmTyped) VmNull.unwrap(VmUtils.readMember(node, Identifier.PARENT)); + } +} diff --git a/pkl-core/src/main/java/org/pkl/core/stdlib/syntax/ParserNodes.java b/pkl-core/src/main/java/org/pkl/core/stdlib/syntax/ParserNodes.java new file mode 100644 index 000000000..b536c5544 --- /dev/null +++ b/pkl-core/src/main/java/org/pkl/core/stdlib/syntax/ParserNodes.java @@ -0,0 +1,146 @@ +/* + * Copyright © 2026 Apple Inc. and the Pkl project authors. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.pkl.core.stdlib.syntax; + +import com.oracle.truffle.api.CompilerDirectives; +import com.oracle.truffle.api.CompilerDirectives.TruffleBoundary; +import com.oracle.truffle.api.dsl.Cached; +import com.oracle.truffle.api.dsl.Specialization; +import com.oracle.truffle.api.nodes.IndirectCallNode; +import org.jspecify.annotations.Nullable; +import org.pkl.core.runtime.Identifier; +import org.pkl.core.runtime.VmNull; +import org.pkl.core.runtime.VmTyped; +import org.pkl.core.runtime.VmUtils; +import org.pkl.core.stdlib.ExternalMethod1Node; +import org.pkl.core.stdlib.syntax.SyntaxNodes.NodeData; +import org.pkl.parser.GenericParser; +import org.pkl.parser.GenericParserError; + +/** Backs the {@code parse*} methods of {@code pkl.syntax#Parser}. */ +public final class ParserNodes { + private ParserNodes() {} + + private static GenericParser parser = new GenericParser(); + + public abstract static class parseModule extends ExternalMethod1Node { + @Specialization + protected Object evalString(VmTyped ignored, String source) { + try { + return parseModuleNode(source, null); + } catch (GenericParserError e) { + CompilerDirectives.transferToInterpreter(); + throw exceptionBuilder().evalError("parserError").withHint(e.toString()).build(); + } + } + + @Specialization + protected Object evalResource( + VmTyped ignored, VmTyped source, @Cached("create()") IndirectCallNode callNode) { + var text = (String) VmUtils.readMember(source, Identifier.TEXT, callNode); + var uri = (String) VmUtils.readMember(source, Identifier.URI, callNode); + try { + return parseModuleNode(text, uri); + } catch (GenericParserError e) { + CompilerDirectives.transferToInterpreter(); + throw exceptionBuilder().evalError("parserError").withHint(e.toString()).build(); + } + } + } + + public abstract static class parseModuleOrNull extends ExternalMethod1Node { + @Specialization + protected Object evalString(VmTyped ignored, String source) { + try { + return parseModuleNode(source, null); + } catch (GenericParserError e) { + return VmNull.withoutDefault(); + } + } + + @Specialization + protected Object evalResource( + VmTyped ignored, VmTyped source, @Cached("create()") IndirectCallNode callNode) { + var text = (String) VmUtils.readMember(source, Identifier.TEXT, callNode); + var uri = (String) VmUtils.readMember(source, Identifier.URI, callNode); + try { + return parseModuleNode(text, uri); + } catch (GenericParserError e) { + return VmNull.withoutDefault(); + } + } + } + + public abstract static class parseExpression extends ExternalMethod1Node { + @Specialization + protected Object evalString(VmTyped ignored, String source) { + try { + return parseExpressionNode(source, null); + } catch (GenericParserError e) { + CompilerDirectives.transferToInterpreter(); + throw exceptionBuilder().evalError("parserError").withHint(e.toString()).build(); + } + } + + @Specialization + protected Object evalResource( + VmTyped ignored, VmTyped source, @Cached("create()") IndirectCallNode callNode) { + var text = (String) VmUtils.readMember(source, Identifier.TEXT, callNode); + var uri = (String) VmUtils.readMember(source, Identifier.URI, callNode); + try { + return parseExpressionNode(text, uri); + } catch (GenericParserError e) { + CompilerDirectives.transferToInterpreter(); + throw exceptionBuilder().evalError("parserError").withHint(e.toString()).build(); + } + } + } + + public abstract static class parseExpressionOrNull extends ExternalMethod1Node { + @Specialization + protected Object evalString(VmTyped ignored, String source) { + try { + return parseExpressionNode(source, null); + } catch (GenericParserError e) { + return VmNull.withoutDefault(); + } + } + + @Specialization + protected Object evalResource( + VmTyped ignored, VmTyped source, @Cached("create()") IndirectCallNode callNode) { + var text = (String) VmUtils.readMember(source, Identifier.TEXT, callNode); + var uri = (String) VmUtils.readMember(source, Identifier.URI, callNode); + try { + return parseExpressionNode(text, uri); + } catch (GenericParserError e) { + return VmNull.withoutDefault(); + } + } + } + + @TruffleBoundary + private static VmTyped parseModuleNode(String src, @Nullable String sourceUri) { + var root = parser.parseModule(src); + return SyntaxNodes.createNode(new NodeData(root, src.toCharArray(), sourceUri, null)); + } + + @TruffleBoundary + private static VmTyped parseExpressionNode(String src, @Nullable String sourceUri) { + var root = parser.parseExpressionInput(src); + return SyntaxNodes.createNode(new NodeData(root, src.toCharArray(), sourceUri, null)); + } +} diff --git a/pkl-core/src/main/java/org/pkl/core/stdlib/syntax/SyntaxNodes.java b/pkl-core/src/main/java/org/pkl/core/stdlib/syntax/SyntaxNodes.java new file mode 100644 index 000000000..e84aff76a --- /dev/null +++ b/pkl-core/src/main/java/org/pkl/core/stdlib/syntax/SyntaxNodes.java @@ -0,0 +1,305 @@ +/* + * Copyright © 2025-2026 Apple Inc. and the Pkl project authors. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.pkl.core.stdlib.syntax; + +import com.oracle.truffle.api.CompilerDirectives.TruffleBoundary; +import com.oracle.truffle.api.nodes.ExplodeLoop; +import com.oracle.truffle.api.nodes.IndirectCallNode; +import java.util.ArrayList; +import java.util.List; +import java.util.Locale; +import org.jspecify.annotations.Nullable; +import org.pkl.core.runtime.Identifier; +import org.pkl.core.runtime.SyntaxModule; +import org.pkl.core.runtime.VmContext; +import org.pkl.core.runtime.VmList; +import org.pkl.core.runtime.VmNull; +import org.pkl.core.runtime.VmTyped; +import org.pkl.core.runtime.VmUtils; +import org.pkl.core.stdlib.VmObjectFactory; +import org.pkl.parser.syntax.generic.FullSpan; +import org.pkl.parser.syntax.generic.Node; +import org.pkl.parser.syntax.generic.NodeType; + +public final class SyntaxNodes { + private SyntaxNodes() {} + + static final FullSpan ZERO_SPAN = new FullSpan(0, 0, 0, 0, 0, 0); + + record SpanData(FullSpan span, @Nullable String sourceUri) {} + + private record SourceLocationData(int line, int column, @Nullable String sourceUri) {} + + private static final VmObjectFactory sourceLocationFactory = + new VmObjectFactory(SyntaxModule::getSourceLocationClass) + .addIntProperty("line", SourceLocationData::line) + .addIntProperty("column", SourceLocationData::column) + .addStringProperty( + "displayUri", sl -> displayUri(sl.sourceUri(), sl.line(), sl.column())); + + static final VmObjectFactory spanFactory = + new VmObjectFactory(SyntaxModule::getSpanClass) + .addTypedProperty( + "start", + sd -> + sourceLocationFactory.create( + new SourceLocationData( + sd.span().lineBegin(), sd.span().colBegin(), sd.sourceUri()))) + .addTypedProperty( + "end", + sd -> + sourceLocationFactory.create( + new SourceLocationData( + sd.span().lineEnd(), sd.span().colEnd(), sd.sourceUri()))) + .addStringProperty( + "displayUri", + sd -> + displayUri( + sd.sourceUri(), + sd.span().lineBegin(), + sd.span().colBegin(), + sd.span().lineEnd(), + sd.span().colEnd())); + + private static String displayUri(@Nullable String sourceUri, int line, int column) { + return displayUri(sourceUri, line, column, line, column); + } + + @TruffleBoundary + private static String displayUri( + @Nullable String sourceUri, int startLine, int startColumn, int endLine, int endColumn) { + if (sourceUri == null) { + return ""; + } + return VmUtils.getDisplayUri( + sourceUri, + startLine, + startColumn, + endLine, + endColumn, + VmContext.get(null).getFrameTransformer()); + } + + /** Extra storage backing a Pkl {@code Node} parsed from source. */ + static final class NodeData { + final Node node; + // the source {@link #node} was parsed from + final char[] source; + // the URI of {@link #source}, or {@code null} if unknown + private final @Nullable String sourceUri; + private final @Nullable VmTyped parentVm; + + // the object this storage backs, so that lazily created children can be parented at it + private @Nullable VmTyped selfVm; + + NodeData(Node node, char[] source, @Nullable String sourceUri, @Nullable VmTyped parentVm) { + this.node = node; + this.source = source; + this.sourceUri = sourceUri; + this.parentVm = parentVm; + } + + String text() { + return node.text(source); + } + + @ExplodeLoop + VmList children() { + var childNodes = node.children; + if (childNodes.isEmpty()) { + return VmList.EMPTY; + } + var builder = VmList.EMPTY.builder(); + for (var childNode : childNodes) { + builder.add(createNode(new NodeData(childNode, source, sourceUri, selfVm))); + } + return builder.build(); + } + + VmTyped span() { + return spanFactory.create(new SpanData(node.span, sourceUri)); + } + } + + static final VmObjectFactory genericNodeFactory = + new VmObjectFactory(SyntaxModule::getNodeClass) + .addStringProperty("type", nd -> nd.node.type.name().toLowerCase(Locale.ROOT)) + .addListProperty("children", NodeData::children) + .addProperty("parent", nd -> VmNull.lift(nd.parentVm)) + .addProperty("text", NodeData::text) + .addProperty("span", NodeData::span); + + /** Create the Pkl {@code Node} backed by {@code data}. */ + static VmTyped createNode(NodeData data) { + var result = genericNodeFactory.create(data); + data.selfVm = result; + return result; + } + + /** + * A node to rebuild with new children. + * + *

Each of {@code children} is either a {@code Rebuild} or a node to keep as it is. + */ + record Rebuild(VmTyped basis, Object[] children) {} + + /** + * Extra storage backing a Pkl {@code Node} rebuilt from {@link #build(VmTyped, Object[], + * VmTyped)}. + */ + private static final class BuiltNodeData { + private final @Nullable VmTyped parentVm; + + // set right after the node is created, so that the children can be parented at it + private VmList children = VmList.EMPTY; + + private BuiltNodeData(@Nullable VmTyped parentVm) { + this.parentVm = parentVm; + } + } + + private static final VmObjectFactory builtNodeFactory = + new VmObjectFactory(SyntaxModule::getNodeClass) + .addListProperty("children", bd -> bd.children) + .addProperty("parent", bd -> VmNull.lift(bd.parentVm)); + + /** + * Build the tree rooted at {@code root}, with its children replaced by {@code children} if not + * {@code null}. + * + *

The returned root has no parent, and every other node is parented in the returned tree. The + * tree {@code root} is left untouched. + */ + @TruffleBoundary + static VmTyped rebuild(VmTyped root, Object @Nullable [] children) { + return children == null ? reparent(root, null) : build(root, children, null); + } + + private static VmTyped build(VmTyped basis, Object[] children, @Nullable VmTyped parent) { + var data = new BuiltNodeData(parent); + var result = builtNodeFactory.create(data, basis); + var listBuilder = VmList.EMPTY.builder(); + for (var child : children) { + var newChild = + child instanceof Rebuild rebuild + ? build(rebuild.basis(), rebuild.children(), result) + : reparent((VmTyped) child, result); + listBuilder.add(newChild); + } + // A node and its children point at each other, so neither can be complete before the other is + // created. This node is therefore created first, without children, so its children can be built + // with it as their parent, and only then are they set here. + // This is safe because factory properties are evaluated lazily, on first read, and no Pkl code + // can see this node before `build` returns, so `children` can't be read before it's set. + data.children = listBuilder.build(); + return result; + } + + /** Copy {@code node} and its subtree to sit below {@code parent}. */ + private static VmTyped reparent(VmTyped node, @Nullable VmTyped parent) { + // a parsed node's subtree is created lazily from its parse-time node, so only the node is + // copied + if (node.hasExtraStorage() && node.getExtraStorage() instanceof NodeData data) { + return createNode(new NodeData(data.node, data.source, data.sourceUri, parent)); + } + var children = (VmList) VmUtils.readMember(node, Identifier.CHILDREN); + return build(node, children.toArray(), parent); + } + + /** + * Convert a Pkl {@code Node} to a generic {@link Node}, reusing the parse-time node when present. + * + *

{@code fallbackSpan} is used for constructed nodes (and their descendants) that carry no + * meaningful span of their own, so that a subtree spliced into reused siblings lines up with + * them. + */ + @TruffleBoundary + static Node convertVmToNode(VmTyped nodeVm, FullSpan fallbackSpan, IndirectCallNode callNode) { + if (nodeVm.hasExtraStorage()) { + var storage = nodeVm.getExtraStorage(); + // a node still carrying its parse-time storage is verbatim from `parse`: reuse it wholesale + if (storage instanceof NodeData data) { + materializeText(data.node, data.source); + return data.node; + } + } + + var typeStr = (String) VmUtils.readMember(nodeVm, Identifier.TYPE, callNode); + var nodeType = NodeType.valueOf(typeStr.toUpperCase(Locale.ROOT)); + + var ownSpan = readSpan(optSpan(nodeVm, callNode), callNode); + // a constructed node that did not set its own span inherits the insertion point's span + var span = ownSpan.equals(ZERO_SPAN) ? fallbackSpan : ownSpan; + + var childrenVm = (VmList) VmUtils.readMember(nodeVm, Identifier.CHILDREN, callNode); + var children = new ArrayList(childrenVm.getLength()); + for (var i = 0; i < childrenVm.getLength(); i++) { + children.add(convertVmToNode((VmTyped) childrenVm.get(i), span, callNode)); + } + + return makeJavaNode( + nodeType, span, children, VmUtils.readMember(nodeVm, Identifier.TEXT, callNode)); + } + + private static @Nullable VmTyped optSpan(VmTyped nodeVm, IndirectCallNode callNode) { + return (VmTyped) VmNull.unwrap(VmUtils.readMember(nodeVm, Identifier.SPAN, callNode)); + } + + private static FullSpan readSpan(@Nullable VmTyped spanVm, IndirectCallNode callNode) { + if (spanVm == null) { + return ZERO_SPAN; + } + var start = (VmTyped) VmUtils.readMember(spanVm, Identifier.START, callNode); + var end = (VmTyped) VmUtils.readMember(spanVm, Identifier.END, callNode); + return new FullSpan( + 0, + 0, + readPosition(start, Identifier.LINE, callNode), + readPosition(start, Identifier.COLUMN, callNode), + readPosition(end, Identifier.LINE, callNode), + readPosition(end, Identifier.COLUMN, callNode)); + } + + private static int readPosition( + VmTyped sourceLocationVm, Identifier name, IndirectCallNode callNode) { + return ((Long) VmUtils.readMember(sourceLocationVm, name, callNode)).intValue(); + } + + private static Node makeJavaNode( + NodeType nodeType, FullSpan span, List children, Object textObj) { + var node = children.isEmpty() ? new Node(nodeType, span) : new Node(nodeType, span, children); + if (textObj instanceof String text) { + node.setText(text); + } + return node; + } + + /** + * Materialize the text of the nodes in {@code node}'s subtree that the formatter reads directly. + * + *

{@link org.pkl.formatter.Formatter#format(Node)} has no access to the source, so a subtree + * reused verbatim from a parse must carry its own text by the time it is handed over. + */ + private static void materializeText(Node node, char[] source) { + // `string_constant` is read by the formatter but is not a leaf + if (node.children.isEmpty() || node.type == NodeType.STRING_CONSTANT) { + node.text(source); + } + for (var child : node.children) { + materializeText(child, source); + } + } +} diff --git a/pkl-core/src/main/java/org/pkl/core/stdlib/syntax/package-info.java b/pkl-core/src/main/java/org/pkl/core/stdlib/syntax/package-info.java new file mode 100644 index 000000000..14f5426cc --- /dev/null +++ b/pkl-core/src/main/java/org/pkl/core/stdlib/syntax/package-info.java @@ -0,0 +1,4 @@ +@NullMarked +package org.pkl.core.stdlib.syntax; + +import org.jspecify.annotations.NullMarked; diff --git a/pkl-core/src/main/resources/org/pkl/core/errorMessages.properties b/pkl-core/src/main/resources/org/pkl/core/errorMessages.properties index 77db29f25..f16567415 100644 --- a/pkl-core/src/main/resources/org/pkl/core/errorMessages.properties +++ b/pkl-core/src/main/resources/org/pkl/core/errorMessages.properties @@ -1226,6 +1226,24 @@ Redirected to: `{1}` invalidReferenceTypeAnnotationWithConstraint=\ `Reference` referent type argument may not include type constraints. +parserError=\ +Could not parse Pkl source. + +cannotRenderSyntaxNode=\ +Could not render node of type `{0}` as Pkl source because it is not a well-formed syntax tree. + +cannotFindMatchingChildNode=\ +Cannot find a child node matching the predicate. + +cannotFindChildNodeOfType=\ +Cannot find a child node of type `{0}`. + +cannotFindMatchingParentNode=\ +Cannot find a parent node matching the predicate. + +cannotFindParentNodeOfType=\ +Cannot find a parent node of type `{0}`. + cannotInstallPackageWithNoCache=\ Cannot install package to module cache dir when module cache is disabled. diff --git a/pkl-core/src/test/files/LanguageSnippetTests/input-helper/syntax/spans/sample.pkl b/pkl-core/src/test/files/LanguageSnippetTests/input-helper/syntax/spans/sample.pkl new file mode 100644 index 000000000..1cd9df9f5 --- /dev/null +++ b/pkl-core/src/test/files/LanguageSnippetTests/input-helper/syntax/spans/sample.pkl @@ -0,0 +1,5 @@ +x = 1 + +foo { + bar = "baz" +} diff --git a/pkl-core/src/test/files/LanguageSnippetTests/input/errors/syntaxParseExpression.pkl b/pkl-core/src/test/files/LanguageSnippetTests/input/errors/syntaxParseExpression.pkl new file mode 100644 index 000000000..756eeeb45 --- /dev/null +++ b/pkl-core/src/test/files/LanguageSnippetTests/input/errors/syntaxParseExpression.pkl @@ -0,0 +1,5 @@ +import "pkl:syntax" + +local parser = new syntax.Parser {} + +res = parser.parseExpression("1 +") diff --git a/pkl-core/src/test/files/LanguageSnippetTests/input/errors/syntaxParseModule.pkl b/pkl-core/src/test/files/LanguageSnippetTests/input/errors/syntaxParseModule.pkl new file mode 100644 index 000000000..f884ecb90 --- /dev/null +++ b/pkl-core/src/test/files/LanguageSnippetTests/input/errors/syntaxParseModule.pkl @@ -0,0 +1,12 @@ +import "pkl:syntax" + +local parser = new syntax.Parser {} + +res = + parser.parseModule( + """ + foo { + bar = + } + """, + ) diff --git a/pkl-core/src/test/files/LanguageSnippetTests/input/errors/syntaxParseModuleLexerError.pkl b/pkl-core/src/test/files/LanguageSnippetTests/input/errors/syntaxParseModuleLexerError.pkl new file mode 100644 index 000000000..e20925866 --- /dev/null +++ b/pkl-core/src/test/files/LanguageSnippetTests/input/errors/syntaxParseModuleLexerError.pkl @@ -0,0 +1,8 @@ +import "pkl:syntax" + +mod = + new syntax.Parser {}.parseModule( + """ + import "foo\nbar" + """, + ) diff --git a/pkl-core/src/test/files/LanguageSnippetTests/input/syntax/query.pkl b/pkl-core/src/test/files/LanguageSnippetTests/input/syntax/query.pkl new file mode 100644 index 000000000..168657313 --- /dev/null +++ b/pkl-core/src/test/files/LanguageSnippetTests/input/syntax/query.pkl @@ -0,0 +1,257 @@ +amends "../snippetTest.pkl" + +import "pkl:syntax" +import "pkl:test" + +local parser = new syntax.Parser {} + +local renderer = new syntax.FormattingRenderer {} + +local function mod(source: String): syntax.Node = parser.parseModule(source) + +local function intLiteral(value: Int): syntax.Node = new syntax.Node { + type = "int_literal_expr" + text = value.toString() +} + +// a node built from scratch, whose children have no parent of their own +local constructedAccess = new syntax.Node { + type = "unqualified_access_expr" + children = List(new syntax.Node { type = "identifier"; text = "foo" }) +} + +local sample: syntax.Node = + mod( + """ + class Point { + x: Int + y: Int + } + + origin = new Point { x = 0; y = 0 } + """, + ) + +facts { + ["findChild returns the first match in pre-order"] { + sample.findChild((n) -> n.type == "identifier").text == "Point" + sample.findChild((n) -> n.type == "int_literal_expr").text == "0" + } + + ["findChild throws when nothing matches"] { + test + .catch(() -> sample.findChild((n) -> n.type == "when_generator")) + .contains("Cannot find a child node matching the predicate") + } + + ["findChildOrNull returns null when nothing matches"] { + sample.findChildOrNull((n) -> n.type == "when_generator") == null + sample.findChildOrNull((n) -> n.type == "identifier")!!.text == "Point" + } + + ["findChild does not consider this node"] { + sample.findChildOrNull((n) -> n.type == "module") == null + sample.type == "module" + } + + ["findChildren returns every match in pre-order"] { + sample + .findChildren((n) -> n.type == "identifier") + .map((n) -> n.text) + == List("Point", "x", "Int", "y", "Int", "origin", "Point", "x", "y") + + sample.findChildren((n) -> n.type == "when_generator") == List() + } + + ["findChildren searches a match for further matches"] { + // the outer object body contains the two entries, which are themselves matches + mod("foo { bar { baz = 1 } }").findChildren((n) -> n.type == "object_body").length == 2 + } + + ["findChildOfType agrees with findChild"] { + sample.findChildOfType("class").text == sample.findChild((n) -> n.type == "class").text + sample.findChildrenOfType("identifier").length == 9 + } + + ["findChildOfType throws when nothing matches"] { + test + .catch(() -> sample.findChildOfType("when_generator")) + .contains("Cannot find a child node of type `when_generator`") + } + + ["findChildOfTypeOrNull returns null when nothing matches"] { + sample.findChildOfTypeOrNull("when_generator") == null + sample.findChildOfTypeOrNull("class")!!.text == sample.findChildOfType("class").text + } + + ["findParent returns the nearest match"] { + local inner = mod("foo { bar { baz = 1 } }").findChild((n) -> n.text == "baz") + inner.findParent((n) -> n.type == "object_body").text == "{ baz = 1 }" + inner.findParent((n) -> n.type == "module").type == "module" + test + .catch(() -> inner.findParent((n) -> n.type == "class")) + .contains("Cannot find a parent node matching the predicate") + } + + ["findParentOrNull returns null when nothing matches"] { + local inner = mod("foo { bar { baz = 1 } }").findChild((n) -> n.text == "baz") + inner.findParentOrNull((n) -> n.type == "class") == null + inner.findParentOrNull((n) -> n.type == "object_body")!!.text == "{ baz = 1 }" + } + + ["findParentOfType agrees with findParent"] { + local inner = mod("foo { bar { baz = 1 } }").findChild((n) -> n.text == "baz") + inner.findParentOfType("object_body").text == "{ baz = 1 }" + test + .catch(() -> inner.findParentOfType("class")) + .contains("Cannot find a parent node of type `class`") + } + + ["findParentOfTypeOrNull returns null when nothing matches"] { + local inner = mod("foo { bar { baz = 1 } }").findChild((n) -> n.text == "baz") + inner.findParentOfTypeOrNull("class") == null + inner.findParentOfTypeOrNull("object_body")!!.text == "{ baz = 1 }" + } + + ["findParents returns every match, nearest first"] { + local inner = mod("foo { bar { baz = 1 } }").findChild((n) -> n.text == "baz") + inner + .findParents((n) -> n.type == "object_body") + .map((n) -> n.text) + == List("{ baz = 1 }", "{ bar { baz = 1 } }") + } + + ["parent lookups do not consider this node"] { + sample.findParentOrNull((n) -> n.type == "module") == null + sample.findParents((n) -> true) == List() + !sample.hasParentOfType("module") + } + + ["hasParent and hasParentOfType follow findParent"] { + local inner = mod("foo { bar = 1 }").findChild((n) -> n.text == "bar") + inner.hasParent((n) -> n.type == "object_body") + inner.hasParentOfType("class_property") + !inner.hasParentOfType("class") + } + + ["replaceChild replaces only the first match"] { + renderer.render( + mod("x = 1 + 1").replaceChild((n) -> n.type == "int_literal_expr", (_n) -> intLiteral(9)), + ) == "x = 9 + 1\n" + } + + ["replaceChild leaves the tree unchanged when nothing matches"] { + renderer.render( + mod("x = 1").replaceChild((n) -> n.type == "when_generator", (_n) -> intLiteral(9)), + ) == "x = 1\n" + } + + ["replaceChildren replaces every match"] { + renderer.render( + mod("x = 1 + 1").replaceChildren((n) -> n.type == "int_literal_expr", (_n) -> intLiteral(9)), + ) == "x = 9 + 9\n" + } + + ["a replacer sees the node it replaces"] { + renderer.render( + mod("x = 1 + 2") + .replaceChildren( + (n) -> n.type == "int_literal_expr", + (n) -> intLiteral(n.text!!.toInt() * 10), + ), + ) == "x = 10 + 20\n" + } + + ["a replacement is not searched for further matches"] { + // replacing an identifier with an identifier terminates instead of looping + renderer.render( + mod("foo = 1").replaceChildrenOfType("identifier", (n) -> (n) { text = "\(n.text!!)Bar" }), + ) == "fooBar = 1\n" + } + + ["replaceChildOfType agrees with replaceChild"] { + renderer.render(mod("x = 1 + 1").replaceChildOfType("int_literal_expr", (_n) -> intLiteral(9))) + == renderer.render( + mod("x = 1 + 1").replaceChild((n) -> n.type == "int_literal_expr", (_n) -> intLiteral(9)), + ) + } + + ["a replacer sees where its node sits in the tree being built"] { + local result = + mod("x = 1") + .replaceChildOfType("int_literal_expr", (n) -> + if (n.hasParentOfType("class_property")) intLiteral(9) else intLiteral(0) + ) + renderer.render(result) == "x = 9\n" + } + + ["a replace does not change the tree it walks"] { + local original = mod("x = 1") + local rewritten = original.replaceChildrenOfType("int_literal_expr", (_n) -> intLiteral(9)) + let (rendered = renderer.render(rewritten)) + rendered == "x = 9\n" && renderer.render(original) == "x = 1\n" + } + + ["the root of a replaced tree has no parent"] { + mod("x = 1").replaceChildrenOfType("int_literal_expr", (_n) -> intLiteral(9)).parent == null + mod("x = 1").replaceChildrenOfType("when_generator", (_n) -> intLiteral(9)).parent == null + } + + ["only the root of a replaced tree has no parent"] { + mod("x = 1") + .replaceChildrenOfType("int_literal_expr", (_n) -> intLiteral(9)) + .fold(0, (acc, n) -> if (n.parent == null) acc + 1 else acc) == 1 + // the constructed node's child has no parent of its own, so the returned tree must supply one + mod("x = 1") + .replaceChildrenOfType("int_literal_expr", (_) -> constructedAccess) + .fold(0, (acc, n) -> if (n.parent == null) acc + 1 else acc) == 1 + } + + ["replacing below a node does not detach it from its own tree"] { + local property = mod("x = 1").findChildOfType("class_property") + local replaced = property.replaceChildrenOfType("int_literal_expr", (_) -> intLiteral(9)) + replaced.parent == null && property.parent != null + } + + ["a replacer sees this tree, not the one being built"] { + // both literals see their unreplaced sibling + local sumOfSiblings = (n: syntax.Node) -> + intLiteral( + n.parent!! + .findChildrenOfType("int_literal_expr") + .fold(0, (acc, c) -> acc + c.text!!.toInt()), + ) + renderer.render(mod("x = 1 + 2").replaceChildrenOfType("int_literal_expr", sumOfSiblings)) + == "x = 3 + 3\n" + } + + ["the children of an amended replacement are parented at the replacement"] { + mod("x = 1 + 2") + .replaceChildOfType("binary_op_expr", (n) -> (n) { text = "changed" }) + .findChildOfType("int_literal_expr") + .parent!!.text + == "changed" + } + + ["rebuilds ancestors of a replacement"] { + renderer.render( + mod("x = if (cond) 41 else 0") + .replaceChildrenOfType("int_literal_expr", (n) -> intLiteral(n.text!!.toInt() + 1)), + ) == "x = if (cond) 42 else 1\n" + } + + ["a match is not searched for further matches"] { + local predicate = (n: syntax.Node) -> + if (n.type == "int_literal_expr") + throw("searched inside a match") + else + n.type == "parenthesized_expr" + renderer.render(mod("x = (1)").replaceChildren(predicate, (_) -> intLiteral(9))) == "x = 9\n" + } + + ["replacement is eager"] { + test + .catch(() -> mod("x = 1").replaceChildrenOfType("int_literal_expr", (_) -> throw("boom"))) + .contains("boom") + } +} diff --git a/pkl-core/src/test/files/LanguageSnippetTests/input/syntax/render.pkl b/pkl-core/src/test/files/LanguageSnippetTests/input/syntax/render.pkl new file mode 100644 index 000000000..b11f7101f --- /dev/null +++ b/pkl-core/src/test/files/LanguageSnippetTests/input/syntax/render.pkl @@ -0,0 +1,413 @@ +amends "../snippetTest.pkl" + +import "pkl:syntax" +import "pkl:test" + +local renderer = new syntax.FormattingRenderer {} + +local function roundTrip(source: String) = renderer.render(parseNode(source)) + +local function parseNode(source: String): syntax.Node = + new syntax.Parser {}.parseModule(source) + +local function replaceLeaf( + node: syntax.Node, + targetType: syntax.NodeType, + oldText: String, + newText: String, +): syntax.Node = + if (node.type == targetType && node.text == oldText) + (node) { text = newText } + else + (node) { + children = node.children.map((c) -> replaceLeaf(c, targetType, oldText, newText)) + } + +local function transformFirst( + node: syntax.Node, + targetType: syntax.NodeType, + transform: (syntax.Node) -> syntax.Node, +): syntax.Node = + if (node.type == targetType) + transform.apply(node) + else + (node) { + children = node.children.map((c) -> transformFirst(c, targetType, transform)) + } + +facts { + ["empty module"] { + roundTrip("") == "\n" + } + + ["simple property"] { + roundTrip("x = 1") == "x = 1\n" + } + + ["multiple properties"] { + roundTrip( + """ + x = 1 + y = 2 + """, + ) + == """ + x = 1 + y = 2 + + """ + } + + ["string literals"] { + roundTrip(#"x = "hello""#) == #"x = "hello"\#n"# + roundTrip(#"x = "hello\nworld""#) == #"x = "hello\nworld"\#n"# + roundTrip(#"x = "hello \(name)""#) == #"x = "hello \(name)"\#n"# + } + + ["multiline string"] { + roundTrip( + #""" + x = """ + hello + world + """ + """#, + ) + == #""" + x = + """ + hello + world + """ + + """# + } + + ["comments"] { + roundTrip( + """ + // this is a comment + x = 1 + """, + ) + == """ + // this is a comment + x = 1 + + """ + } + + ["doc comments"] { + roundTrip( + """ + /// A documented property. + x = 1 + """, + ) + == """ + /// A documented property. + x = 1 + + """ + } + + ["class declaration"] { + roundTrip( + """ + class Bird { + name: String + canFly: Boolean = true + } + """, + ) + == """ + class Bird { + name: String + canFly: Boolean = true + } + + """ + } + + ["module declaration"] { + roundTrip("module my.app") == "module my.app\n" + } + + ["imports"] { + roundTrip( + """ + import "foo.pkl" + import "bar.pkl" as myBar + """, + ) + == """ + import "bar.pkl" as myBar + import "foo.pkl" + + """ + } + + ["typealias"] { + roundTrip("typealias Positive = Int(this>0)") == "typealias Positive = Int(this > 0)\n" + } + + ["boolean literals"] { + roundTrip("x = true") == "x = true\n" + roundTrip("x = false") == "x = false\n" + } + + ["numeric literals"] { + roundTrip("x = 42") == "x = 42\n" + roundTrip("x = 3.14") == "x = 3.14\n" + roundTrip("x = 0xFF") == "x = 0xFF\n" + } + + ["null literal"] { + roundTrip("x = null") == "x = null\n" + } + + ["keyword expressions"] { + roundTrip("x = this") == "x = this\n" + roundTrip("x = outer") == "x = outer\n" + roundTrip("x = module") == "x = module\n" + } + + ["binary operators"] { + roundTrip("x = 1 + 2") == "x = 1 + 2\n" + roundTrip("x = a && b") == "x = a && b\n" + roundTrip("x = a is String") == "x = a is String\n" + } + + ["unary operators"] { + roundTrip("x = -1") == "x = -1\n" + roundTrip("x = !flag") == "x = !flag\n" + roundTrip("x = value!!") == "x = value!!\n" + } + + ["if expression"] { + roundTrip("x = if(a) b else c") == "x = if (a) b else c\n" + } + + ["function literal"] { + roundTrip("x = (a) -> a + 1") == "x = (a) -> a + 1\n" + } + + ["method declaration"] { + roundTrip("function greet(name: String): String = name") + == "function greet(name: String): String = name\n" + } + + ["modifiers"] { + roundTrip("hidden x = 1") == "hidden x = 1\n" + roundTrip("local x = 1") == "local x = 1\n" + } + + ["object body"] { + roundTrip( + """ + x { z -> + a = 1 + b = 2 + } + """, + ) + == """ + x { z -> + a = 1 + b = 2 + } + + """ + } + + ["annotations"] { + roundTrip( + """ + @Deprecated { message = "use other" } + x = 1 + """, + ) + == """ + @Deprecated { message = "use other" } + x = 1 + + """ + } + + ["type annotations"] { + roundTrip("x: String = \"hello\"") == "x: String = \"hello\"\n" + roundTrip("x: Int? = null") == "x: Int? = null\n" + roundTrip("x: String|Int = 1") == "x: String | Int = 1\n" + } + + ["for generator"] { + roundTrip( + """ + x { + for (k, v in items) { + [k] = v + } + } + """, + ) + == """ + x { + for (k, v in items) { + [k] = v + } + } + + """ + } + + ["when generator"] { + roundTrip( + """ + x { + when (flag) { + a = 1 + } else { + b = 2 + } + } + """, + ) + == """ + x { + when (flag) { + a = 1 + } else { + b = 2 + } + } + + """ + } + + ["let expression"] { + roundTrip("x = let (y = 1) y + 1") == "x = let (y = 1) y + 1\n" + } + + ["new expression"] { + roundTrip( + """ + x = new Dynamic { + a = 1 + } + """, + ) + == """ + x = new Dynamic { + a = 1 + } + + """ + } + + ["qualified access"] { + roundTrip("x = foo.bar.baz") == "x = foo.bar.baz\n" + } + + ["subscript"] { + roundTrip("x = list[0]") == "x = list[0]\n" + } + + ["parenthesized expression"] { + roundTrip("x = (1 + 2)") == "x = (1 + 2)\n" + } + + ["quoted identifiers"] { + roundTrip("const `my prop` = 0") == "const `my prop` = 0\n" + roundTrip("x = `some`.`path`") == "x = `some`.`path`\n" + roundTrip("module `my mod`.`sub pkg`") == "module `my mod`.`sub pkg`\n" + roundTrip("function `do it`(`the arg`: Int) = `the arg`") + == "function `do it`(`the arg`: Int) = `the arg`\n" + roundTrip( + """ + class `My Class` { + `a prop`: String + } + """, + ) + == """ + class `My Class` { + `a prop`: String + } + + """ + } + + ["modify identifier"] { + local root = parseNode("x = 1") + local modified = replaceLeaf(root, "identifier", "x", "y") + renderer.render(modified) == "y = 1\n" + } + + ["modify modifier"] { + local root = parseNode("hidden x = 1") + local modified = replaceLeaf(root, "modifier", "hidden", "local") + renderer.render(modified) == "local x = 1\n" + } + + ["modify string content"] { + local root = parseNode(#"x = "hello""#) + local modified = replaceLeaf(root, "string_chars", "hello", "world") + renderer.render(modified) == #"x = "world"\#n"# + } + + ["modify int literal"] { + local root = parseNode("x = 42") + local modified = replaceLeaf(root, "int_literal_expr", "42", "99") + renderer.render(modified) == "x = 99\n" + } + + ["modify boolean literal"] { + local root = parseNode("x = true") + local modified = replaceLeaf(root, "bool_literal_expr", "true", "false") + renderer.render(modified) == "x = false\n" + } + + ["modify float literal"] { + local root = parseNode("x = 3.14") + local modified = replaceLeaf(root, "float_literal_expr", "3.14", "2.72") + renderer.render(modified) == "x = 2.72\n" + } + + ["add new modifier"] { + local root = parseNode("local x = 1") + local constModifier = new syntax.Node { + type = "modifier" + text = "const" + } + local modified = + transformFirst(root, "modifier_list", (n) -> (n) { + children = List(constModifier) + n.children + }) + // modifier order is switched by the formatter + renderer.render(modified) == """ + local const x = 1 + + """ + } + + ["a default renderer targets the V2 grammar"] { + local node = parseNode("x = 1") + renderer.render(node) == new syntax.FormattingRenderer { grammarVersion = "V2" }.render(node) + } + + ["renderer honors the grammar version"] { + local node = + parseNode("x = fn(argument1, argument2, argument3, argument4, argument5, argument6, argument7, argument8, argument9)") + new syntax.FormattingRenderer { grammarVersion = "V1" }.render(node) + != new syntax.FormattingRenderer { grammarVersion = "V2" }.render(node) + } + + ["a malformed tree is reported as such"] { + local malformed = new syntax.Node { + type = "object_element" + children = List() + } + test.catch(() -> renderer.render(malformed)) + .contains("Could not render node of type `object_element` as Pkl source") + } +} diff --git a/pkl-core/src/test/files/LanguageSnippetTests/input/syntax/spans.pkl b/pkl-core/src/test/files/LanguageSnippetTests/input/syntax/spans.pkl new file mode 100644 index 000000000..b81e6a0d9 --- /dev/null +++ b/pkl-core/src/test/files/LanguageSnippetTests/input/syntax/spans.pkl @@ -0,0 +1,80 @@ +amends "../snippetTest.pkl" + +import "pkl:syntax" + +local parser = new syntax.Parser {} + +local stringModule = parser.parseModule(""" + x = 1 + + foo { + bar = "baz" + } + """) + +local resourceModule = parser.parseModule(read(".../input-helper/syntax/spans/sample.pkl")) + +local function nestedProperty(mod: syntax.Node): syntax.Node = + mod + .fold(List(), (acc: List, n) -> if (n.type == "object_property") acc.add(n) else acc) + .last + +examples { + ["span of a module parsed from a string"] { + stringModule.span + } + + ["span of a leaf parsed from a string"] { + nestedProperty(stringModule).span + } + + ["span of a module parsed from a resource carries the resource URI"] { + resourceModule.span + } + + ["span of a leaf parsed from a resource carries the resource URI"] { + nestedProperty(resourceModule).span + } +} + +facts { + ["span positions are 1-based"] { + stringModule.span!!.start.line == 1 + stringModule.span!!.start.column == 1 + } + + ["end is exclusive"] { + local span = nestedProperty(stringModule).span!! + span.start.line == 4 + span.start.column == 3 + span.end.line == 4 + span.end.column == 14 + } + + ["a child's span is contained in its parent's"] { + local child = nestedProperty(stringModule) + local parent = child.parent!! + parent.span!!.start.line <= child.span!!.start.line + parent.span!!.end.line >= child.span!!.end.line + } + + ["nodes constructed from scratch have no span"] { + new syntax.Node { type = "int_literal_expr"; text = "42" }.span == null + } + + ["a spanless constructed node still renders"] { + new syntax.FormattingRenderer {} + .render(new syntax.Node { type = "int_literal_expr"; text = "42" }) + .trim() == "42" + } + + ["parsing the same text from a string and a resource differs only in displayUri"] { + local fromString = stringModule.span!! + local fromResource = resourceModule.span!! + fromString.start.line == fromResource.start.line + fromString.start.column == fromResource.start.column + fromString.end.line == fromResource.end.line + fromString.end.column == fromResource.end.column + fromString.displayUri != fromResource.displayUri + } +} diff --git a/pkl-core/src/test/files/LanguageSnippetTests/input/syntax/traversal.pkl b/pkl-core/src/test/files/LanguageSnippetTests/input/syntax/traversal.pkl new file mode 100644 index 000000000..12f7d9c96 --- /dev/null +++ b/pkl-core/src/test/files/LanguageSnippetTests/input/syntax/traversal.pkl @@ -0,0 +1,101 @@ +amends "../snippetTest.pkl" + +import "pkl:syntax" + +local function mod(source: String): syntax.Node = new syntax.Parser {}.parseModule(source) + +local sample: syntax.Node = + mod( + """ + import "foo.pkl" + import "bar.pkl" as baz + + class Point { + x: Int + y: Int + } + + origin = if (useOrigin) new Point { x = 0; y = 0 } else null + + scaled = origin.x * 2 + """, + ) + +facts { + ["fold counts nodes by predicate"] { + sample.fold(0, (acc, n) -> if (n.type == "import") acc + 1 else acc) == 2 + + sample.fold(0, (acc, n) -> if (n.type == "if_expr") acc + 1 else acc) == 1 + + sample.fold(0, (acc, n) -> if (n.type == "when_generator") acc + 1 else acc) == 0 + } + + ["fold accumulates into a collection"] { + sample.fold(List(), (acc, n) -> if (n.type == "identifier") acc.add(n.text) else acc) + == List( + "baz", + "Point", + "x", + "Int", + "y", + "Int", + "origin", + "useOrigin", + "Point", + "x", + "y", + "scaled", + "origin", + "x", + ) + } + + ["fold visits a node before its children (pre-order)"] { + // the module node is visited first + sample.fold(List(), (acc, n) -> acc.add(n.type)).first == "module" + } + + ["terminals have no children"] { + sample.fold(true, (acc, n) -> if (n.isTerminal) acc && n.children.isEmpty else acc) + } + + ["a parsed module is a branch"] { + !sample.isTerminal + } + + ["identifiers are terminals"] { + sample.fold(true, (acc, n) -> if (n.type == "identifier") acc && n.isTerminal else acc) + } + + ["terminals carry text"] { + sample.fold(true, (acc, n) -> if (n.isTerminal) acc && n.text != null else acc) + } + + ["branches carry the text of their entire subtree"] { + sample.fold(true, (acc, n) -> acc && n.text != null) + + // the module node spans the whole source + sample.text == """ + import "foo.pkl" + import "bar.pkl" as baz + + class Point { + x: Int + y: Int + } + + origin = if (useOrigin) new Point { x = 0; y = 0 } else null + + scaled = origin.x * 2 + """ + + // a branch's text includes the whitespace and comments between its children + sample.fold(List(), (acc, n) -> if (n.type == "class_body") acc.add(n.text) else acc) + == List(""" + { + x: Int + y: Int + } + """) + } +} diff --git a/pkl-core/src/test/files/LanguageSnippetTests/output/errors/cannotFindStdLibModule.err b/pkl-core/src/test/files/LanguageSnippetTests/output/errors/cannotFindStdLibModule.err index 2d46bb94f..93e172304 100644 --- a/pkl-core/src/test/files/LanguageSnippetTests/output/errors/cannotFindStdLibModule.err +++ b/pkl-core/src/test/files/LanguageSnippetTests/output/errors/cannotFindStdLibModule.err @@ -27,6 +27,7 @@ pkl:release pkl:semver pkl:settings pkl:shell +pkl:syntax pkl:test pkl:xml pkl:yaml diff --git a/pkl-core/src/test/files/LanguageSnippetTests/output/errors/syntaxParseExpression.err b/pkl-core/src/test/files/LanguageSnippetTests/output/errors/syntaxParseExpression.err new file mode 100644 index 000000000..e4b39bf41 --- /dev/null +++ b/pkl-core/src/test/files/LanguageSnippetTests/output/errors/syntaxParseExpression.err @@ -0,0 +1,16 @@ +–– Pkl Error –– +Could not parse Pkl source. + +x | res = parser.parseExpression("1 +") + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +at syntaxParseExpression#res (file:///$snippetsDir/input/errors/syntaxParseExpression.pkl) + +Unexpected end of file. at (1:4) + +xxx | renderer.renderDocument(value) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +at pkl.base#Module.output.text (pkl:base) + +xxx | if (renderer is BytesRenderer) renderer.renderDocument(value) else text.encodeToBytes("UTF-8") + ^^^^ +at pkl.base#Module.output.bytes (pkl:base) diff --git a/pkl-core/src/test/files/LanguageSnippetTests/output/errors/syntaxParseModule.err b/pkl-core/src/test/files/LanguageSnippetTests/output/errors/syntaxParseModule.err new file mode 100644 index 000000000..0ec2a3c03 --- /dev/null +++ b/pkl-core/src/test/files/LanguageSnippetTests/output/errors/syntaxParseModule.err @@ -0,0 +1,16 @@ +–– Pkl Error –– +Could not parse Pkl source. + +x | parser.parseModule( + ^^^^^^^^^^^^^^^^^^^ +at syntaxParseModule#res (file:///$snippetsDir/input/errors/syntaxParseModule.pkl) + +Unexpected token `}`. Expected `}`. at (3:1 - 3:2) + +xxx | renderer.renderDocument(value) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +at pkl.base#Module.output.text (pkl:base) + +xxx | if (renderer is BytesRenderer) renderer.renderDocument(value) else text.encodeToBytes("UTF-8") + ^^^^ +at pkl.base#Module.output.bytes (pkl:base) diff --git a/pkl-core/src/test/files/LanguageSnippetTests/output/errors/syntaxParseModuleLexerError.err b/pkl-core/src/test/files/LanguageSnippetTests/output/errors/syntaxParseModuleLexerError.err new file mode 100644 index 000000000..be96bb154 --- /dev/null +++ b/pkl-core/src/test/files/LanguageSnippetTests/output/errors/syntaxParseModuleLexerError.err @@ -0,0 +1,16 @@ +–– Pkl Error –– +Could not parse Pkl source. + +x | new syntax.Parser {}.parseModule( + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +at syntaxParseModuleLexerError#mod (file:///$snippetsDir/input/errors/syntaxParseModuleLexerError.pkl) + +Missing `"` delimiter. at (1:9 - 2:1) + +xxx | renderer.renderDocument(value) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +at pkl.base#Module.output.text (pkl:base) + +xxx | if (renderer is BytesRenderer) renderer.renderDocument(value) else text.encodeToBytes("UTF-8") + ^^^^ +at pkl.base#Module.output.bytes (pkl:base) diff --git a/pkl-core/src/test/files/LanguageSnippetTests/output/syntax/query.pcf b/pkl-core/src/test/files/LanguageSnippetTests/output/syntax/query.pcf new file mode 100644 index 000000000..d6371c7ee --- /dev/null +++ b/pkl-core/src/test/files/LanguageSnippetTests/output/syntax/query.pcf @@ -0,0 +1,115 @@ +facts { + ["findChild returns the first match in pre-order"] { + true + true + } + ["findChild throws when nothing matches"] { + true + } + ["findChildOrNull returns null when nothing matches"] { + true + true + } + ["findChild does not consider this node"] { + true + true + } + ["findChildren returns every match in pre-order"] { + true + true + } + ["findChildren searches a match for further matches"] { + true + } + ["findChildOfType agrees with findChild"] { + true + true + } + ["findChildOfType throws when nothing matches"] { + true + } + ["findChildOfTypeOrNull returns null when nothing matches"] { + true + true + } + ["findParent returns the nearest match"] { + true + true + true + } + ["findParentOrNull returns null when nothing matches"] { + true + true + } + ["findParentOfType agrees with findParent"] { + true + true + } + ["findParentOfTypeOrNull returns null when nothing matches"] { + true + true + } + ["findParents returns every match, nearest first"] { + true + } + ["parent lookups do not consider this node"] { + true + true + true + } + ["hasParent and hasParentOfType follow findParent"] { + true + true + true + } + ["replaceChild replaces only the first match"] { + true + } + ["replaceChild leaves the tree unchanged when nothing matches"] { + true + } + ["replaceChildren replaces every match"] { + true + } + ["a replacer sees the node it replaces"] { + true + } + ["a replacement is not searched for further matches"] { + true + } + ["replaceChildOfType agrees with replaceChild"] { + true + } + ["a replacer sees where its node sits in the tree being built"] { + true + } + ["a replace does not change the tree it walks"] { + true + } + ["the root of a replaced tree has no parent"] { + true + true + } + ["only the root of a replaced tree has no parent"] { + true + true + } + ["replacing below a node does not detach it from its own tree"] { + true + } + ["a replacer sees this tree, not the one being built"] { + true + } + ["the children of an amended replacement are parented at the replacement"] { + true + } + ["rebuilds ancestors of a replacement"] { + true + } + ["a match is not searched for further matches"] { + true + } + ["replacement is eager"] { + true + } +} diff --git a/pkl-core/src/test/files/LanguageSnippetTests/output/syntax/render.pcf b/pkl-core/src/test/files/LanguageSnippetTests/output/syntax/render.pcf new file mode 100644 index 000000000..440e9af6f --- /dev/null +++ b/pkl-core/src/test/files/LanguageSnippetTests/output/syntax/render.pcf @@ -0,0 +1,146 @@ +facts { + ["empty module"] { + true + } + ["simple property"] { + true + } + ["multiple properties"] { + true + } + ["string literals"] { + true + true + true + } + ["multiline string"] { + true + } + ["comments"] { + true + } + ["doc comments"] { + true + } + ["class declaration"] { + true + } + ["module declaration"] { + true + } + ["imports"] { + true + } + ["typealias"] { + true + } + ["boolean literals"] { + true + true + } + ["numeric literals"] { + true + true + true + } + ["null literal"] { + true + } + ["keyword expressions"] { + true + true + true + } + ["binary operators"] { + true + true + true + } + ["unary operators"] { + true + true + true + } + ["if expression"] { + true + } + ["function literal"] { + true + } + ["method declaration"] { + true + } + ["modifiers"] { + true + true + } + ["object body"] { + true + } + ["annotations"] { + true + } + ["type annotations"] { + true + true + true + } + ["for generator"] { + true + } + ["when generator"] { + true + } + ["let expression"] { + true + } + ["new expression"] { + true + } + ["qualified access"] { + true + } + ["subscript"] { + true + } + ["parenthesized expression"] { + true + } + ["quoted identifiers"] { + true + true + true + true + true + } + ["modify identifier"] { + true + } + ["modify modifier"] { + true + } + ["modify string content"] { + true + } + ["modify int literal"] { + true + } + ["modify boolean literal"] { + true + } + ["modify float literal"] { + true + } + ["add new modifier"] { + true + } + ["a default renderer targets the V2 grammar"] { + true + } + ["renderer honors the grammar version"] { + true + } + ["a malformed tree is reported as such"] { + true + } +} diff --git a/pkl-core/src/test/files/LanguageSnippetTests/output/syntax/spans.pcf b/pkl-core/src/test/files/LanguageSnippetTests/output/syntax/spans.pcf new file mode 100644 index 000000000..ac386f891 --- /dev/null +++ b/pkl-core/src/test/files/LanguageSnippetTests/output/syntax/spans.pcf @@ -0,0 +1,91 @@ +facts { + ["span positions are 1-based"] { + true + true + } + ["end is exclusive"] { + true + true + true + true + } + ["a child's span is contained in its parent's"] { + true + true + } + ["nodes constructed from scratch have no span"] { + true + } + ["a spanless constructed node still renders"] { + true + } + ["parsing the same text from a string and a resource differs only in displayUri"] { + true + true + true + true + true + } +} +examples { + ["span of a module parsed from a string"] { + new { + start { + line = X + column = 1 + displayUri = "" + } + end { + line = X + column = 2 + displayUri = "" + } + displayUri = "" + } + } + ["span of a leaf parsed from a string"] { + new { + start { + line = X + column = 3 + displayUri = "" + } + end { + line = X + column = 14 + displayUri = "" + } + displayUri = "" + } + } + ["span of a module parsed from a resource carries the resource URI"] { + new { + start { + line = X + column = 1 + displayUri = "file:///$snippetsDir/input-helper/syntax/spans/sample.pkl" + } + end { + line = X + column = 2 + displayUri = "file:///$snippetsDir/input-helper/syntax/spans/sample.pkl" + } + displayUri = "file:///$snippetsDir/input-helper/syntax/spans/sample.pkl" + } + } + ["span of a leaf parsed from a resource carries the resource URI"] { + new { + start { + line = X + column = 3 + displayUri = "file:///$snippetsDir/input-helper/syntax/spans/sample.pkl" + } + end { + line = X + column = 14 + displayUri = "file:///$snippetsDir/input-helper/syntax/spans/sample.pkl" + } + displayUri = "file:///$snippetsDir/input-helper/syntax/spans/sample.pkl" + } + } +} diff --git a/pkl-core/src/test/files/LanguageSnippetTests/output/syntax/traversal.pcf b/pkl-core/src/test/files/LanguageSnippetTests/output/syntax/traversal.pcf new file mode 100644 index 000000000..395399007 --- /dev/null +++ b/pkl-core/src/test/files/LanguageSnippetTests/output/syntax/traversal.pcf @@ -0,0 +1,30 @@ +facts { + ["fold counts nodes by predicate"] { + true + true + true + } + ["fold accumulates into a collection"] { + true + } + ["fold visits a node before its children (pre-order)"] { + true + } + ["terminals have no children"] { + true + } + ["a parsed module is a branch"] { + true + } + ["identifiers are terminals"] { + true + } + ["terminals carry text"] { + true + } + ["branches carry the text of their entire subtree"] { + true + true + true + } +} diff --git a/pkl-formatter/src/main/java/org/pkl/formatter/Builder.java b/pkl-formatter/src/main/java/org/pkl/formatter/Builder.java index f1d82638d..1db2f0316 100644 --- a/pkl-formatter/src/main/java/org/pkl/formatter/Builder.java +++ b/pkl-formatter/src/main/java/org/pkl/formatter/Builder.java @@ -42,6 +42,7 @@ final class Builder { } FormatNode format(Node node) { + if (node.type.isTerminal()) return new Text(node.text(source)); return switch (node.type) { case MODULE -> formatModule(node); case DOC_COMMENT, @@ -56,10 +57,10 @@ final class Builder { case DOC_COMMENT_LINE -> formatDocComment(node); case LINE_COMMENT, BLOCK_COMMENT, - TERMINAL, MODIFIER, IDENTIFIER, STRING_CHARS, + STRING_CONSTANT, STRING_ESCAPE, INT_LITERAL_EXPR, FLOAT_LITERAL_EXPR, @@ -115,11 +116,11 @@ final class Builder { formatClassPropertyHeaderBegin(node); case CLASS_PROPERTY_BODY, OBJECT_PROPERTY_BODY -> formatClassPropertyBody(node); case CLASS_METHOD, OBJECT_METHOD -> formatClassMethod(node); - case CLASS_METHOD_HEADER -> formatClassMethodHeader(node); - case CLASS_METHOD_BODY -> formatClassMethodBody(node); + case METHOD_HEADER -> formatClassMethodHeader(node); + case METHOD_BODY -> formatClassMethodBody(node); case OBJECT_BODY -> formatObjectBody(node); case OBJECT_ELEMENT -> format(node.children.get(0)); // has a single element - case OBJECT_ENTRY_HEADER -> formatObjectEntryHeader(node); + case OBJECT_ENTRY_HEADER, MEMBER_PREDICATE_HEADER -> formatObjectEntryHeader(node); case FOR_GENERATOR -> formatForGenerator(node); case FOR_GENERATOR_HEADER -> formatForGeneratorHeader(node); case FOR_GENERATOR_HEADER_DEFINITION -> formatForGeneratorHeaderDefinition(node); @@ -147,7 +148,6 @@ final class Builder { case SUBSCRIPT_EXPR, SUPER_SUBSCRIPT_EXPR -> formatSubscriptExpr(node); case TRACE_EXPR, THROW_EXPR, READ_EXPR -> formatTraceThrowReadExpr(node); case PARENTHESIZED_EXPR -> formatParenthesizedExpr(node); - case PARENTHESIZED_EXPR_ELEMENTS -> formatParenthesizedExprElements(node); case LET_EXPR -> formatLetExpr(node); case LET_PARAMETER_DEFINITION -> formatLetParameterDefinition(node); case LET_PARAMETER -> formatLetParameter(node); @@ -156,7 +156,7 @@ final class Builder { case CONSTRAINED_TYPE -> formatConstrainedType(node); case UNION_TYPE -> formatUnionType(node); case FUNCTION_TYPE -> formatFunctionType(node); - case STRING_CONSTANT_TYPE -> format(node.children.get(0)); + case STRING_LITERAL_TYPE -> format(node.children.get(0)); case PARENTHESIZED_TYPE -> formatParenthesizedType(node); case PARENTHESIZED_TYPE_ELEMENTS -> formatParenthesizedTypeElements(node); default -> throw new RuntimeException("Unknown node type: " + node.type); @@ -231,7 +231,7 @@ final class Builder { first.add(format(node.children.get(0))); first.add(line()); var rest = node.children.subList(1, node.children.size()); - var nodes = formatGeneric(rest, (n1, next) -> n1.type == NodeType.TERMINAL ? null : line()); + var nodes = formatGeneric(rest, (n1, next) -> n1.type.isTerminal() ? null : line()); first.add(new Indent(nodes)); return new Group(newId(), first); } @@ -397,7 +397,7 @@ final class Builder { formatGenericWithGen( node.children, spaceOrLine(), - (n, next) -> isTerminal(n, "import") ? format(n) : indent(format(n)))); + (n, next) -> n.type == NodeType.IMPORT_KEYWORD ? format(n) : indent(format(n)))); } private FormatNode formatAnnotation(Node node) { @@ -504,12 +504,12 @@ final class Builder { private FormatNode formatClassMethod(Node node) { var prefixes = new ArrayList(); List methodNodes; - if (node.children.get(0).type == NodeType.CLASS_METHOD_HEADER) { + if (node.children.get(0).type == NodeType.METHOD_HEADER) { methodNodes = node.children; } else { var idx = -1; for (var i = 0; i < node.children.size(); i++) { - if (node.children.get(i).type == NodeType.CLASS_METHOD_HEADER) { + if (node.children.get(i).type == NodeType.METHOD_HEADER) { idx = i; break; } @@ -525,7 +525,7 @@ final class Builder { // Separate header (before =) and body (= and after) var bodyIdx = -1; for (var i = 0; i < methodNodes.size(); i++) { - if (methodNodes.get(i).type == NodeType.CLASS_METHOD_BODY) { + if (methodNodes.get(i).type == NodeType.METHOD_BODY) { bodyIdx = i - 1; break; } @@ -594,8 +594,8 @@ final class Builder { formatGeneric( node.children, (prev, next) -> { - if (isTerminal(prev, "(") || isTerminal(next, ")")) { - if (isTerminal(next, ")")) { + if (insideParens(prev, next)) { + if (next.type == NodeType.RPAREN) { // trailing comma if (grammarVersion == GrammarVersion.V1) { return line(); @@ -618,9 +618,9 @@ final class Builder { formatGenericWithGen( node.children, (prev, next) -> { - if (isTerminal(prev, "(") || isTerminal(next, ")")) { + if (insideParens(prev, next)) { var lineNode = hasTrailingLambda ? Empty.INSTANCE : line(); - if (isTerminal(next, ")") && !hasTrailingLambda) { + if (next.type == NodeType.RPAREN && !hasTrailingLambda) { // trailing comma if (grammarVersion == GrammarVersion.V1) { return lineNode; @@ -642,7 +642,7 @@ final class Builder { private FormatNode formatArgumentListElements( Node node, boolean hasTrailingLambda, boolean twoBy2) { var children = node.children; - var shouldMultiline = shouldMultilineNodes(node, n -> isTerminal(n, ",")); + var shouldMultiline = shouldMultilineNodes(node, n -> n.type == NodeType.COMMA); BiFunction sep = (prev, next) -> shouldMultiline ? forceSpaceyLine() : spaceOrLine(); if (twoBy2) { @@ -698,7 +698,7 @@ final class Builder { while (!node.children.isEmpty()) { node = node.children.get(node.children.size() - 1); } - return isTerminalSingle(node, "}"); + return node.type == NodeType.RBRACE; } /** @@ -741,7 +741,8 @@ final class Builder { return false; } } - return properArgCount <= 2 || !shouldMultilineNodes(elementsNode, n -> isTerminal(n, ",")); + return properArgCount <= 2 + || !shouldMultilineNodes(elementsNode, n -> n.type == NodeType.COMMA); } private List pairArguments(List nodes) { @@ -749,7 +750,7 @@ final class Builder { var tmp = new ArrayList(); var commas = 0; for (var node : nodes) { - if (isTerminalSingle(node, ",")) { + if (node.type == NodeType.COMMA) { commas++; if (commas == 2) { var suffixes = new ArrayList(); @@ -791,8 +792,8 @@ final class Builder { formatGeneric( node.children, (prev, next) -> { - if (isTerminal(prev, "<") || isTerminal(next, ">")) { - if (isTerminal(next, ">")) { + if (prev.type == NodeType.LT || next.type == NodeType.GT) { + if (next.type == NodeType.GT) { // trailing comma if (grammarVersion == GrammarVersion.V1) { return new Line(); @@ -825,14 +826,14 @@ final class Builder { node.children, (prev, next) -> { if (next.type == NodeType.OBJECT_PARAMETER_LIST) return Empty.INSTANCE; - if (isTerminal(prev, "{") || isTerminal(next, "}")) { + if (prev.type == NodeType.LBRACE || next.type == NodeType.RBRACE) { var lines = linesBetween(prev, next); return lines == 0 ? spaceOrLine() : forceSpaceyLine(); } return spaceOrLine(); }, (n, next) -> - n.type == NodeType.OBJECT_MEMBER_LIST + n.type == NodeType.OBJECT_MEMBER_ELEMENTS ? formatObjectMemberList(n, groupId) : format(n)); return new Group(groupId, nodes); @@ -869,9 +870,7 @@ final class Builder { private FormatNode formatForGeneratorHeader(Node node) { var nodes = - formatGeneric( - node.children, - (prev, next) -> isTerminal(prev, "(") || isTerminal(next, ")") ? line() : null); + formatGeneric(node.children, (prev, next) -> insideParens(prev, next) ? line() : null); return new Group(newId(), nodes); } @@ -897,8 +896,9 @@ final class Builder { node.children, (prev, next) -> prev.type == NodeType.WHEN_GENERATOR_HEADER - || isTerminal(prev, "when", "else") - || isTerminal(next, "else") + || prev.type == NodeType.WHEN_KEYWORD + || prev.type == NodeType.ELSE_KEYWORD + || next.type == NodeType.ELSE_KEYWORD ? Space.INSTANCE : spaceOrLine()); return new Group(newId(), nodes); @@ -908,9 +908,8 @@ final class Builder { var nodes = formatGenericWithGen( node.children, - (prev, next) -> isTerminal(prev, "(") || isTerminal(next, ")") ? line() : spaceOrLine(), - (n, next) -> - !n.type.isAffix() && n.type != NodeType.TERMINAL ? indent(format(n)) : format(n)); + (prev, next) -> insideParens(prev, next) ? line() : spaceOrLine(), + (n, next) -> isProper(n) ? indent(format(n)) : format(n)); return new Group(newId(), nodes); } @@ -932,7 +931,7 @@ final class Builder { while (cursor.hasNext()) { if (isInStringInterpolation) { var prevNoNewlines = noNewlines; - var elems = cursor.takeUntilBefore(n -> isTerminalSingle(n, ")")); + var elems = cursor.takeUntilBefore(n -> n.type == NodeType.RPAREN); noNewlines = !isMultilineList(elems); //noinspection ConstantValue assert prev != null; @@ -946,7 +945,7 @@ final class Builder { continue; } var elem = cursor.next(); - if (elem.type == NodeType.TERMINAL && text(elem).endsWith("(")) { + if (elem.type == NodeType.INTERPOLATION_START) { isInStringInterpolation = true; } var formatted = format(elem); @@ -996,9 +995,7 @@ final class Builder { private FormatNode formatIfCondition(Node node) { var nodes = formatGeneric( - node.children, - (prev, next) -> - isTerminal(prev, "(") || isTerminal(next, ")") ? line() : spaceOrLine()); + node.children, (prev, next) -> insideParens(prev, next) ? line() : spaceOrLine()); return new Group(newId(), nodes); } @@ -1029,21 +1026,19 @@ final class Builder { } private FormatNode formatParenthesizedExpr(Node node) { - if (node.children.size() == 2) return new Text("()"); var nodes = formatGenericWithGen( node.children, - (prev, next) -> isTerminal(prev, "(") || isTerminal(next, ")") ? line() : spaceOrLine(), - (n, next) -> n.type.isExpression() ? indent(format(n)) : format(n)); + (prev, next) -> insideParens(prev, next) ? line() : spaceOrLine(), + (n, next) -> + (n.type == NodeType.LPAREN || n.type == NodeType.RPAREN) + ? format(n) + : indent(format(n))); return new Group(newId(), nodes); } - private FormatNode formatParenthesizedExprElements(Node node) { - return indent(new Group(newId(), formatGeneric(node.children, (FormatNode) null))); - } - private FormatNode formatFunctionLiteralExpr(Node node) { - var splitResult = splitOn(node.children, n -> isTerminalSingle(n, "->")); + var splitResult = splitOn(node.children, n -> n.type == NodeType.ARROW); var params = splitResult[0]; var rest = splitResult[1]; Node bodyNode = null; @@ -1112,9 +1107,7 @@ final class Builder { private FormatNode formatLetParameterDefinition(Node node) { var nodes = formatGeneric( - node.children, - (prev, next) -> - isTerminal(prev, "(") || isTerminal(next, ")") ? line() : spaceOrLine()); + node.children, (prev, next) -> insideParens(prev, next) ? line() : spaceOrLine()); return new Group(newId(), nodes); } @@ -1160,7 +1153,7 @@ final class Builder { var nodes = formatGenericWithGen( node.children, - (prev, next) -> isTerminal(prev, "(") || isTerminal(next, ")") ? line() : null, + (prev, next) -> insideParens(prev, next) ? line() : null, (n, next) -> n.type.isExpression() ? indent(format(n)) : format(n)); return new Group(newId(), nodes); } @@ -1183,8 +1176,8 @@ final class Builder { formatGeneric( node.children, (prev, next) -> { - if (isTerminal(next, "|")) return spaceOrLine(); - if (isTerminal(prev, "|")) return Space.INSTANCE; + if (next.type == NodeType.UNION) return spaceOrLine(); + if (prev.type == NodeType.UNION) return Space.INSTANCE; return null; }); return new Group(newId(), indentAfterFirstNewline(nodes, false)); @@ -1194,7 +1187,7 @@ final class Builder { var nodes = formatGenericWithGen( node.children, - (prev, next) -> isTerminal(prev, "(") || isTerminal(next, ")") ? line() : spaceOrLine(), + (prev, next) -> insideParens(prev, next) ? line() : spaceOrLine(), (n, next) -> next == null ? indent(format(n)) : format(n)); return new Group(newId(), nodes); } @@ -1204,9 +1197,7 @@ final class Builder { var groupId = newId(); var nodes = formatGeneric( - node.children, - (prev, next) -> - isTerminal(prev, "(") || isTerminal(next, ")") ? line() : spaceOrLine()); + node.children, (prev, next) -> insideParens(prev, next) ? line() : spaceOrLine()); return new Group(groupId, nodes); } @@ -1288,9 +1279,7 @@ final class Builder { var regularImports = new ArrayList(); var globImports = new ArrayList(); for (var entry : allImportsWithComments) { - var terminalNode = entry.importNode.findChildByType(NodeType.TERMINAL); - var terminalText = terminalNode != null ? terminalNode.text(source) : null; - if ("import*".equals(terminalText)) { + if (entry.importNode.findChildByType(NodeType.IMPORT_STAR_KEYWORD) != null) { globImports.add(entry); } else { regularImports.add(entry); @@ -1417,7 +1406,7 @@ final class Builder { } private boolean isSemicolon(Node node) { - return node.type.isAffix() && text(node).equals(";"); + return node.type == NodeType.SEMICOLON; } /** Groups all non prefixes (comments, doc comments, annotations) of this node together. */ @@ -1446,9 +1435,9 @@ final class Builder { } private String getImportUrl(Node node) { - var strChars = node.findChildByType(NodeType.STRING_CHARS); - assert strChars != null; - var txt = strChars.text(source); + var strConst = node.findChildByType(NodeType.STRING_CONSTANT); + assert strConst != null; + var txt = strConst.text(source); return txt.substring(1, txt.length() - 1); } @@ -1468,7 +1457,7 @@ final class Builder { return linesBetween(prev, next) > 1 ? TWO_NEWLINES : mustForceLine(); } if (hasTrailingAffix(prev, next)) { - if (isTerminal(prev, "(") && startsWithBlockComment(next)) return line(); + if (prev.type == NodeType.LPAREN && startsWithBlockComment(next)) return line(); return Space.INSTANCE; } if (prev.type == NodeType.DOC_COMMENT) return mustForceLine(); @@ -1479,13 +1468,11 @@ final class Builder { if (prev.type == NodeType.BLOCK_COMMENT) { return linesBetween(prev, next) > 0 ? forceSpaceyLine() : Space.INSTANCE; } - if (EMPTY_SUFFIXES.contains(next.type) - || isTerminal(prev, "[", "!", "@", "[[") - || isTerminal(next, "]", "?", ",")) { + if (EMPTY_SUFFIXES.contains(next.type) || EMPTY_PREFIXES.contains(prev.type)) { return Empty.INSTANCE; } - if (isTerminal(prev, "class", "function", "new") - || isTerminal(next, "=", "{", "->", "class", "function") + if (SPACE_PREFIXES.contains(prev.type) + || SPACE_SUFFIXES.contains(next.type) || next.type == NodeType.OBJECT_BODY || prev.type == NodeType.MODIFIER_LIST) { return Space.INSTANCE; @@ -1659,17 +1646,8 @@ final class Builder { return node.text(source); } - private boolean isTerminal(Node node, String... texts) { - if (node.type != NodeType.TERMINAL) return false; - var t = node.text(source); - for (var text : texts) { - if (t.equals(text)) return true; - } - return false; - } - - private boolean isTerminalSingle(Node node, String text) { - return node.type == NodeType.TERMINAL && node.text(source).equals(text); + private static boolean insideParens(Node prev, Node next) { + return prev.type == NodeType.LPAREN || next.type == NodeType.RPAREN; } private int newId() { @@ -1704,7 +1682,7 @@ final class Builder { // returns true if this node is not an affix or terminal private static boolean isProper(Node node) { - return !node.type.isAffix() && node.type != NodeType.TERMINAL; + return !node.type.isAffix() && !node.type.isTerminal(); } private static boolean isMultiline(Node node) { @@ -1805,7 +1783,24 @@ final class Builder { NodeType.TYPE_ARGUMENT_LIST, NodeType.TYPE_ANNOTATION, NodeType.TYPE_PARAMETER_LIST, - NodeType.PARAMETER_LIST); + NodeType.PARAMETER_LIST, + NodeType.RBRACK, + NodeType.QUESTION, + NodeType.COMMA); + + private static final EnumSet EMPTY_PREFIXES = + EnumSet.of(NodeType.LBRACK, NodeType.NOT, NodeType.AT, NodeType.LPRED); + + private static final EnumSet SPACE_PREFIXES = + EnumSet.of(NodeType.CLASS_KEYWORD, NodeType.FUNCTION_KEYWORD, NodeType.NEW_KEYWORD); + + private static final EnumSet SPACE_SUFFIXES = + EnumSet.of( + NodeType.ASSIGN, + NodeType.LBRACE, + NodeType.ARROW, + NodeType.CLASS_KEYWORD, + NodeType.FUNCTION_KEYWORD); private static final EnumSet SAME_LINE_EXPRS = EnumSet.of(NodeType.NEW_EXPR, NodeType.AMENDS_EXPR, NodeType.FUNCTION_LITERAL_EXPR); @@ -1819,8 +1814,8 @@ final class Builder { @Override public int compare(Node o1, Node o2) { - var import1 = o1.findChildByType(NodeType.STRING_CHARS); - var import2 = o2.findChildByType(NodeType.STRING_CHARS); + var import1 = o1.findChildByType(NodeType.STRING_CONSTANT); + var import2 = o2.findChildByType(NodeType.STRING_CONSTANT); if (import1 == null || import2 == null) { // should never happen throw new RuntimeException("ImportComparator: not an import"); diff --git a/pkl-formatter/src/main/java/org/pkl/formatter/Formatter.java b/pkl-formatter/src/main/java/org/pkl/formatter/Formatter.java index 6e9487a76..8f37f9d00 100644 --- a/pkl-formatter/src/main/java/org/pkl/formatter/Formatter.java +++ b/pkl-formatter/src/main/java/org/pkl/formatter/Formatter.java @@ -21,6 +21,7 @@ import java.nio.file.Files; import java.nio.file.Path; import java.util.List; import org.pkl.parser.GenericParser; +import org.pkl.parser.syntax.generic.Node; /** * A formatter for Pkl files that applies canonical formatting rules. @@ -111,9 +112,43 @@ public class Formatter { format(sb.toString(), output); } + /** + * Format the given Pkl AST as text. + * + *

The AST terminal nodes should have their text property set before calling format, as this + * method does not have access to the original source. + * + * @param ast the Pkl module node to format + * @return the formatted Pkl source code + */ + public String format(Node ast) { + var output = new StringBuilder(); + format(ast, output); + return output.toString(); + } + + /** + * Formats the given Pkl AST as text. + * + *

The AST terminal nodes should have their text property set before calling format, as this + * method does not have access to the original source. + * + *

It is the caller's responsibility to close {@code output}, if applicable. + * + * @param ast the Pkl module node to format + * @param output the formatted Pkl source code + * @throws java.io.UncheckedIOException if an I/O error occurs during writing + */ + public void format(Node ast, Appendable output) { + generate(new Builder("", grammarVersion).format(ast), output); + } + private void format(String input, Appendable output) { var ast = new GenericParser().parseModule(input); - var formatAst = new Builder(input, grammarVersion).format(ast); + generate(new Builder(input, grammarVersion).format(ast), output); + } + + private static void generate(FormatNode formatAst, Appendable output) { // force a line at the end of the file var nodes = new Nodes(List.of(formatAst, ForceLine.INSTANCE)); new Generator(output).generate(nodes); diff --git a/pkl-formatter/src/test/kotlin/org/pkl/formatter/FormatterTest.kt b/pkl-formatter/src/test/kotlin/org/pkl/formatter/FormatterTest.kt index 2bbc9c158..ae8cf53f3 100644 --- a/pkl-formatter/src/test/kotlin/org/pkl/formatter/FormatterTest.kt +++ b/pkl-formatter/src/test/kotlin/org/pkl/formatter/FormatterTest.kt @@ -25,7 +25,9 @@ import org.junit.jupiter.api.assertThrows import org.pkl.commons.readString import org.pkl.commons.test.FileTestUtils import org.pkl.commons.test.failWithDiff +import org.pkl.parser.GenericParser import org.pkl.parser.GenericParserError +import org.pkl.parser.syntax.generic.Node class FormatterTest { @@ -128,4 +130,21 @@ class FormatterTest { Formatter().format(input, output) assertThat(output.toString()).isEqualTo("x = 42\n") } + + @Test + fun `format AST into Appendable`() { + val source = " x = 42" + val ast = GenericParser().parseModule(source) + fun setText(node: Node) { + if (node.children.isEmpty()) { + node.text(source.toCharArray()) + } else { + node.children.forEach(::setText) + } + } + setText(ast) + val output = StringBuilder() + Formatter().format(ast, output) + assertThat(output.toString()).isEqualTo("x = 42\n") + } } diff --git a/pkl-parser/src/main/java/org/pkl/parser/BaseParserVisitor.java b/pkl-parser/src/main/java/org/pkl/parser/BaseParserVisitor.java index 2b1fbfd6f..dee59df0e 100644 --- a/pkl-parser/src/main/java/org/pkl/parser/BaseParserVisitor.java +++ b/pkl-parser/src/main/java/org/pkl/parser/BaseParserVisitor.java @@ -81,7 +81,7 @@ import org.pkl.parser.syntax.Type.ModuleType; import org.pkl.parser.syntax.Type.NothingType; import org.pkl.parser.syntax.Type.NullableType; import org.pkl.parser.syntax.Type.ParenthesizedType; -import org.pkl.parser.syntax.Type.StringConstantType; +import org.pkl.parser.syntax.Type.StringLiteralType; import org.pkl.parser.syntax.Type.ThisType; import org.pkl.parser.syntax.Type.UnionType; import org.pkl.parser.syntax.Type.UnknownType; @@ -114,7 +114,7 @@ public abstract class BaseParserVisitor implements ParserVisitor { } @Override - public T visitStringConstantType(StringConstantType type) { + public T visitStringLiteralType(StringLiteralType type) { return visitChildren(type); } diff --git a/pkl-parser/src/main/java/org/pkl/parser/GenericParser.java b/pkl-parser/src/main/java/org/pkl/parser/GenericParser.java index 3cb01dc64..a770c6a47 100644 --- a/pkl-parser/src/main/java/org/pkl/parser/GenericParser.java +++ b/pkl-parser/src/main/java/org/pkl/parser/GenericParser.java @@ -21,4 +21,8 @@ public final class GenericParser { public Node parseModule(String source) { return new GenericParserImpl(source).parseModule(); } + + public Node parseExpressionInput(String source) { + return new GenericParserImpl(source).parseExpressionInput(); + } } diff --git a/pkl-parser/src/main/java/org/pkl/parser/GenericParserImpl.java b/pkl-parser/src/main/java/org/pkl/parser/GenericParserImpl.java index 46b6da538..93ec8793e 100644 --- a/pkl-parser/src/main/java/org/pkl/parser/GenericParserImpl.java +++ b/pkl-parser/src/main/java/org/pkl/parser/GenericParserImpl.java @@ -17,6 +17,7 @@ package org.pkl.parser; import java.util.ArrayList; import java.util.List; +import java.util.Objects; import java.util.function.Supplier; import org.jspecify.annotations.Nullable; import org.pkl.parser.syntax.Operator; @@ -37,10 +38,14 @@ class GenericParserImpl { GenericParserImpl(String source) { this.lexer = new Lexer(source); - while (true) { - var ft = new FullToken(lexer.next(), lexer.fullSpan(), lexer.getNewLinesBetween()); - tokens.add(ft); - if (ft.token == Token.EOF) break; + try { + while (true) { + var ft = new FullToken(lexer.next(), lexer.fullSpan(), lexer.getNewLinesBetween()); + tokens.add(ft); + if (ft.token == Token.EOF) break; + } + } catch (ParserError e) { + throw parserError(Objects.requireNonNull(e.getMessage()), lexer.fullSpan()); } _lookahead = tokens.get(cursor); lookahead = _lookahead.token; @@ -102,6 +107,16 @@ class GenericParserImpl { return new Node(NodeType.MODULE, nodes); } + Node parseExpressionInput() { + ff(); + var expr = parseExpr(); + ff(); + if (lookahead != Token.EOF) { + throw parserError("unexpectedToken", _lookahead.text(lexer), "end of file"); + } + return expr; + } + private Node parseModuleDecl(List preChildren) { var headerParts = getHeaderParts(preChildren); var children = new ArrayList<>(headerParts.preffixes); @@ -139,7 +154,7 @@ class GenericParserImpl { children.add(parseIdentifier()); while (lookahead() == Token.DOT) { ff(children); - children.add(new Node(NodeType.TERMINAL, next().span)); + children.add(makeTerminal(next())); ff(children); children.add(parseIdentifier()); } @@ -222,13 +237,13 @@ class GenericParserImpl { headers.add(makeTerminal(next())); ff(headers); headers.add(parseIdentifier()); - ff(headers); - if (lookahead == Token.LT) { - headers.add(parseTypeParameterList()); + if (lookahead() == Token.LT) { ff(headers); + headers.add(parseTypeParameterList()); } - expect(Token.ASSIGN, headers, "unexpectedToken", "="); children.add(new Node(NodeType.TYPEALIAS_HEADER, headers)); + ff(children); + expect(Token.ASSIGN, children, "unexpectedToken", "="); var body = new ArrayList(); ff(body); body.add(parseType()); @@ -307,13 +322,13 @@ class GenericParserImpl { } headerBegin.add(parseIdentifier()); header.add(new Node(NodeType.CLASS_PROPERTY_HEADER_BEGIN, headerBegin)); + children.add(new Node(NodeType.CLASS_PROPERTY_HEADER, header)); var hasTypeAnnotation = false; if (lookahead() == Token.COLON) { - ff(header); - header.add(parseTypeAnnotation()); + ff(children); + children.add(parseTypeAnnotation()); hasTypeAnnotation = true; } - children.add(new Node(NodeType.CLASS_PROPERTY_HEADER, header)); if (lookahead() == Token.ASSIGN) { ff(children); children.add(makeTerminal(next())); @@ -343,7 +358,7 @@ class GenericParserImpl { expect(Token.FUNCTION, headers, "unexpectedToken", "function"); ff(headers); headers.add(parseIdentifier()); - children.add(new Node(NodeType.CLASS_METHOD_HEADER, headers)); + children.add(new Node(NodeType.METHOD_HEADER, headers)); ff(children); if (lookahead == Token.LT) { children.add(parseTypeParameterList()); @@ -360,7 +375,7 @@ class GenericParserImpl { var body = new ArrayList(); ff(body); body.add(parseExpr()); - children.add(new Node(NodeType.CLASS_METHOD_BODY, body)); + children.add(new Node(NodeType.METHOD_BODY, body)); } return new Node(NodeType.CLASS_METHOD, children); } @@ -391,7 +406,7 @@ class GenericParserImpl { ff(members); } if (!members.isEmpty()) { - children.add(new Node(NodeType.OBJECT_MEMBER_LIST, members)); + children.add(new Node(NodeType.OBJECT_MEMBER_ELEMENTS, members)); } children.add(makeTerminal(next())); // RBRACE return new Node(NodeType.OBJECT_BODY, children); @@ -484,13 +499,13 @@ class GenericParserImpl { } headerBegin.add(parseIdentifier()); header.add(new Node(NodeType.OBJECT_PROPERTY_HEADER_BEGIN, headerBegin)); + children.add(new Node(NodeType.OBJECT_PROPERTY_HEADER, header)); var hasTypeAnnotation = false; if (lookahead() == Token.COLON) { - ff(header); - header.add(parseTypeAnnotation()); + ff(children); + children.add(parseTypeAnnotation()); hasTypeAnnotation = true; } - children.add(new Node(NodeType.OBJECT_PROPERTY_HEADER, header)); if (hasTypeAnnotation || lookahead() == Token.ASSIGN) { ff(children); expect(Token.ASSIGN, children, "unexpectedToken", "="); @@ -515,7 +530,7 @@ class GenericParserImpl { expect(Token.FUNCTION, headers, "unexpectedToken", "function"); ff(headers); headers.add(parseIdentifier()); - children.add(new Node(NodeType.CLASS_METHOD_HEADER, headers)); + children.add(new Node(NodeType.METHOD_HEADER, headers)); ff(children); if (lookahead == Token.LT) { children.add(parseTypeParameterList()); @@ -531,20 +546,22 @@ class GenericParserImpl { var body = new ArrayList(); ff(body); body.add(parseExpr()); - children.add(new Node(NodeType.CLASS_METHOD_BODY, body)); + children.add(new Node(NodeType.METHOD_BODY, body)); return new Node(NodeType.OBJECT_METHOD, children); } private Node parseMemberPredicate() { var children = new ArrayList(); - children.add(makeTerminal(next())); - ff(children); - children.add(parseExpr()); - ff(children); + var header = new ArrayList(); + header.add(makeTerminal(next())); + ff(header); + header.add(parseExpr()); + ff(header); var firstBrack = expect(Token.RBRACK, "unexpectedToken", "]]"); - children.add(makeTerminal(firstBrack)); + header.add(makeTerminal(firstBrack)); var secondbrack = expect(Token.RBRACK, "unexpectedToken", "]]"); - children.add(makeTerminal(secondbrack)); + header.add(makeTerminal(secondbrack)); + children.add(new Node(NodeType.MEMBER_PREDICATE_HEADER, header)); if (firstBrack.span.charIndex() != secondbrack.span.charIndex() - 1) { // There shouldn't be any whitespace between the first and second ']'. var span = firstBrack.span.endWith(secondbrack.span); @@ -569,16 +586,14 @@ class GenericParserImpl { ff(header); header.add(parseExpr()); expect(Token.RBRACK, header, "unexpectedToken", "]"); - if (lookahead() == Token.ASSIGN) { - ff(header); - header.add(makeTerminal(next())); - children.add(new Node(NodeType.OBJECT_ENTRY_HEADER, header)); + children.add(new Node(NodeType.OBJECT_ENTRY_HEADER, header)); + ff(children); + if (lookahead == Token.ASSIGN) { + children.add(makeTerminal(next())); ff(children); children.add(parseExpr()); return new Node(NodeType.OBJECT_ENTRY, children); } - children.add(new Node(NodeType.OBJECT_ENTRY_HEADER, header)); - ff(children); children.addAll(parseBodyList()); return new Node(NodeType.OBJECT_ENTRY, children); } @@ -937,7 +952,9 @@ class GenericParserImpl { var token = next().token; ff(); if (token == Token.RPAREN) { - return lookahead == Token.ARROW; + // `()` is only valid as an empty parameter list + // let `parseFunctionLiteral` report the missing `->` + return true; } if (token == Token.UNDERSCORE) { return true; @@ -1072,16 +1089,9 @@ class GenericParserImpl { private Node parseParenthesizedExpr() { var children = new ArrayList(); expect(Token.LPAREN, children, "unexpectedToken", "("); - if (lookahead() == Token.RPAREN) { - ff(children); - children.add(makeTerminal(next())); - return new Node(NodeType.PARENTHESIZED_EXPR, children); - } - var elements = new ArrayList(); - ff(elements); - elements.add(parseExpr(")")); - ff(elements); - children.add(new Node(NodeType.PARENTHESIZED_EXPR_ELEMENTS, elements)); + ff(children); + children.add(parseExpr(")")); + ff(children); expect(Token.RPAREN, children, "unexpectedToken", ")"); return new Node(NodeType.PARENTHESIZED_EXPR, children); } @@ -1211,7 +1221,7 @@ class GenericParserImpl { yield new Node(NodeType.DECLARED_TYPE, children); } case STRING_START -> - new Node(NodeType.STRING_CONSTANT_TYPE, List.of(parseStringConstant())); + new Node(NodeType.STRING_LITERAL_TYPE, List.of(parseStringConstant())); default -> { var text = _lookahead.text(lexer); if (expectation != null) { @@ -1381,14 +1391,14 @@ class GenericParserImpl { children.add(makeTerminal(startTk)); while (lookahead != Token.STRING_END) { switch (lookahead) { - case STRING_PART, - STRING_ESCAPE_NEWLINE, + case STRING_PART -> children.add(make(NodeType.STRING_CHARS, next().span)); + case STRING_ESCAPE_NEWLINE, STRING_ESCAPE_TAB, STRING_ESCAPE_QUOTE, STRING_ESCAPE_BACKSLASH, STRING_ESCAPE_RETURN, STRING_ESCAPE_UNICODE -> - children.add(makeTerminal(next())); + children.add(make(NodeType.STRING_ESCAPE, next().span)); case EOF -> { var delimiter = new StringBuilder(startTk.text(lexer)).reverse().toString(); throw parserError("missingDelimiter", delimiter); @@ -1399,7 +1409,7 @@ class GenericParserImpl { } } children.add(makeTerminal(next())); // string end - return new Node(NodeType.STRING_CHARS, children); + return new Node(NodeType.STRING_CONSTANT, children); } private FullToken expect(Token type, String errorKey, Object... messageArgs) { @@ -1542,7 +1552,64 @@ class GenericParserImpl { } private Node makeTerminal(FullToken tk) { - return new Node(NodeType.TERMINAL, tk.span); + var nodeType = + switch (tk.token) { + // keywords + case AMENDS -> NodeType.AMENDS_KEYWORD; + case AS -> NodeType.AS_KEYWORD; + case CLASS -> NodeType.CLASS_KEYWORD; + case ELSE -> NodeType.ELSE_KEYWORD; + case EXTENDS -> NodeType.EXTENDS_KEYWORD; + case FOR -> NodeType.FOR_KEYWORD; + case FUNCTION -> NodeType.FUNCTION_KEYWORD; + case IF -> NodeType.IF_KEYWORD; + case IMPORT -> NodeType.IMPORT_KEYWORD; + case IMPORT_STAR -> NodeType.IMPORT_STAR_KEYWORD; + case IN -> NodeType.IN_KEYWORD; + case LET -> NodeType.LET_KEYWORD; + case MODULE -> NodeType.MODULE_KEYWORD; + case NEW -> NodeType.NEW_KEYWORD; + case OUT -> NodeType.OUT_KEYWORD; + case READ -> NodeType.READ_KEYWORD; + case READ_QUESTION -> NodeType.READ_QUESTION_KEYWORD; + case READ_STAR -> NodeType.READ_STAR_KEYWORD; + case SUPER -> NodeType.SUPER_KEYWORD; + case THROW -> NodeType.THROW_KEYWORD; + case TRACE -> NodeType.TRACE_KEYWORD; + case TYPE_ALIAS -> NodeType.TYPEALIAS_KEYWORD; + case WHEN -> NodeType.WHEN_KEYWORD; + // punctuation + case ARROW -> NodeType.ARROW; + case ASSIGN -> NodeType.ASSIGN; + case AT -> NodeType.AT; + case COLON -> NodeType.COLON; + case COMMA -> NodeType.COMMA; + case DOT -> NodeType.DOT; + case GT -> NodeType.GT; + case LBRACE -> NodeType.LBRACE; + case LBRACK -> NodeType.LBRACK; + case LPAREN -> NodeType.LPAREN; + case LPRED -> NodeType.LPRED; + case LT -> NodeType.LT; + case MINUS -> NodeType.MINUS; + case NOT -> NodeType.NOT; + case QSPREAD -> NodeType.QSPREAD; + case QUESTION -> NodeType.QUESTION; + case RBRACE -> NodeType.RBRACE; + case RBRACK -> NodeType.RBRACK; + case RPAREN -> NodeType.RPAREN; + case SPREAD -> NodeType.SPREAD; + case STAR -> NodeType.STAR; + case UNDERSCORE -> NodeType.UNDERSCORE; + case UNION -> NodeType.UNION; + // string delimiters + case INTERPOLATION_START -> NodeType.INTERPOLATION_START; + case STRING_START -> NodeType.STRING_START; + case STRING_MULTI_START -> NodeType.STRING_MULTI_START; + case STRING_END -> NodeType.STRING_END; + default -> throw new RuntimeException("Unreacheable code"); + }; + return new Node(nodeType, tk.span); } // fast-forward over affix tokens diff --git a/pkl-parser/src/main/java/org/pkl/parser/ParserImpl.java b/pkl-parser/src/main/java/org/pkl/parser/ParserImpl.java index 47636e76f..7576e29bc 100644 --- a/pkl-parser/src/main/java/org/pkl/parser/ParserImpl.java +++ b/pkl-parser/src/main/java/org/pkl/parser/ParserImpl.java @@ -79,7 +79,7 @@ import org.pkl.parser.syntax.StringPart.StringChars; import org.pkl.parser.syntax.Type; import org.pkl.parser.syntax.Type.DeclaredType; import org.pkl.parser.syntax.Type.ParenthesizedType; -import org.pkl.parser.syntax.Type.StringConstantType; +import org.pkl.parser.syntax.Type.StringLiteralType; import org.pkl.parser.syntax.TypeAlias; import org.pkl.parser.syntax.TypeAnnotation; import org.pkl.parser.syntax.TypeArgumentList; @@ -1427,7 +1427,7 @@ final class ParserImpl { } case STRING_START -> { var str = parseStringConstant(); - typ = new StringConstantType(str, str.span()); + typ = new StringLiteralType(str, str.span()); } default -> { var text = _lookahead.text(lexer); diff --git a/pkl-parser/src/main/java/org/pkl/parser/ParserVisitor.java b/pkl-parser/src/main/java/org/pkl/parser/ParserVisitor.java index d8febb245..495ba548e 100644 --- a/pkl-parser/src/main/java/org/pkl/parser/ParserVisitor.java +++ b/pkl-parser/src/main/java/org/pkl/parser/ParserVisitor.java @@ -84,7 +84,7 @@ public interface ParserVisitor { Result visitThisType(Type.ThisType type); - Result visitStringConstantType(Type.StringConstantType type); + Result visitStringLiteralType(Type.StringLiteralType type); Result visitDeclaredType(Type.DeclaredType type); diff --git a/pkl-parser/src/main/java/org/pkl/parser/syntax/Type.java b/pkl-parser/src/main/java/org/pkl/parser/syntax/Type.java index f77688a58..676210198 100644 --- a/pkl-parser/src/main/java/org/pkl/parser/syntax/Type.java +++ b/pkl-parser/src/main/java/org/pkl/parser/syntax/Type.java @@ -72,14 +72,14 @@ public abstract sealed class Type extends AbstractNode { } } - public static final class StringConstantType extends Type { - public StringConstantType(StringConstant str, Span span) { + public static final class StringLiteralType extends Type { + public StringLiteralType(StringConstant str, Span span) { super(span, List.of(str)); } @Override public T accept(ParserVisitor visitor) { - return visitor.visitStringConstantType(this); + return visitor.visitStringLiteralType(this); } public StringConstant getStr() { diff --git a/pkl-parser/src/main/java/org/pkl/parser/syntax/generic/FullSpan.java b/pkl-parser/src/main/java/org/pkl/parser/syntax/generic/FullSpan.java index f53f1774d..05b79d57c 100644 --- a/pkl-parser/src/main/java/org/pkl/parser/syntax/generic/FullSpan.java +++ b/pkl-parser/src/main/java/org/pkl/parser/syntax/generic/FullSpan.java @@ -1,5 +1,5 @@ /* - * Copyright © 2025 Apple Inc. and the Pkl project authors. All rights reserved. + * Copyright © 2025-2026 Apple Inc. and the Pkl project authors. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -38,6 +38,9 @@ public record FullSpan( @Override public String toString() { + if (lineBegin == lineEnd && colBegin == colEnd) { + return "(%d:%d)".formatted(lineBegin, colBegin); + } return "(%d:%d - %d:%d)".formatted(lineBegin, colBegin, lineEnd, colEnd); } } diff --git a/pkl-parser/src/main/java/org/pkl/parser/syntax/generic/Node.java b/pkl-parser/src/main/java/org/pkl/parser/syntax/generic/Node.java index bc481b15d..d15cd7308 100644 --- a/pkl-parser/src/main/java/org/pkl/parser/syntax/generic/Node.java +++ b/pkl-parser/src/main/java/org/pkl/parser/syntax/generic/Node.java @@ -51,6 +51,10 @@ public final class Node { return text; } + public void setText(String text) { + this.text = text; + } + /** Returns the first child of type {@code type} or {@code null}. */ public @Nullable Node findChildByType(NodeType type) { for (var child : children) { diff --git a/pkl-parser/src/main/java/org/pkl/parser/syntax/generic/NodeType.java b/pkl-parser/src/main/java/org/pkl/parser/syntax/generic/NodeType.java index 793e5d261..f5e10923a 100644 --- a/pkl-parser/src/main/java/org/pkl/parser/syntax/generic/NodeType.java +++ b/pkl-parser/src/main/java/org/pkl/parser/syntax/generic/NodeType.java @@ -16,13 +16,68 @@ package org.pkl.parser.syntax.generic; public enum NodeType { - TERMINAL, SHEBANG, // affixes, LINE_COMMENT(NodeKind.AFFIX), BLOCK_COMMENT(NodeKind.AFFIX), SEMICOLON(NodeKind.AFFIX), + // terminals: keywords + AMENDS_KEYWORD(NodeKind.TERMINAL), + AS_KEYWORD(NodeKind.TERMINAL), + CLASS_KEYWORD(NodeKind.TERMINAL), + ELSE_KEYWORD(NodeKind.TERMINAL), + EXTENDS_KEYWORD(NodeKind.TERMINAL), + FOR_KEYWORD(NodeKind.TERMINAL), + FUNCTION_KEYWORD(NodeKind.TERMINAL), + IF_KEYWORD(NodeKind.TERMINAL), + IMPORT_KEYWORD(NodeKind.TERMINAL), + IMPORT_STAR_KEYWORD(NodeKind.TERMINAL), + IN_KEYWORD(NodeKind.TERMINAL), + LET_KEYWORD(NodeKind.TERMINAL), + MODULE_KEYWORD(NodeKind.TERMINAL), + NEW_KEYWORD(NodeKind.TERMINAL), + OUT_KEYWORD(NodeKind.TERMINAL), + READ_KEYWORD(NodeKind.TERMINAL), + READ_QUESTION_KEYWORD(NodeKind.TERMINAL), + READ_STAR_KEYWORD(NodeKind.TERMINAL), + SUPER_KEYWORD(NodeKind.TERMINAL), + THROW_KEYWORD(NodeKind.TERMINAL), + TRACE_KEYWORD(NodeKind.TERMINAL), + TYPEALIAS_KEYWORD(NodeKind.TERMINAL), + WHEN_KEYWORD(NodeKind.TERMINAL), + + // terminals: punctuation + ARROW(NodeKind.TERMINAL), + ASSIGN(NodeKind.TERMINAL), + AT(NodeKind.TERMINAL), + COLON(NodeKind.TERMINAL), + COMMA(NodeKind.TERMINAL), + DOT(NodeKind.TERMINAL), + GT(NodeKind.TERMINAL), + LBRACE(NodeKind.TERMINAL), + LBRACK(NodeKind.TERMINAL), + LPAREN(NodeKind.TERMINAL), + LPRED(NodeKind.TERMINAL), + LT(NodeKind.TERMINAL), + MINUS(NodeKind.TERMINAL), + NOT(NodeKind.TERMINAL), + QSPREAD(NodeKind.TERMINAL), + QUESTION(NodeKind.TERMINAL), + RBRACE(NodeKind.TERMINAL), + RBRACK(NodeKind.TERMINAL), + RPAREN(NodeKind.TERMINAL), + SPREAD(NodeKind.TERMINAL), + STAR(NodeKind.TERMINAL), + UNDERSCORE(NodeKind.TERMINAL), + UNION(NodeKind.TERMINAL), + + // terminals: string delimiters + INTERPOLATION_START(NodeKind.TERMINAL), + STRING_START(NodeKind.TERMINAL), + STRING_MULTI_START(NodeKind.TERMINAL), + STRING_END(NodeKind.TERMINAL), + MODULE, DOC_COMMENT, DOC_COMMENT_LINE, @@ -47,14 +102,14 @@ public enum NodeType { CLASS_BODY, CLASS_BODY_ELEMENTS, CLASS_METHOD, - CLASS_METHOD_HEADER, - CLASS_METHOD_BODY, + METHOD_HEADER, + METHOD_BODY, CLASS_PROPERTY, CLASS_PROPERTY_HEADER, CLASS_PROPERTY_HEADER_BEGIN, CLASS_PROPERTY_BODY, OBJECT_BODY, - OBJECT_MEMBER_LIST, + OBJECT_MEMBER_ELEMENTS, PARAMETER, TYPE_ANNOTATION, PARAMETER_LIST, @@ -67,7 +122,8 @@ public enum NodeType { TYPE_ARGUMENT_LIST_ELEMENTS, OBJECT_PARAMETER_LIST, TYPE_PARAMETER, - STRING_CHARS, + STRING_CHARS(NodeKind.TERMINAL), + STRING_CONSTANT, OPERATOR, STRING_NEWLINE, STRING_CONTINUATION, @@ -81,6 +137,7 @@ public enum NodeType { OBJECT_PROPERTY_BODY, OBJECT_METHOD, MEMBER_PREDICATE, + MEMBER_PREDICATE_HEADER, OBJECT_ENTRY, OBJECT_ENTRY_HEADER, OBJECT_SPREAD, @@ -107,7 +164,6 @@ public enum NodeType { FUNCTION_LITERAL_EXPR(NodeKind.EXPR), FUNCTION_LITERAL_BODY, PARENTHESIZED_EXPR(NodeKind.EXPR), - PARENTHESIZED_EXPR_ELEMENTS, SUPER_SUBSCRIPT_EXPR(NodeKind.EXPR), SUPER_ACCESS_EXPR(NodeKind.EXPR), SUBSCRIPT_EXPR(NodeKind.EXPR), @@ -143,7 +199,7 @@ public enum NodeType { PARENTHESIZED_TYPE_ELEMENTS, DECLARED_TYPE(NodeKind.TYPE), NULLABLE_TYPE(NodeKind.TYPE), - STRING_CONSTANT_TYPE(NodeKind.TYPE), + STRING_LITERAL_TYPE(NodeKind.TYPE), CONSTRAINED_TYPE(NodeKind.TYPE), CONSTRAINED_TYPE_CONSTRAINT, CONSTRAINED_TYPE_ELEMENTS; @@ -162,6 +218,10 @@ public enum NodeType { return kind == NodeKind.AFFIX; } + public boolean isTerminal() { + return kind == NodeKind.TERMINAL; + } + public boolean isExpression() { return kind == NodeKind.EXPR; } @@ -174,6 +234,7 @@ public enum NodeType { TYPE, EXPR, AFFIX, + TERMINAL, NONE; } } diff --git a/pkl-parser/src/test/kotlin/org/pkl/parser/GenericSexpRenderer.kt b/pkl-parser/src/test/kotlin/org/pkl/parser/GenericSexpRenderer.kt index 4a3770945..349446b57 100644 --- a/pkl-parser/src/test/kotlin/org/pkl/parser/GenericSexpRenderer.kt +++ b/pkl-parser/src/test/kotlin/org/pkl/parser/GenericSexpRenderer.kt @@ -55,13 +55,13 @@ class GenericSexpRenderer(code: String) { buf.append("(") buf.append(name(node)) val oldTab = increaseTab() - var previousTerminal: Node? = null + var isDefault = false for (child in node.children) { - if (child.type == NodeType.TERMINAL) previousTerminal = child - if (child.type in IGNORED_CHILDREN) continue + if (child.type == NodeType.STAR) isDefault = true + if (child.type.isIgnored()) continue buf.append('\n') - if (previousTerminal != null && previousTerminal.text(source) == "*") { - previousTerminal = null + if (isDefault) { + isDefault = false renderDefaultUnionType(child) } else { innerRender(child) @@ -83,9 +83,9 @@ class GenericSexpRenderer(code: String) { private fun collectChildren(node: Node): List = when (node.type) { NodeType.MULTI_LINE_STRING_LITERAL_EXPR -> - node.children.filter { it.type !in IGNORED_CHILDREN && !it.type.isStringData() } + node.children.filter { !it.type.isIgnored() && !it.type.isStringData() } NodeType.SINGLE_LINE_STRING_LITERAL_EXPR -> { - val children = node.children.filter { it.type !in IGNORED_CHILDREN } + val children = node.children.filter { !it.type.isIgnored() } val res = mutableListOf() var prev: Node? = null for (child in children) { @@ -97,11 +97,13 @@ class GenericSexpRenderer(code: String) { } res } + // a string constant is rendered as a single leaf + NodeType.STRING_CONSTANT, NodeType.DOC_COMMENT -> listOf() else -> { val nodes = mutableListOf() for (child in node.children) { - if (child.type in IGNORED_CHILDREN) continue + if (child.type.isIgnored()) continue if (child.type in UNPACK_CHILDREN) { nodes += collectChildren(child) } else { @@ -112,6 +114,9 @@ class GenericSexpRenderer(code: String) { } } + private fun NodeType.isIgnored(): Boolean = + this != NodeType.STRING_CHARS && (isTerminal || this in IGNORED_CHILDREN) + private fun NodeType.isStringData(): Boolean = this == NodeType.STRING_CHARS || this == NodeType.STRING_ESCAPE @@ -125,19 +130,18 @@ class GenericSexpRenderer(code: String) { NodeType.EXTENDS_CLAUSE, NodeType.AMENDS_CLAUSE -> "extendsOrAmendsClause" NodeType.TYPEALIAS -> "typeAlias" + NodeType.STRING_CONSTANT, NodeType.STRING_ESCAPE -> "stringChars" NodeType.QUALIFIED_ACCESS_EXPR -> { val op = node.findChildByType(NodeType.OPERATOR)!! if (op.text(source) == ".") "qualifiedAccessExpr" else "nullableQualifiedAccessExpr" } - NodeType.READ_EXPR -> { - val terminal = node.children.find { it.type == NodeType.TERMINAL }!!.text(source) - when (terminal) { - "read*" -> "readGlobExpr" - "read?" -> "readNullExpr" + NodeType.READ_EXPR -> + when { + node.findChildByType(NodeType.READ_STAR_KEYWORD) != null -> "readGlobExpr" + node.findChildByType(NodeType.READ_QUESTION_KEYWORD) != null -> "readNullExpr" else -> "readExpr" } - } else -> { val names = node.type.name.split('_').map { it.lowercase() } if (names.size > 1) { @@ -148,9 +152,9 @@ class GenericSexpRenderer(code: String) { } private fun importName(node: Node, isExpr: Boolean): String { - val terminal = node.children.find { it.type == NodeType.TERMINAL }!!.text(source) + val isGlob = node.findChildByType(NodeType.IMPORT_STAR_KEYWORD) != null val suffix = if (isExpr) "Expr" else "Clause" - return if (terminal == "import*") "importGlob$suffix" else "import$suffix" + return if (isGlob) "importGlob$suffix" else "import$suffix" } private fun binopName(node: Node): String { @@ -194,7 +198,6 @@ class GenericSexpRenderer(code: String) { NodeType.BLOCK_COMMENT, NodeType.SHEBANG, NodeType.SEMICOLON, - NodeType.TERMINAL, NodeType.OPERATOR, NodeType.STRING_NEWLINE, NodeType.STRING_CONTINUATION, @@ -210,15 +213,16 @@ class GenericSexpRenderer(code: String) { NodeType.CLASS_PROPERTY_HEADER, NodeType.CLASS_PROPERTY_HEADER_BEGIN, NodeType.CLASS_PROPERTY_BODY, - NodeType.CLASS_METHOD_HEADER, - NodeType.CLASS_METHOD_BODY, + NodeType.METHOD_HEADER, + NodeType.METHOD_BODY, NodeType.CLASS_HEADER, NodeType.CLASS_HEADER_EXTENDS, NodeType.CLASS_BODY_ELEMENTS, NodeType.MODIFIER_LIST, NodeType.NEW_HEADER, - NodeType.OBJECT_MEMBER_LIST, + NodeType.OBJECT_MEMBER_ELEMENTS, NodeType.OBJECT_ENTRY_HEADER, + NodeType.MEMBER_PREDICATE_HEADER, NodeType.OBJECT_PROPERTY_HEADER, NodeType.OBJECT_PROPERTY_HEADER_BEGIN, NodeType.OBJECT_PROPERTY_BODY, @@ -241,7 +245,6 @@ class GenericSexpRenderer(code: String) { NodeType.TYPE_ARGUMENT_LIST_ELEMENTS, NodeType.LET_PARAMETER_DEFINITION, NodeType.LET_PARAMETER, - NodeType.PARENTHESIZED_EXPR_ELEMENTS, NodeType.PARENTHESIZED_TYPE_ELEMENTS, NodeType.FUNCTION_TYPE_PARAMETERS, ) diff --git a/pkl-parser/src/test/kotlin/org/pkl/parser/SexpRenderer.kt b/pkl-parser/src/test/kotlin/org/pkl/parser/SexpRenderer.kt index 076971424..fbf6c9423 100644 --- a/pkl-parser/src/test/kotlin/org/pkl/parser/SexpRenderer.kt +++ b/pkl-parser/src/test/kotlin/org/pkl/parser/SexpRenderer.kt @@ -777,7 +777,7 @@ class SexpRenderer { buf.append(tab) buf.append("(thisType)") } - is StringConstantType -> renderStringConstantType(type) + is StringLiteralType -> renderStringLiteralType(type) is DeclaredType -> renderDeclaredType(type) is ParenthesizedType -> renderParenthesizedType(type) is NullableType -> renderNullableType(type) @@ -787,9 +787,9 @@ class SexpRenderer { } } - fun renderStringConstantType(type: StringConstantType) { + fun renderStringLiteralType(type: StringLiteralType) { buf.append(tab) - buf.append("(stringConstantType") + buf.append("(stringLiteralType") val oldTab = increaseTab() buf.append('\n') renderStringConstant(type.str) diff --git a/stdlib/reflect.pkl b/stdlib/reflect.pkl index 675c18650..7daa7c40b 100644 --- a/stdlib/reflect.pkl +++ b/stdlib/reflect.pkl @@ -30,6 +30,7 @@ module pkl.reflect import "pkl:base" +import "pkl:syntax" /// A mirror for the [Any] type. const anyType: DeclaredType = DeclaredType(Class(Any)) @@ -166,7 +167,7 @@ external const function TypeVariable(referent: TypeParameter): TypeVariable /// A program declaration. abstract external class Declaration { /// The source location of this declaration. - location: SourceLocation + location: syntax.SourceLocation /// The documentation comment for this declaration, if any. docComment: String? @@ -457,13 +458,5 @@ external class TypeVariable extends Type { } /// A source location within a module. -external class SourceLocation { - /// The 1-based line number of this source location. - line: UInt - - /// The 1-based column number of this source location. - column: UInt - - /// The display URI of this source location. - displayUri: String -} +@Deprecated { since = "0.33.0"; replaceWith = "syntax.SourceLocation" } +typealias SourceLocation = syntax.SourceLocation diff --git a/stdlib/syntax.pkl b/stdlib/syntax.pkl new file mode 100644 index 000000000..5466a4afc --- /dev/null +++ b/stdlib/syntax.pkl @@ -0,0 +1,422 @@ +//===----------------------------------------------------------------------===// +// Copyright © 2026 Apple Inc. and the Pkl project authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//===----------------------------------------------------------------------===// + +/// Utilities for managing Pkl source code. +@ModuleInfo { minPklVersion = "0.33.0" } +@Since { version = "0.33.0" } +module pkl.syntax + +/// Utilities to parse Pkl source code to a syntax tree. +class Parser { + /// Parse the string or resource as a Pkl module, returning either a syntax node or + /// throwing an error. + external function parseModule(source: String | Resource): Node + + /// Parse the string or resource as a Pkl module, returning either a syntax node or null + /// in case of an error. + external function parseModuleOrNull(source: String | Resource): Node? + + /// Parse the string or resource as a Pkl expression, returning either a syntax node or + /// throwing an error. + external function parseExpression(source: String | Resource): Node + + /// Parse the string or resource as a Pkl expression, returning either a syntax node or null + /// in case of an error. + external function parseExpressionOrNull(source: String | Resource): Node? +} + +/// Renders a [Node] back to Pkl source code, using the canonical Pkl formatter. +class FormattingRenderer { + /// The grammar version to target. + /// + /// `"V1"` matches Pkl 0.25 - 0.29, `"V2"` matches Pkl 0.30+. + grammarVersion: "V1" | "V2" = "V2" + + /// Render [node] as Pkl source code. + external function render(node: Node): String +} + +/// A generic, untyped node in the Pkl syntax tree. +/// +/// A node is either a *leaf*, carrying no [children], or a *branch*, whose content is its +/// [children]. Either way, [text] is the source code the node spans. +class Node { + /// The kind of this node. + type: NodeType + + /// The child nodes, in source order. Empty for leaf nodes. + children: List(isTerminal.implies(isEmpty)) + + /// The parent node, or [null] for the root and for nodes not attached to a tree. + hidden parent: Node? + + /// The verbatim source text for this node. + /// + /// [FormattingRenderer] only renders this text if this is a [terminal][isTerminal] node. + text: String?(isTerminal.implies(this != null)) + + /// The source location of this node. + span: Span? + + /// Tells if this is a terminal node; i.e. a node that spans a single token of the grammar. + /// + /// A terminal never has [children], but not every node without [children] is a terminal. + fixed isTerminal: Boolean = type is TerminalNodeType + + /// Fold [operator] over this node and its descendants, top-down in pre-order. + /// + /// Example: + /// ``` + /// // count the if-expressions in a module + /// module.fold(0, (acc, n) -> if (n.type == "if_expr") acc + 1 else acc) + /// ``` + external function fold(initial: Result, operator: (Result, Node) -> Result): Result + + /// Returns the first child matching [predicate], searching depth-first in pre-order. + /// + /// *Child* means any node below this one, at any depth. + /// This node itself is not considered. + /// The search stops at the first match. + /// + /// Throws if no child matches [predicate]. + /// + /// Example: + /// ``` + /// // the name of the first class declared in a module + /// module.findChild((n) -> n.type == "class").findChildOfType("identifier").text + /// ``` + external function findChild(predicate: (Node) -> Boolean): Node + + /// Same as [findChild()] but returns [null] if no child matches [predicate]. + external function findChildOrNull(predicate: (Node) -> Boolean): Node? + + /// Returns every child matching [predicate], in depth-first pre-order. + /// + /// *Child* means any node below this one, at any depth. + /// This node itself is not considered. + /// A matching node is searched for further matches, so a match may contain another. + external function findChildren(predicate: (Node) -> Boolean): List + + /// Returns the first child whose [type] is [type]. + /// + /// Throws if no child has this type. + external function findChildOfType(type: NodeType): Node + + /// Same as [findChildOfType()] but returns [null] if no child has this type. + external function findChildOfTypeOrNull(type: NodeType): Node? + + /// Returns every child whose [type] is [type], in depth-first pre-order. + external function findChildrenOfType(type: NodeType): List + + /// Returns this node with the first child matching [predicate] replaced by [replacer]'s result. + /// + /// The child replaced is the one [findChild()] returns. If there is none, the tree is + /// returned unchanged. + /// The replacement is left as it is, so it is not searched for further matches. + external function replaceChild(predicate: (Node) -> Boolean, replacer: (Node) -> Node): Node + + /// Returns this node with every child matching [predicate] replaced by [replacer]'s result. + /// + /// The children replaced are those [findChildren()] returns, except that a replacement is left + /// as it is, so a match nested in another match is left to [replacer]. + external function replaceChildren(predicate: (Node) -> Boolean, replacer: (Node) -> Node): Node + + /// Returns this node with the first child whose [type] is [type] replaced by [replacer]'s result. + /// + /// If there is no such child, the tree is returned unchanged. + external function replaceChildOfType(type: NodeType, replacer: (Node) -> Node): Node + + /// Returns this node with every child whose [type] is [type] replaced by [replacer]'s result. + external function replaceChildrenOfType(type: NodeType, replacer: (Node) -> Node): Node + + /// Returns the nearest parent matching [predicate]. + /// + /// Throws if no parent matches [predicate]. + external function findParent(predicate: (Node) -> Boolean): Node + + /// Same as [findParent()] but returns [null] if no parent matches [predicate]. + external function findParentOrNull(predicate: (Node) -> Boolean): Node? + + /// Returns the nearest parent whose [type] is [type]. + /// + /// Throws if no parent has this type. + external function findParentOfType(type: NodeType): Node + + /// Same as [findParentOfType()] but returns [null] if no parent has this type. + external function findParentOfTypeOrNull(type: NodeType): Node? + + /// Returns every parent matching [predicate], nearest first. + external function findParents(predicate: (Node) -> Boolean): List + + /// Tells whether any parent matches [predicate]. + external function hasParent(predicate: (Node) -> Boolean): Boolean + + /// Tells whether any parent's [type] is [type]. + external function hasParentOfType(type: NodeType): Boolean +} + +/// A range of source code, spanning [start] (inclusive) to [end] (exclusive). +class Span { + /// The start of this span. + start: SourceLocation + + /// The end of this span. + end: SourceLocation + + /// The display URI of this span. + /// + /// Rendered for the editor configured in the Pkl settings file, the same way a location is + /// rendered in a stack trace. + displayUri: String +} + +/// A position in source code, given as a 1-based line and column. +class SourceLocation { + /// The 1-based line number of this source location. + line: UInt + + /// The 1-based column number of this source location. + column: UInt + + /// A URI identifying this source location, suitable for display. + /// + /// The exact format depends on where this location came from and on the editor configured in the + /// Pkl settings file. + displayUri: String +} + +/// The kinds of node in the Pkl syntax tree. +typealias NodeType = + TerminalNodeType + | ExprNodeType + | TypeNodeType + | ObjectMemberNodeType + | ClassMemberNodeType + | ModuleMemberNodeType + // module structure + | "module" + | "doc_comment" + | "modifier_list" + | "amends_clause" + | "extends_clause" + | "module_declaration" + | "module_definition" + | "annotation" + | "qualified_identifier" + | "import" + | "import_alias" + | "import_list" + | "typealias_header" + | "typealias_body" + | "class_header" + | "class_header_extends" + | "class_body" + | "class_body_elements" + | "method_header" + | "method_body" + | "class_property_header" + | "class_property_header_begin" + | "class_property_body" + | "object_body" + | "object_member_elements" + | "parameter" + | "type_annotation" + | "parameter_list" + | "parameter_list_elements" + | "type_parameter_list" + | "type_parameter_list_elements" + | "argument_list" + | "argument_list_elements" + | "type_argument_list" + | "type_argument_list_elements" + | "object_parameter_list" + | "type_parameter" + | "string_constant" + // parts of members + | "object_property_header" + | "object_property_header_begin" + | "object_property_body" + | "object_entry_header" + | "member_predicate_header" + | "when_generator_header" + | "for_generator_header" + | "for_generator_header_definition" + | "for_generator_header_definition_header" + // parts of expressions + | "new_header" + | "function_literal_body" + | "if_header" + | "if_condition" + | "if_condition_expr" + | "if_then_expr" + | "if_else_expr" + | "let_parameter_definition" + | "let_parameter" + // parts of types + | "function_type_parameters" + | "parenthesized_type_elements" + | "constrained_type_constraint" + | "constrained_type_elements" + +/// The [NodeType]s that describe a terminal: a node that spans a single token of the grammar, such +/// as a keyword, a punctuation character, an identifier or a literal. +/// +/// A terminal never has [Node.children]. +typealias TerminalNodeType = + // keywords + "amends_keyword" + | "as_keyword" + | "class_keyword" + | "else_keyword" + | "extends_keyword" + | "for_keyword" + | "function_keyword" + | "if_keyword" + | "import_keyword" + | "import_star_keyword" + | "in_keyword" + | "let_keyword" + | "module_keyword" + | "new_keyword" + | "out_keyword" + | "read_keyword" + | "read_question_keyword" + | "read_star_keyword" + | "super_keyword" + | "throw_keyword" + | "trace_keyword" + | "typealias_keyword" + | "when_keyword" + // punctuation + | "arrow" + | "assign" + | "at" + | "colon" + | "comma" + | "dot" + | "gt" + | "lbrace" + | "lbrack" + | "lparen" + | "lpred" + | "lt" + | "minus" + | "not" + | "qspread" + | "question" + | "rbrace" + | "rbrack" + | "rparen" + | "spread" + | "star" + | "underscore" + | "union" + // string delimiters + | "interpolation_start" + | "string_start" + | "string_multi_start" + | "string_end" + // string content + | "string_chars" + | "string_newline" + | "string_continuation" + | "string_escape" + // affixes + | "shebang" + | "line_comment" + | "block_comment" + | "semicolon" + | "doc_comment_line" + // names, modifiers and operators + | "identifier" + | "modifier" + | "operator" + // literals + | "bool_literal_expr" + | "int_literal_expr" + | "float_literal_expr" + // expressions without operands + | "this_expr" + | "outer_expr" + | "module_expr" + | "null_expr" + // types without operands + | "unknown_type" + | "nothing_type" + | "module_type" + | "this_type" + +/// The [NodeType]s that describe an expression. +typealias ExprNodeType = + "this_expr" + | "outer_expr" + | "module_expr" + | "null_expr" + | "throw_expr" + | "trace_expr" + | "import_expr" + | "read_expr" + | "new_expr" + | "unary_minus_expr" + | "logical_not_expr" + | "function_literal_expr" + | "parenthesized_expr" + | "super_subscript_expr" + | "super_access_expr" + | "subscript_expr" + | "qualified_access_expr" + | "if_expr" + | "let_expr" + | "bool_literal_expr" + | "int_literal_expr" + | "float_literal_expr" + | "single_line_string_literal_expr" + | "multi_line_string_literal_expr" + | "unqualified_access_expr" + | "non_null_expr" + | "amends_expr" + | "binary_op_expr" + +/// The [NodeType]s that describe a type. +typealias TypeNodeType = + "unknown_type" + | "nothing_type" + | "module_type" + | "this_type" + | "union_type" + | "function_type" + | "parenthesized_type" + | "declared_type" + | "nullable_type" + | "string_literal_type" + | "constrained_type" + +/// The [NodeType]s that describe a member of an object body. +typealias ObjectMemberNodeType = + "object_element" + | "object_property" + | "object_method" + | "member_predicate" + | "object_entry" + | "object_spread" + | "when_generator" + | "for_generator" + +/// The [NodeType]s that describe a member of a class body. +typealias ClassMemberNodeType = "class_property" | "class_method" + +/// The [NodeType]s that describe a member of a module. +typealias ModuleMemberNodeType = "class_property" | "class_method" | "class" | "typealias"