Implement canonical formatter (#1107)

CLI commands also added: `pkl format check` and `pkl format apply`.

pkl-parser/src/main/java/org/pkl/parser/GenericParserError.java

@@ -0,0 +1,36 @@
+/*
+ * Copyright © 2025 Apple Inc. and the Pkl project authors. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.pkl.parser;
+
+import org.pkl.parser.syntax.generic.FullSpan;
+
+public class GenericParserError extends RuntimeException {
+  private final FullSpan span;
+
+  public GenericParserError(String msg, FullSpan span) {
+    super(msg);
+    this.span = span;
+  }
+
+  public FullSpan getSpan() {
+    return span;
+  }
+
+  @Override
+  public String toString() {
+    return getMessage() + " at " + span;
+  }
+}

pkl-parser/src/main/java/org/pkl/parser/Lexer.java

@@ -17,6 +17,7 @@ package org.pkl.parser;
 
 import java.util.ArrayDeque;
 import java.util.Deque;
+import org.pkl.parser.syntax.generic.FullSpan;
 import org.pkl.parser.util.ErrorMessages;
 
 public class Lexer {
@@ -25,6 +26,10 @@ public class Lexer {
   private final int size;
   protected int cursor = 0;
   protected int sCursor = 0;
+  private int line = 1;
+  private int sLine = 1;
+  private int col = 1;
+  private int sCol = 1;
   private char lookahead;
   private State state = State.DEFAULT;
   private final Deque<InterpolationScope> interpolationStack = new ArrayDeque<>();
@@ -50,6 +55,11 @@ public class Lexer {
     return new Span(sCursor, cursor - sCursor);
   }
 
+  // The full span of the last lexed token
+  public FullSpan fullSpan() {
+    return new FullSpan(sCursor, cursor - sCursor, sLine, sCol, line, col);
+  }
+
   // The text of the last lexed token
   public String text() {
     return new String(source, sCursor, cursor - sCursor);
@@ -65,6 +75,8 @@ public class Lexer {
 
   public Token next() {
     sCursor = cursor;
+    sLine = line;
+    sCol = col;
     newLinesBetween = 0;
     return switch (state) {
       case DEFAULT -> nextDefault();
@@ -79,7 +91,9 @@ public class Lexer {
       sCursor = cursor;
       if (ch == '\n') {
         newLinesBetween++;
+        sLine = line;
       }
+      sCol = col;
       ch = nextChar();
     }
     return switch (ch) {
@@ -678,15 +692,23 @@ public class Lexer {
     } else {
       lookahead = source[cursor];
     }
+    if (tmp == '\n') {
+      line++;
+      col = 1;
+    } else {
+      col++;
+    }
     return tmp;
   }
 
   private void backup() {
     lookahead = source[--cursor];
+    col--;
   }
 
   private void backup(int amount) {
     cursor -= amount;
+    col -= amount;
     lookahead = source[cursor];
   }
 

pkl-parser/src/main/java/org/pkl/parser/Parser.java

@@ -18,7 +18,6 @@ package org.pkl.parser;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
-import java.util.EnumSet;
 import java.util.List;
 import java.util.function.Supplier;
 import org.pkl.parser.syntax.Annotation;
@@ -1809,16 +1808,13 @@ public class Parser {
   private FullToken forceNext() {
     var tk = lexer.next();
     precededBySemicolon = false;
-    while (AFFIXES.contains(tk)) {
+    while (tk.isAffix()) {
       precededBySemicolon = precededBySemicolon || tk == Token.SEMICOLON;
       tk = lexer.next();
     }
     return new FullToken(tk, lexer.span(), lexer.newLinesBetween);
   }
 
-  private static final EnumSet<Token> AFFIXES =
-      EnumSet.of(Token.LINE_COMMENT, Token.BLOCK_COMMENT, Token.SEMICOLON);
-
   // Like next, but don't ignore comments
   private FullToken nextComment() {
     prev = _lookahead;

pkl-parser/src/main/java/org/pkl/parser/Token.java

@@ -191,6 +191,13 @@ public enum Token {
     };
   }
 
+  public boolean isAffix() {
+    return switch (this) {
+      case LINE_COMMENT, BLOCK_COMMENT, SEMICOLON -> true;
+      default -> false;
+    };
+  }
+
   public String text() {
     if (this == UNDERSCORE) {
       return "_";

pkl-parser/src/main/java/org/pkl/parser/syntax/Operator.java

@@ -35,8 +35,10 @@ public enum Operator {
   INT_DIV(9, true),
   MOD(9, true),
   POW(10, false),
-  DOT(11, true),
-  QDOT(11, true);
+  NON_NULL(16, true),
+  SUBSCRIPT(18, true),
+  DOT(20, true),
+  QDOT(20, true);
 
   private final int prec;
   private final boolean isLeftAssoc;
@@ -53,4 +55,33 @@ public enum Operator {
   public boolean isLeftAssoc() {
     return isLeftAssoc;
   }
+
+  public static Operator byName(String name) {
+    return switch (name) {
+      case "??" -> NULL_COALESCE;
+      case "|>" -> PIPE;
+      case "||" -> OR;
+      case "&&" -> AND;
+      case "==" -> EQ_EQ;
+      case "!=" -> NOT_EQ;
+      case "is" -> IS;
+      case "as" -> AS;
+      case "<" -> LT;
+      case "<=" -> LTE;
+      case ">" -> GT;
+      case ">=" -> GTE;
+      case "+" -> PLUS;
+      case "-" -> MINUS;
+      case "*" -> MULT;
+      case "/" -> DIV;
+      case "~/" -> INT_DIV;
+      case "%" -> MOD;
+      case "**" -> POW;
+      case "!!" -> NON_NULL;
+      case "[" -> SUBSCRIPT;
+      case "." -> DOT;
+      case "?." -> QDOT;
+      default -> throw new RuntimeException("Unknown operator: " + name);
+    };
+  }
 }

pkl-parser/src/main/java/org/pkl/parser/syntax/generic/FullSpan.java

@@ -0,0 +1,49 @@
+/*
+ * Copyright © 2025 Apple Inc. and the Pkl project authors. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.pkl.parser.syntax.generic;
+
+import org.pkl.parser.Span;
+
+public record FullSpan(
+    int charIndex, int length, int lineBegin, int colBegin, int lineEnd, int colEnd) {
+
+  public FullSpan endWith(FullSpan end) {
+    return new FullSpan(
+        charIndex,
+        end.charIndex - charIndex + end.length,
+        lineBegin,
+        colBegin,
+        end.lineEnd,
+        end.colEnd);
+  }
+
+  public Span toSpan() {
+    return new Span(charIndex, length);
+  }
+
+  public boolean sameLine(FullSpan other) {
+    return lineEnd == other.lineBegin;
+  }
+
+  public FullSpan stopSpan() {
+    return new FullSpan(charIndex + length - 1, 1, lineEnd, colEnd, lineEnd, colEnd);
+  }
+
+  @Override
+  public String toString() {
+    return "(%d:%d - %d:%d)".formatted(lineBegin, colBegin, lineEnd, colEnd);
+  }
+}

pkl-parser/src/main/java/org/pkl/parser/syntax/generic/Node.java

@@ -0,0 +1,85 @@
+/*
+ * Copyright © 2025 Apple Inc. and the Pkl project authors. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.pkl.parser.syntax.generic;
+
+import java.util.Collections;
+import java.util.List;
+import java.util.Objects;
+import org.pkl.parser.util.Nullable;
+
+public class Node {
+  public final List<Node> children;
+  public final FullSpan span;
+  public final NodeType type;
+  private @Nullable String text;
+
+  public Node(NodeType type, FullSpan span) {
+    this(type, span, Collections.emptyList());
+  }
+
+  public Node(NodeType type, FullSpan span, List<Node> children) {
+    this.type = type;
+    this.span = span;
+    this.children = Collections.unmodifiableList(children);
+  }
+
+  public Node(NodeType type, List<Node> children) {
+    this.type = type;
+    if (children.isEmpty()) throw new RuntimeException("No children or span given for node");
+    var end = children.get(children.size() - 1).span;
+    this.span = children.get(0).span.endWith(end);
+    this.children = Collections.unmodifiableList(children);
+  }
+
+  public String text(char[] source) {
+    if (text == null) {
+      text = new String(source, span.charIndex(), span.length());
+    }
+    return text;
+  }
+
+  /** Returns the first child of type {@code type} or {@code null}. */
+  public @Nullable Node findChildByType(NodeType type) {
+    for (var child : children) {
+      if (child.type == type) return child;
+    }
+    return null;
+  }
+
+  @Override
+  public boolean equals(Object o) {
+    if (this == o) {
+      return true;
+    }
+    if (o == null || getClass() != o.getClass()) {
+      return false;
+    }
+    Node node = (Node) o;
+    return Objects.equals(children, node.children)
+        && Objects.equals(span, node.span)
+        && Objects.equals(type, node.type);
+  }
+
+  @Override
+  public int hashCode() {
+    return Objects.hash(children, span, type);
+  }
+
+  @Override
+  public String toString() {
+    return "Node{type='" + type + "', span=" + span + ", children=" + children + '}';
+  }
+}

pkl-parser/src/main/java/org/pkl/parser/syntax/generic/NodeType.java

@@ -0,0 +1,176 @@
+/*
+ * Copyright © 2025 Apple Inc. and the Pkl project authors. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.pkl.parser.syntax.generic;
+
+public enum NodeType {
+  TERMINAL,
+  SHEBANG,
+  // affixes,
+  LINE_COMMENT(NodeKind.AFFIX),
+  BLOCK_COMMENT(NodeKind.AFFIX),
+  SEMICOLON(NodeKind.AFFIX),
+
+  MODULE,
+  DOC_COMMENT,
+  DOC_COMMENT_LINE,
+  MODIFIER,
+  MODIFIER_LIST,
+  AMENDS_CLAUSE,
+  EXTENDS_CLAUSE,
+  MODULE_DECLARATION,
+  MODULE_DEFINITION,
+  ANNOTATION,
+  IDENTIFIER,
+  QUALIFIED_IDENTIFIER,
+  IMPORT,
+  IMPORT_ALIAS,
+  IMPORT_LIST,
+  TYPEALIAS,
+  TYPEALIAS_HEADER,
+  TYPEALIAS_BODY,
+  CLASS,
+  CLASS_HEADER,
+  CLASS_HEADER_EXTENDS,
+  CLASS_BODY,
+  CLASS_BODY_ELEMENTS,
+  CLASS_METHOD,
+  CLASS_METHOD_HEADER,
+  CLASS_METHOD_BODY,
+  CLASS_PROPERTY,
+  CLASS_PROPERTY_HEADER,
+  CLASS_PROPERTY_HEADER_BEGIN,
+  CLASS_PROPERTY_BODY,
+  OBJECT_BODY,
+  OBJECT_MEMBER_LIST,
+  PARAMETER,
+  TYPE_ANNOTATION,
+  PARAMETER_LIST,
+  PARAMETER_LIST_ELEMENTS,
+  TYPE_PARAMETER_LIST,
+  TYPE_PARAMETER_LIST_ELEMENTS,
+  ARGUMENT_LIST,
+  ARGUMENT_LIST_ELEMENTS,
+  TYPE_ARGUMENT_LIST,
+  TYPE_ARGUMENT_LIST_ELEMENTS,
+  OBJECT_PARAMETER_LIST,
+  TYPE_PARAMETER,
+  STRING_CONSTANT,
+  OPERATOR,
+  STRING_NEWLINE,
+  STRING_ESCAPE,
+
+  // members
+  OBJECT_ELEMENT,
+  OBJECT_PROPERTY,
+  OBJECT_PROPERTY_HEADER,
+  OBJECT_PROPERTY_HEADER_BEGIN,
+  OBJECT_PROPERTY_BODY,
+  OBJECT_METHOD,
+  MEMBER_PREDICATE,
+  OBJECT_ENTRY,
+  OBJECT_ENTRY_HEADER,
+  OBJECT_SPREAD,
+  WHEN_GENERATOR,
+  WHEN_GENERATOR_HEADER,
+  FOR_GENERATOR,
+  FOR_GENERATOR_HEADER,
+  FOR_GENERATOR_HEADER_DEFINITION,
+  FOR_GENERATOR_HEADER_DEFINITION_HEADER,
+
+  // expressions
+  THIS_EXPR(NodeKind.EXPR),
+  OUTER_EXPR(NodeKind.EXPR),
+  MODULE_EXPR(NodeKind.EXPR),
+  NULL_EXPR(NodeKind.EXPR),
+  THROW_EXPR(NodeKind.EXPR),
+  TRACE_EXPR(NodeKind.EXPR),
+  IMPORT_EXPR(NodeKind.EXPR),
+  READ_EXPR(NodeKind.EXPR),
+  NEW_EXPR(NodeKind.EXPR),
+  NEW_HEADER,
+  UNARY_MINUS_EXPR(NodeKind.EXPR),
+  LOGICAL_NOT_EXPR(NodeKind.EXPR),
+  FUNCTION_LITERAL_EXPR(NodeKind.EXPR),
+  FUNCTION_LITERAL_BODY,
+  PARENTHESIZED_EXPR(NodeKind.EXPR),
+  PARENTHESIZED_EXPR_ELEMENTS,
+  SUPER_SUBSCRIPT_EXPR(NodeKind.EXPR),
+  SUPER_ACCESS_EXPR(NodeKind.EXPR),
+  SUBSCRIPT_EXPR(NodeKind.EXPR),
+  IF_EXPR(NodeKind.EXPR),
+  IF_HEADER,
+  IF_CONDITION,
+  IF_CONDITION_EXPR,
+  IF_THEN_EXPR,
+  IF_ELSE_EXPR,
+  LET_EXPR(NodeKind.EXPR),
+  LET_PARAMETER_DEFINITION,
+  LET_PARAMETER,
+  BOOL_LITERAL_EXPR(NodeKind.EXPR),
+  INT_LITERAL_EXPR(NodeKind.EXPR),
+  FLOAT_LITERAL_EXPR(NodeKind.EXPR),
+  SINGLE_LINE_STRING_LITERAL_EXPR(NodeKind.EXPR),
+  MULTI_LINE_STRING_LITERAL_EXPR(NodeKind.EXPR),
+  UNQUALIFIED_ACCESS_EXPR(NodeKind.EXPR),
+  NON_NULL_EXPR(NodeKind.EXPR),
+  AMENDS_EXPR(NodeKind.EXPR),
+  BINARY_OP_EXPR(NodeKind.EXPR),
+
+  // types
+  UNKNOWN_TYPE(NodeKind.TYPE),
+  NOTHING_TYPE(NodeKind.TYPE),
+  MODULE_TYPE(NodeKind.TYPE),
+  UNION_TYPE(NodeKind.TYPE),
+  FUNCTION_TYPE(NodeKind.TYPE),
+  FUNCTION_TYPE_PARAMETERS,
+  PARENTHESIZED_TYPE(NodeKind.TYPE),
+  PARENTHESIZED_TYPE_ELEMENTS,
+  DECLARED_TYPE(NodeKind.TYPE),
+  NULLABLE_TYPE(NodeKind.TYPE),
+  STRING_CONSTANT_TYPE(NodeKind.TYPE),
+  CONSTRAINED_TYPE(NodeKind.TYPE),
+  CONSTRAINED_TYPE_CONSTRAINT,
+  CONSTRAINED_TYPE_ELEMENTS;
+
+  private final NodeKind kind;
+
+  NodeType() {
+    this.kind = NodeKind.NONE;
+  }
+
+  NodeType(NodeKind kind) {
+    this.kind = kind;
+  }
+
+  public boolean isAffix() {
+    return kind == NodeKind.AFFIX;
+  }
+
+  public boolean isExpression() {
+    return kind == NodeKind.EXPR;
+  }
+
+  public boolean isType() {
+    return kind == NodeKind.TYPE;
+  }
+
+  private enum NodeKind {
+    TYPE,
+    EXPR,
+    AFFIX,
+    NONE;
+  }
+}

pkl-parser/src/main/java/org/pkl/parser/syntax/generic/package-info.java

@@ -0,0 +1,4 @@
+@NonnullByDefault
+package org.pkl.parser.syntax.generic;
+
+import org.pkl.parser.util.NonnullByDefault;