From fdc501a35ce03808ec2ec11ede88b98b56e858cb Mon Sep 17 00:00:00 2001 From: Islon Scherer Date: Wed, 17 Sep 2025 11:12:04 +0200 Subject: [PATCH] Implement canonical formatter (#1107) CLI commands also added: `pkl format check` and `pkl format apply`. --- docs/modules/pkl-cli/pages/index.adoc | 27 + pkl-cli/pkl-cli.gradle.kts | 1 + .../kotlin/org/pkl/cli/CliFormatterApply.kt | 52 + .../kotlin/org/pkl/cli/CliFormatterCheck.kt | 43 + .../kotlin/org/pkl/cli/CliFormatterCommand.kt | 53 + .../org/pkl/cli/commands/FormatterCommand.kt | 71 + .../org/pkl/cli/commands/RootCommand.kt | 1 + pkl-formatter/gradle.lockfile | 36 + pkl-formatter/pkl-formatter.gradle.kts | 48 + .../main/kotlin/org/pkl/formatter/Builder.kt | 1258 +++++++++++++ .../kotlin/org/pkl/formatter/Formatter.kt | 53 + .../kotlin/org/pkl/formatter/Generator.kt | 149 ++ .../pkl/formatter/NaturalOrderComparator.kt | 66 + .../org/pkl/formatter/ast/FormatNode.kt | 66 + .../input/class-bodies.pkl | 10 + .../input/comma-termination.pkl | 16 + .../input/comment-interleaved.pkl | 31 + .../input/dangling-doc-comment.pkl | 8 + .../input/doc-comments.pkl | 14 + .../input/expr-binary.pkl | 25 + .../input/expr-chain.pkl | 5 + .../FormatterSnippetTests/input/expr-if.pkl | 7 + .../FormatterSnippetTests/input/expr-let.pkl | 11 + .../FormatterSnippetTests/input/imports.pkl | 19 + .../input/indentation.pkl | 51 + .../input/line-breaks.pkl | 64 + .../input/line-breaks2.pkl | 20 + .../input/line-width.pkl | 18 + .../input/map-function.pkl | 4 + .../FormatterSnippetTests/input/modifiers.pkl | 3 + .../input/module-definitions.pkl | 13 + .../input/multi-line-strings.pkl | 29 + .../input/object-members.pkl | 22 + .../FormatterSnippetTests/input/prefixes.pkl | 12 + .../FormatterSnippetTests/input/spaces.pkl | 23 + .../input/type-aliases.pkl | 6 + .../FormatterSnippetTests/input/when.pkl | 9 + .../output/class-bodies.pkl | 11 + .../output/comma-termination.pkl | 35 + .../output/comment-interleaved.pkl | 34 + .../output/dangling-doc-comment.pkl | 8 + .../output/doc-comments.pkl | 13 + .../output/expr-binary.pkl | 34 + .../output/expr-chain.pkl | 8 + .../FormatterSnippetTests/output/expr-if.pkl | 9 + .../FormatterSnippetTests/output/expr-let.pkl | 18 + .../FormatterSnippetTests/output/imports.pkl | 21 + .../output/indentation.pkl | 61 + .../output/line-breaks.pkl | 75 + .../output/line-breaks2.pkl | 42 + .../output/line-width.pkl | 32 + .../output/map-function.pkl | 14 + .../output/modifiers.pkl | 3 + .../output/module-definitions.pkl | 7 + .../output/multi-line-strings.pkl | 33 + .../output/object-members.pkl | 51 + .../FormatterSnippetTests/output/prefixes.pkl | 11 + .../FormatterSnippetTests/output/spaces.pkl | 23 + .../output/type-aliases.pkl | 4 + .../FormatterSnippetTests/output/when.pkl | 7 + .../pkl/formatter/FormatterSnippetTests.kt | 20 + .../formatter/FormatterSnippetTestsEngine.kt | 73 + .../kotlin/org/pkl/formatter/FormatterTest.kt | 79 + .../org.junit.platform.engine.TestEngine | 1 + .../java/org/pkl/parser/GenericParser.java | 1574 +++++++++++++++++ .../org/pkl/parser/GenericParserError.java | 36 + .../src/main/java/org/pkl/parser/Lexer.java | 22 + .../src/main/java/org/pkl/parser/Parser.java | 6 +- .../src/main/java/org/pkl/parser/Token.java | 7 + .../java/org/pkl/parser/syntax/Operator.java | 35 +- .../pkl/parser/syntax/generic/FullSpan.java | 49 + .../org/pkl/parser/syntax/generic/Node.java | 85 + .../pkl/parser/syntax/generic/NodeType.java | 176 ++ .../parser/syntax/generic/package-info.java | 4 + .../org/pkl/parser/GenericSexpRenderer.kt | 261 +++ .../org/pkl/parser/ParserComparisonTest.kt | 118 ++ .../kotlin/org/pkl/parser/SexpRenderer.kt | 71 +- settings.gradle.kts | 2 + 78 files changed, 5491 insertions(+), 26 deletions(-) create mode 100644 pkl-cli/src/main/kotlin/org/pkl/cli/CliFormatterApply.kt create mode 100644 pkl-cli/src/main/kotlin/org/pkl/cli/CliFormatterCheck.kt create mode 100644 pkl-cli/src/main/kotlin/org/pkl/cli/CliFormatterCommand.kt create mode 100644 pkl-cli/src/main/kotlin/org/pkl/cli/commands/FormatterCommand.kt create mode 100644 pkl-formatter/gradle.lockfile create mode 100644 pkl-formatter/pkl-formatter.gradle.kts create mode 100644 pkl-formatter/src/main/kotlin/org/pkl/formatter/Builder.kt create mode 100644 pkl-formatter/src/main/kotlin/org/pkl/formatter/Formatter.kt create mode 100644 pkl-formatter/src/main/kotlin/org/pkl/formatter/Generator.kt create mode 100644 pkl-formatter/src/main/kotlin/org/pkl/formatter/NaturalOrderComparator.kt create mode 100644 pkl-formatter/src/main/kotlin/org/pkl/formatter/ast/FormatNode.kt create mode 100644 pkl-formatter/src/test/files/FormatterSnippetTests/input/class-bodies.pkl create mode 100644 pkl-formatter/src/test/files/FormatterSnippetTests/input/comma-termination.pkl create mode 100644 pkl-formatter/src/test/files/FormatterSnippetTests/input/comment-interleaved.pkl create mode 100644 pkl-formatter/src/test/files/FormatterSnippetTests/input/dangling-doc-comment.pkl create mode 100644 pkl-formatter/src/test/files/FormatterSnippetTests/input/doc-comments.pkl create mode 100644 pkl-formatter/src/test/files/FormatterSnippetTests/input/expr-binary.pkl create mode 100644 pkl-formatter/src/test/files/FormatterSnippetTests/input/expr-chain.pkl create mode 100644 pkl-formatter/src/test/files/FormatterSnippetTests/input/expr-if.pkl create mode 100644 pkl-formatter/src/test/files/FormatterSnippetTests/input/expr-let.pkl create mode 100644 pkl-formatter/src/test/files/FormatterSnippetTests/input/imports.pkl create mode 100644 pkl-formatter/src/test/files/FormatterSnippetTests/input/indentation.pkl create mode 100644 pkl-formatter/src/test/files/FormatterSnippetTests/input/line-breaks.pkl create mode 100644 pkl-formatter/src/test/files/FormatterSnippetTests/input/line-breaks2.pkl create mode 100644 pkl-formatter/src/test/files/FormatterSnippetTests/input/line-width.pkl create mode 100644 pkl-formatter/src/test/files/FormatterSnippetTests/input/map-function.pkl create mode 100644 pkl-formatter/src/test/files/FormatterSnippetTests/input/modifiers.pkl create mode 100644 pkl-formatter/src/test/files/FormatterSnippetTests/input/module-definitions.pkl create mode 100644 pkl-formatter/src/test/files/FormatterSnippetTests/input/multi-line-strings.pkl create mode 100644 pkl-formatter/src/test/files/FormatterSnippetTests/input/object-members.pkl create mode 100644 pkl-formatter/src/test/files/FormatterSnippetTests/input/prefixes.pkl create mode 100644 pkl-formatter/src/test/files/FormatterSnippetTests/input/spaces.pkl create mode 100644 pkl-formatter/src/test/files/FormatterSnippetTests/input/type-aliases.pkl create mode 100644 pkl-formatter/src/test/files/FormatterSnippetTests/input/when.pkl create mode 100644 pkl-formatter/src/test/files/FormatterSnippetTests/output/class-bodies.pkl create mode 100644 pkl-formatter/src/test/files/FormatterSnippetTests/output/comma-termination.pkl create mode 100644 pkl-formatter/src/test/files/FormatterSnippetTests/output/comment-interleaved.pkl create mode 100644 pkl-formatter/src/test/files/FormatterSnippetTests/output/dangling-doc-comment.pkl create mode 100644 pkl-formatter/src/test/files/FormatterSnippetTests/output/doc-comments.pkl create mode 100644 pkl-formatter/src/test/files/FormatterSnippetTests/output/expr-binary.pkl create mode 100644 pkl-formatter/src/test/files/FormatterSnippetTests/output/expr-chain.pkl create mode 100644 pkl-formatter/src/test/files/FormatterSnippetTests/output/expr-if.pkl create mode 100644 pkl-formatter/src/test/files/FormatterSnippetTests/output/expr-let.pkl create mode 100644 pkl-formatter/src/test/files/FormatterSnippetTests/output/imports.pkl create mode 100644 pkl-formatter/src/test/files/FormatterSnippetTests/output/indentation.pkl create mode 100644 pkl-formatter/src/test/files/FormatterSnippetTests/output/line-breaks.pkl create mode 100644 pkl-formatter/src/test/files/FormatterSnippetTests/output/line-breaks2.pkl create mode 100644 pkl-formatter/src/test/files/FormatterSnippetTests/output/line-width.pkl create mode 100644 pkl-formatter/src/test/files/FormatterSnippetTests/output/map-function.pkl create mode 100644 pkl-formatter/src/test/files/FormatterSnippetTests/output/modifiers.pkl create mode 100644 pkl-formatter/src/test/files/FormatterSnippetTests/output/module-definitions.pkl create mode 100644 pkl-formatter/src/test/files/FormatterSnippetTests/output/multi-line-strings.pkl create mode 100644 pkl-formatter/src/test/files/FormatterSnippetTests/output/object-members.pkl create mode 100644 pkl-formatter/src/test/files/FormatterSnippetTests/output/prefixes.pkl create mode 100644 pkl-formatter/src/test/files/FormatterSnippetTests/output/spaces.pkl create mode 100644 pkl-formatter/src/test/files/FormatterSnippetTests/output/type-aliases.pkl create mode 100644 pkl-formatter/src/test/files/FormatterSnippetTests/output/when.pkl create mode 100644 pkl-formatter/src/test/kotlin/org/pkl/formatter/FormatterSnippetTests.kt create mode 100644 pkl-formatter/src/test/kotlin/org/pkl/formatter/FormatterSnippetTestsEngine.kt create mode 100644 pkl-formatter/src/test/kotlin/org/pkl/formatter/FormatterTest.kt create mode 100644 pkl-formatter/src/test/resources/META-INF/services/org.junit.platform.engine.TestEngine create mode 100644 pkl-parser/src/main/java/org/pkl/parser/GenericParser.java create mode 100644 pkl-parser/src/main/java/org/pkl/parser/GenericParserError.java create mode 100644 pkl-parser/src/main/java/org/pkl/parser/syntax/generic/FullSpan.java create mode 100644 pkl-parser/src/main/java/org/pkl/parser/syntax/generic/Node.java create mode 100644 pkl-parser/src/main/java/org/pkl/parser/syntax/generic/NodeType.java create mode 100644 pkl-parser/src/main/java/org/pkl/parser/syntax/generic/package-info.java create mode 100644 pkl-parser/src/test/kotlin/org/pkl/parser/GenericSexpRenderer.kt create mode 100644 pkl-parser/src/test/kotlin/org/pkl/parser/ParserComparisonTest.kt diff --git a/docs/modules/pkl-cli/pages/index.adoc b/docs/modules/pkl-cli/pages/index.adoc index 05cec769..8f817341 100644 --- a/docs/modules/pkl-cli/pages/index.adoc +++ b/docs/modules/pkl-cli/pages/index.adoc @@ -733,6 +733,33 @@ pkl shell-completion bash pkl shell-completion zsh ---- +[[command-format-check]] +=== `pkl format check` + +*Synopsis*: `pkl format check ` + +This command checks for format violations on the given file or directory and print their names to stdout. + +It returns a non-zero status code in case violations are found. + +If the path is a directory, recursively looks for files with a `.pkl` extension, or files named `PklProject`. + +[[command-format-apply]] +=== `pkl format apply` + +*Synopsis*: `pkl format apply [] ` + +This command formats the given files overwriting them. + +If the path is a directory, recursively looks for files with a `.pkl` extension, or files named `PklProject`. + +==== Options + +.-s, --silent +[%collapsible] +==== +Do not write the name of wrongly formatted files to stdout. +==== + [[common-options]] === Common options diff --git a/pkl-cli/pkl-cli.gradle.kts b/pkl-cli/pkl-cli.gradle.kts index 321df12d..1c7cdf3f 100644 --- a/pkl-cli/pkl-cli.gradle.kts +++ b/pkl-cli/pkl-cli.gradle.kts @@ -61,6 +61,7 @@ dependencies { implementation(libs.jlineTerminal) implementation(libs.jlineTerminalJansi) implementation(projects.pklServer) + implementation(projects.pklFormatter) implementation(libs.clikt) testImplementation(projects.pklCommonsTest) diff --git a/pkl-cli/src/main/kotlin/org/pkl/cli/CliFormatterApply.kt b/pkl-cli/src/main/kotlin/org/pkl/cli/CliFormatterApply.kt new file mode 100644 index 00000000..e524e78e --- /dev/null +++ b/pkl-cli/src/main/kotlin/org/pkl/cli/CliFormatterApply.kt @@ -0,0 +1,52 @@ +/* + * Copyright © 2025 Apple Inc. and the Pkl project authors. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.pkl.cli + +import java.io.IOException +import java.nio.file.Files +import java.nio.file.Path +import kotlin.io.path.writeText +import org.pkl.commons.cli.CliBaseOptions +import org.pkl.commons.cli.CliException + +class CliFormatterApply(cliBaseOptions: CliBaseOptions, path: Path, private val silent: Boolean) : + CliFormatterCommand(cliBaseOptions, path) { + + override fun doRun() { + var status = 0 + + for (path in paths()) { + val contents = Files.readString(path) + val (formatted, stat) = format(path, contents) + status = if (status == 0) stat else status + if (stat != 0) continue + if (!silent && contents != formatted) { + consoleWriter.write(path.toAbsolutePath().toString()) + consoleWriter.flush() + } + try { + path.writeText(formatted, Charsets.UTF_8) + } catch (e: IOException) { + consoleWriter.write("Could not overwrite `$path`: ${e.message}") + consoleWriter.flush() + status = 1 + } + } + if (status != 0) { + throw CliException("Formatting violations found.", status) + } + } +} diff --git a/pkl-cli/src/main/kotlin/org/pkl/cli/CliFormatterCheck.kt b/pkl-cli/src/main/kotlin/org/pkl/cli/CliFormatterCheck.kt new file mode 100644 index 00000000..3657808f --- /dev/null +++ b/pkl-cli/src/main/kotlin/org/pkl/cli/CliFormatterCheck.kt @@ -0,0 +1,43 @@ +/* + * Copyright © 2025 Apple Inc. and the Pkl project authors. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.pkl.cli + +import java.nio.file.Files +import java.nio.file.Path +import org.pkl.commons.cli.CliBaseOptions +import org.pkl.commons.cli.CliException + +class CliFormatterCheck(cliBaseOptions: CliBaseOptions, path: Path) : + CliFormatterCommand(cliBaseOptions, path) { + + override fun doRun() { + var status = 0 + + for (path in paths()) { + val contents = Files.readString(path) + val (formatted, stat) = format(path, contents) + status = if (status == 0) stat else status + if (contents != formatted) { + consoleWriter.write(path.toAbsolutePath().toString()) + consoleWriter.flush() + status = 1 + } + } + if (status != 0) { + throw CliException("Formatting violations found.", status) + } + } +} diff --git a/pkl-cli/src/main/kotlin/org/pkl/cli/CliFormatterCommand.kt b/pkl-cli/src/main/kotlin/org/pkl/cli/CliFormatterCommand.kt new file mode 100644 index 00000000..e206e1c8 --- /dev/null +++ b/pkl-cli/src/main/kotlin/org/pkl/cli/CliFormatterCommand.kt @@ -0,0 +1,53 @@ +/* + * Copyright © 2025 Apple Inc. and the Pkl project authors. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.pkl.cli + +import java.io.Writer +import java.nio.file.Path +import kotlin.io.path.ExperimentalPathApi +import kotlin.io.path.extension +import kotlin.io.path.isDirectory +import kotlin.io.path.name +import kotlin.io.path.walk +import org.pkl.commons.cli.CliBaseOptions +import org.pkl.commons.cli.CliCommand +import org.pkl.formatter.Formatter +import org.pkl.parser.GenericParserError + +abstract class CliFormatterCommand +@JvmOverloads +constructor( + options: CliBaseOptions, + protected val path: Path, + protected val consoleWriter: Writer = System.out.writer(), +) : CliCommand(options) { + protected fun format(file: Path, contents: String): Pair { + try { + return Formatter().format(contents) to 0 + } catch (pe: GenericParserError) { + consoleWriter.write("Could not format `$file`: $pe") + consoleWriter.flush() + return "" to 1 + } + } + + @OptIn(ExperimentalPathApi::class) + protected fun paths(): Sequence { + return if (path.isDirectory()) { + path.walk().filter { it.extension == "pkl" || it.name == "PklProject" } + } else sequenceOf(path) + } +} diff --git a/pkl-cli/src/main/kotlin/org/pkl/cli/commands/FormatterCommand.kt b/pkl-cli/src/main/kotlin/org/pkl/cli/commands/FormatterCommand.kt new file mode 100644 index 00000000..e881db33 --- /dev/null +++ b/pkl-cli/src/main/kotlin/org/pkl/cli/commands/FormatterCommand.kt @@ -0,0 +1,71 @@ +/* + * Copyright © 2025 Apple Inc. and the Pkl project authors. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.pkl.cli.commands + +import com.github.ajalt.clikt.core.Context +import com.github.ajalt.clikt.core.NoOpCliktCommand +import com.github.ajalt.clikt.core.subcommands +import com.github.ajalt.clikt.parameters.arguments.argument +import com.github.ajalt.clikt.parameters.options.flag +import com.github.ajalt.clikt.parameters.options.option +import com.github.ajalt.clikt.parameters.types.path +import java.nio.file.Path +import org.pkl.cli.CliFormatterApply +import org.pkl.cli.CliFormatterCheck +import org.pkl.commons.cli.commands.BaseCommand + +class FormatterCommand : NoOpCliktCommand(name = "format") { + override fun help(context: Context) = "Run commands related to formatting" + + override fun helpEpilog(context: Context) = "For more information, visit $helpLink" + + init { + subcommands(FormatterCheckCommand(), FormatterApplyCommand()) + } +} + +class FormatterCheckCommand : BaseCommand(name = "check", helpLink = helpLink) { + override val helpString: String = + "Check if the given files are properly formatted, printing the file name to stdout in case they are not. Returns non-zero in case of failure." + + val path: Path by + argument(name = "path", help = "File or directory to check.") + .path(mustExist = true, canBeDir = true) + + override fun run() { + CliFormatterCheck(baseOptions.baseOptions(emptyList()), path).run() + } +} + +class FormatterApplyCommand : BaseCommand(name = "apply", helpLink = helpLink) { + override val helpString: String = + "Overwrite all the files in place with the formatted version. Returns non-zero in case of failure." + + val path: Path by + argument(name = "path", help = "File or directory to format.") + .path(mustExist = true, canBeDir = true) + + val silent: Boolean by + option( + names = arrayOf("-s", "--silent"), + help = "Do not write the name of the files that failed formatting to stdout.", + ) + .flag() + + override fun run() { + CliFormatterApply(baseOptions.baseOptions(emptyList()), path, silent) + } +} diff --git a/pkl-cli/src/main/kotlin/org/pkl/cli/commands/RootCommand.kt b/pkl-cli/src/main/kotlin/org/pkl/cli/commands/RootCommand.kt index ff077f90..690ccf9e 100644 --- a/pkl-cli/src/main/kotlin/org/pkl/cli/commands/RootCommand.kt +++ b/pkl-cli/src/main/kotlin/org/pkl/cli/commands/RootCommand.kt @@ -49,6 +49,7 @@ class RootCommand : NoOpCliktCommand(name = "pkl") { ProjectCommand(), DownloadPackageCommand(), AnalyzeCommand(), + FormatterCommand(), CompletionCommand( name = "shell-completion", help = "Generate a completion script for the given shell", diff --git a/pkl-formatter/gradle.lockfile b/pkl-formatter/gradle.lockfile new file mode 100644 index 00000000..0c1ad64b --- /dev/null +++ b/pkl-formatter/gradle.lockfile @@ -0,0 +1,36 @@ +# This is a Gradle generated file for dependency locking. +# Manual edits can break the build and are not advised. +# This file is expected to be part of source control. +net.bytebuddy:byte-buddy:1.15.11=testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath +org.apiguardian:apiguardian-api:1.1.2=testCompileClasspath,testImplementationDependenciesMetadata +org.assertj:assertj-core:3.27.3=testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath +org.jetbrains.intellij.deps:trove4j:1.0.20200330=kotlinBuildToolsApiClasspath,kotlinCompilerClasspath,kotlinKlibCommonizerClasspath +org.jetbrains.kotlin:kotlin-build-common:2.0.21=kotlinBuildToolsApiClasspath +org.jetbrains.kotlin:kotlin-build-tools-api:2.0.21=kotlinBuildToolsApiClasspath +org.jetbrains.kotlin:kotlin-build-tools-impl:2.0.21=kotlinBuildToolsApiClasspath +org.jetbrains.kotlin:kotlin-compiler-embeddable:2.0.21=kotlinBuildToolsApiClasspath,kotlinCompilerClasspath,kotlinKlibCommonizerClasspath +org.jetbrains.kotlin:kotlin-compiler-runner:2.0.21=kotlinBuildToolsApiClasspath +org.jetbrains.kotlin:kotlin-daemon-client:2.0.21=kotlinBuildToolsApiClasspath +org.jetbrains.kotlin:kotlin-daemon-embeddable:2.0.21=kotlinBuildToolsApiClasspath,kotlinCompilerClasspath,kotlinKlibCommonizerClasspath +org.jetbrains.kotlin:kotlin-klib-commonizer-embeddable:2.0.21=kotlinKlibCommonizerClasspath +org.jetbrains.kotlin:kotlin-native-prebuilt:2.0.21=kotlinNativeBundleConfiguration +org.jetbrains.kotlin:kotlin-reflect:2.0.21=kotlinBuildToolsApiClasspath,kotlinCompilerClasspath,kotlinKlibCommonizerClasspath +org.jetbrains.kotlin:kotlin-script-runtime:2.0.21=kotlinBuildToolsApiClasspath,kotlinCompilerClasspath,kotlinCompilerPluginClasspathMain,kotlinCompilerPluginClasspathTest,kotlinKlibCommonizerClasspath +org.jetbrains.kotlin:kotlin-scripting-common:2.0.21=kotlinBuildToolsApiClasspath,kotlinCompilerPluginClasspathMain,kotlinCompilerPluginClasspathTest +org.jetbrains.kotlin:kotlin-scripting-compiler-embeddable:2.0.21=kotlinBuildToolsApiClasspath,kotlinCompilerPluginClasspathMain,kotlinCompilerPluginClasspathTest +org.jetbrains.kotlin:kotlin-scripting-compiler-impl-embeddable:2.0.21=kotlinBuildToolsApiClasspath,kotlinCompilerPluginClasspathMain,kotlinCompilerPluginClasspathTest +org.jetbrains.kotlin:kotlin-scripting-jvm:2.0.21=kotlinBuildToolsApiClasspath,kotlinCompilerPluginClasspathMain,kotlinCompilerPluginClasspathTest +org.jetbrains.kotlin:kotlin-stdlib-jdk7:2.0.21=apiDependenciesMetadata,compileClasspath,implementationDependenciesMetadata,runtimeClasspath,testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath +org.jetbrains.kotlin:kotlin-stdlib-jdk8:2.0.21=apiDependenciesMetadata,compileClasspath,implementationDependenciesMetadata,runtimeClasspath,testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath +org.jetbrains.kotlin:kotlin-stdlib:2.0.21=apiDependenciesMetadata,compileClasspath,implementationDependenciesMetadata,kotlinBuildToolsApiClasspath,kotlinCompilerClasspath,kotlinCompilerPluginClasspathMain,kotlinCompilerPluginClasspathTest,kotlinKlibCommonizerClasspath,runtimeClasspath,testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath +org.jetbrains.kotlinx:kotlinx-coroutines-core-jvm:1.6.4=kotlinBuildToolsApiClasspath,kotlinCompilerClasspath,kotlinKlibCommonizerClasspath +org.jetbrains:annotations:13.0=compileClasspath,kotlinBuildToolsApiClasspath,kotlinCompilerClasspath,kotlinCompilerPluginClasspathMain,kotlinCompilerPluginClasspathTest,kotlinKlibCommonizerClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath +org.junit.jupiter:junit-jupiter-api:5.13.0=testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath +org.junit.jupiter:junit-jupiter-engine:5.13.0=testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath +org.junit.jupiter:junit-jupiter-params:5.13.0=testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath +org.junit.platform:junit-platform-commons:1.13.0=testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath +org.junit.platform:junit-platform-engine:1.13.0=testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath +org.junit.platform:junit-platform-launcher:1.13.0=testRuntimeClasspath +org.junit:junit-bom:5.13.0=testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath +org.opentest4j:opentest4j:1.3.0=testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath +empty=annotationProcessor,compileOnlyDependenciesMetadata,intransitiveDependenciesMetadata,kotlinCompilerPluginClasspath,kotlinNativeCompilerPluginClasspath,kotlinScriptDefExtensions,sourcesJar,testAnnotationProcessor,testApiDependenciesMetadata,testCompileOnlyDependenciesMetadata,testIntransitiveDependenciesMetadata,testKotlinScriptDefExtensions diff --git a/pkl-formatter/pkl-formatter.gradle.kts b/pkl-formatter/pkl-formatter.gradle.kts new file mode 100644 index 00000000..995f58ec --- /dev/null +++ b/pkl-formatter/pkl-formatter.gradle.kts @@ -0,0 +1,48 @@ +/* + * Copyright © 2025 Apple Inc. and the Pkl project authors. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +plugins { + pklAllProjects + pklKotlinLibrary + pklPublishLibrary +} + +dependencies { + api(projects.pklParser) + + testImplementation(projects.pklCommonsTest) +} + +tasks.test { + inputs + .dir("src/test/files/FormatterSnippetTests/input") + .withPropertyName("formatterSnippetTestsInput") + .withPathSensitivity(PathSensitivity.RELATIVE) + inputs + .dir("src/test/files/FormatterSnippetTests/output") + .withPropertyName("formatterSnippetTestsOutput") + .withPathSensitivity(PathSensitivity.RELATIVE) +} + +publishing { + publications { + named("library") { + pom { + url.set("https://github.com/apple/pkl/tree/main/pkl-formatter") + description.set("Formatter for Pkl") + } + } + } +} diff --git a/pkl-formatter/src/main/kotlin/org/pkl/formatter/Builder.kt b/pkl-formatter/src/main/kotlin/org/pkl/formatter/Builder.kt new file mode 100644 index 00000000..0e07007e --- /dev/null +++ b/pkl-formatter/src/main/kotlin/org/pkl/formatter/Builder.kt @@ -0,0 +1,1258 @@ +/* + * Copyright © 2025 Apple Inc. and the Pkl project authors. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.pkl.formatter + +import java.util.EnumSet +import kotlin.collections.withIndex +import org.pkl.formatter.ast.Empty +import org.pkl.formatter.ast.ForceLine +import org.pkl.formatter.ast.FormatNode +import org.pkl.formatter.ast.Group +import org.pkl.formatter.ast.IfWrap +import org.pkl.formatter.ast.Indent +import org.pkl.formatter.ast.Line +import org.pkl.formatter.ast.MultilineStringGroup +import org.pkl.formatter.ast.Nodes +import org.pkl.formatter.ast.Space +import org.pkl.formatter.ast.SpaceOrLine +import org.pkl.formatter.ast.Text +import org.pkl.parser.syntax.Operator +import org.pkl.parser.syntax.generic.Node +import org.pkl.parser.syntax.generic.NodeType + +internal class Builder(sourceText: String) { + private var id: Int = 0 + private val source: CharArray = sourceText.toCharArray() + private var prevNode: Node? = null + + fun format(node: Node): FormatNode { + prevNode = node + return when (node.type) { + NodeType.MODULE -> formatModule(node) + NodeType.DOC_COMMENT -> Nodes(formatGeneric(node.children, null)) + NodeType.DOC_COMMENT_LINE -> formatDocComment(node) + NodeType.LINE_COMMENT, + NodeType.BLOCK_COMMENT, + NodeType.TERMINAL, + NodeType.MODIFIER, + NodeType.IDENTIFIER, + NodeType.STRING_CONSTANT, + NodeType.STRING_ESCAPE, + NodeType.SINGLE_LINE_STRING_LITERAL_EXPR, + NodeType.INT_LITERAL_EXPR, + NodeType.FLOAT_LITERAL_EXPR, + NodeType.BOOL_LITERAL_EXPR, + NodeType.THIS_EXPR, + NodeType.OUTER_EXPR, + NodeType.MODULE_EXPR, + NodeType.NULL_EXPR, + NodeType.MODULE_TYPE, + NodeType.UNKNOWN_TYPE, + NodeType.NOTHING_TYPE, + NodeType.SHEBANG, + NodeType.OPERATOR -> Text(node.text(source)) + NodeType.STRING_NEWLINE -> ForceLine + NodeType.MODULE_DECLARATION -> formatModuleDeclaration(node) + NodeType.MODULE_DEFINITION -> formatModuleDefinition(node) + NodeType.MULTI_LINE_STRING_LITERAL_EXPR -> formatMultilineString(node) + NodeType.ANNOTATION -> formatAnnotation(node) + NodeType.TYPEALIAS -> formatTypealias(node) + NodeType.TYPEALIAS_HEADER -> formatTypealiasHeader(node) + NodeType.TYPEALIAS_BODY -> formatTypealiasBody(node) + NodeType.MODIFIER_LIST -> formatModifierList(node) + NodeType.PARAMETER_LIST -> formatParameterList(node) + NodeType.PARAMETER_LIST_ELEMENTS -> formatParameterListElements(node) + NodeType.TYPE_PARAMETER_LIST -> formatTypeParameterList(node) + NodeType.TYPE_PARAMETER_LIST_ELEMENTS -> formatParameterListElements(node) + NodeType.TYPE_PARAMETER -> Group(newId(), formatGeneric(node.children, SpaceOrLine)) + NodeType.PARAMETER -> formatParameter(node) + NodeType.EXTENDS_CLAUSE, + NodeType.AMENDS_CLAUSE -> formatAmendsExtendsClause(node) + NodeType.IMPORT_LIST -> formatImportList(node) + NodeType.IMPORT -> formatImport(node) + NodeType.IMPORT_ALIAS -> Group(newId(), formatGeneric(node.children, SpaceOrLine)) + NodeType.CLASS -> formatClass(node) + NodeType.CLASS_HEADER -> formatClassHeader(node) + NodeType.CLASS_HEADER_EXTENDS -> formatClassHeaderExtends(node) + NodeType.CLASS_BODY -> formatClassBody(node) + NodeType.CLASS_BODY_ELEMENTS -> formatClassBodyElements(node) + NodeType.CLASS_PROPERTY, + NodeType.OBJECT_PROPERTY, + NodeType.OBJECT_ENTRY -> formatClassProperty(node) + NodeType.CLASS_PROPERTY_HEADER, + NodeType.OBJECT_PROPERTY_HEADER -> formatClassPropertyHeader(node) + NodeType.CLASS_PROPERTY_HEADER_BEGIN, + NodeType.OBJECT_PROPERTY_HEADER_BEGIN -> formatClassPropertyHeaderBegin(node) + NodeType.CLASS_PROPERTY_BODY, + NodeType.OBJECT_PROPERTY_BODY -> formatClassPropertyBody(node) + NodeType.CLASS_METHOD, + NodeType.OBJECT_METHOD -> formatClassMethod(node) + NodeType.CLASS_METHOD_HEADER -> formatClassMethodHeader(node) + NodeType.CLASS_METHOD_BODY -> formatClassMethodBody(node) + NodeType.OBJECT_BODY -> formatObjectBody(node) + NodeType.OBJECT_ELEMENT -> format(node.children[0]) // has a single element + NodeType.OBJECT_ENTRY_HEADER -> formatObjectEntryHeader(node) + NodeType.FOR_GENERATOR -> formatForGenerator(node) + NodeType.FOR_GENERATOR_HEADER -> formatForGeneratorHeader(node) + NodeType.FOR_GENERATOR_HEADER_DEFINITION -> formatForGeneratorHeaderDefinition(node) + NodeType.FOR_GENERATOR_HEADER_DEFINITION_HEADER -> + formatForGeneratorHeaderDefinitionHeader(node) + NodeType.WHEN_GENERATOR -> formatWhenGenerator(node) + NodeType.WHEN_GENERATOR_HEADER -> formatWhenGeneratorHeader(node) + NodeType.OBJECT_SPREAD -> Nodes(formatGeneric(node.children, null)) + NodeType.MEMBER_PREDICATE -> formatMemberPredicate(node) + NodeType.QUALIFIED_IDENTIFIER -> formatQualifiedIdentifier(node) + NodeType.ARGUMENT_LIST -> formatArgumentList(node) + NodeType.ARGUMENT_LIST_ELEMENTS -> formatArgumentListElements(node) + NodeType.OBJECT_PARAMETER_LIST -> formatObjectParameterList(node) + NodeType.IF_EXPR -> formatIf(node) + NodeType.IF_HEADER -> formatIfHeader(node) + NodeType.IF_CONDITION -> formatIfCondition(node) + NodeType.IF_CONDITION_EXPR -> Indent(formatGeneric(node.children, null)) + NodeType.IF_THEN_EXPR -> formatIfThen(node) + NodeType.IF_ELSE_EXPR -> formatIfElse(node) + NodeType.NEW_EXPR, + NodeType.AMENDS_EXPR -> formatNewExpr(node) + NodeType.NEW_HEADER -> formatNewHeader(node) + NodeType.UNQUALIFIED_ACCESS_EXPR -> formatUnqualifiedAccessExpression(node) + NodeType.BINARY_OP_EXPR -> formatBinaryOpExpr(node) + NodeType.FUNCTION_LITERAL_EXPR -> formatFunctionLiteralExpr(node) + NodeType.FUNCTION_LITERAL_BODY -> formatFunctionLiteralBody(node) + NodeType.SUBSCRIPT_EXPR, + NodeType.SUPER_SUBSCRIPT_EXPR -> formatSubscriptExpr(node) + NodeType.TRACE_EXPR -> formatTraceThrowReadExpr(node) + NodeType.THROW_EXPR -> formatTraceThrowReadExpr(node) + NodeType.READ_EXPR -> formatTraceThrowReadExpr(node) + NodeType.NON_NULL_EXPR -> Nodes(formatGeneric(node.children, null)) + NodeType.SUPER_ACCESS_EXPR -> Nodes(formatGeneric(node.children, null)) + NodeType.PARENTHESIZED_EXPR -> formatParenthesizedExpr(node) + NodeType.PARENTHESIZED_EXPR_ELEMENTS -> formatParenthesizedExprElements(node) + NodeType.IMPORT_EXPR -> Nodes(formatGeneric(node.children, null)) + NodeType.LET_EXPR -> formatLetExpr(node) + NodeType.LET_PARAMETER_DEFINITION -> formatLetParameterDefinition(node) + NodeType.LET_PARAMETER -> formatLetParameter(node) + NodeType.UNARY_MINUS_EXPR -> Nodes(formatGeneric(node.children, null)) + NodeType.LOGICAL_NOT_EXPR -> Nodes(formatGeneric(node.children, null)) + NodeType.TYPE_ANNOTATION -> formatTypeAnnotation(node) + NodeType.TYPE_ARGUMENT_LIST -> formatTypeParameterList(node) + NodeType.TYPE_ARGUMENT_LIST_ELEMENTS -> formatParameterListElements(node) + NodeType.DECLARED_TYPE -> formatDeclaredType(node) + NodeType.CONSTRAINED_TYPE -> formatConstrainedType(node) + NodeType.CONSTRAINED_TYPE_CONSTRAINT -> formatParameterList(node) + NodeType.CONSTRAINED_TYPE_ELEMENTS -> formatParameterListElements(node) + NodeType.NULLABLE_TYPE -> Nodes(formatGeneric(node.children, null)) + NodeType.UNION_TYPE -> formatUnionType(node) + NodeType.FUNCTION_TYPE -> formatFunctionType(node) + NodeType.FUNCTION_TYPE_PARAMETERS -> formatParameterList(node) + NodeType.STRING_CONSTANT_TYPE -> format(node.children[0]) + NodeType.PARENTHESIZED_TYPE -> formatParenthesizedType(node) + NodeType.PARENTHESIZED_TYPE_ELEMENTS -> formatParenthesizedTypeElements(node) + else -> throw RuntimeException("Unknown node type: ${node.type}") + } + } + + private fun formatModule(node: Node): FormatNode { + val nodes = + formatGeneric(node.children) { prev, next -> + if (prev.linesBetween(next) > 1) TWO_NEWLINES else ForceLine + } + return Nodes(nodes) + } + + private fun formatModuleDeclaration(node: Node): FormatNode { + return Nodes(formatGeneric(node.children, TWO_NEWLINES)) + } + + private fun formatModuleDefinition(node: Node): FormatNode { + val (prefixes, nodes) = splitPrefixes(node.children) + val fnodes = + formatGenericWithGen(nodes, SpaceOrLine) { node, next -> + if (next == null) { + indent(format(node)) + } else { + format(node) + } + } + val res = Group(newId(), fnodes) + return if (prefixes.isEmpty()) { + res + } else { + val sep = getSeparator(prefixes.last(), nodes.first()) + Nodes(formatGeneric(prefixes, SpaceOrLine) + listOf(sep, res)) + } + } + + private fun formatDocComment(node: Node): FormatNode { + val txt = node.text() + if (txt == "///" || txt == "/// ") return Text("///") + + var comment = txt.substring(3) + if (comment.isStrictBlank()) return Text("///") + + if (comment.isNotEmpty() && comment[0] != ' ') comment = " $comment" + return Text("///$comment") + } + + private fun String.isStrictBlank(): Boolean { + for (ch in this) { + if (ch != ' ' && ch != '\t') return false + } + return true + } + + private fun formatQualifiedIdentifier(node: Node): FormatNode { + // short circuit + if (node.children.size == 1) return format(node.children[0]) + + val first = listOf(format(node.children[0]), Line) + val nodes = + formatGeneric(node.children.drop(1)) { n1, _ -> + if (n1.type == NodeType.TERMINAL) null else Line + } + return Group(newId(), first + listOf(Indent(nodes))) + } + + private fun formatUnqualifiedAccessExpression(node: Node): FormatNode { + val children = node.children + if (children.size == 1) return format(children[0]) + val firstNode = node.firstProperChild()!! + return if (firstNode.text() == "Map") { + val nodes = mutableListOf() + nodes += format(firstNode) + nodes += formatArgumentList(children[1], twoBy2 = true) + Nodes(nodes) + } else { + Nodes(formatGeneric(children, null)) + } + } + + private fun formatAmendsExtendsClause(node: Node): FormatNode { + val prefix = formatGeneric(node.children.dropLast(1), SpaceOrLine) + // string constant + val suffix = Indent(listOf(format(node.children.last()))) + return Group(newId(), prefix + listOf(SpaceOrLine) + suffix) + } + + private fun formatImport(node: Node): FormatNode { + return Group( + newId(), + formatGenericWithGen(node.children, SpaceOrLine) { node, _ -> + if (node.isTerminal("import")) format(node) else indent(format(node)) + }, + ) + } + + private fun formatAnnotation(node: Node): FormatNode { + return Group(newId(), formatGeneric(node.children, SpaceOrLine)) + } + + private fun formatTypealias(node: Node): FormatNode { + val nodes = + groupNonPrefixes(node) { children -> Group(newId(), formatGeneric(children, SpaceOrLine)) } + return Nodes(nodes) + } + + private fun formatTypealiasHeader(node: Node): FormatNode { + return Group(newId(), formatGeneric(node.children, Space)) + } + + private fun formatTypealiasBody(node: Node): FormatNode { + return Indent(formatGeneric(node.children, SpaceOrLine)) + } + + private fun formatClass(node: Node): FormatNode { + return Nodes(formatGeneric(node.children, SpaceOrLine)) + } + + private fun formatClassHeader(node: Node): FormatNode { + return groupOnSpace(formatGeneric(node.children, SpaceOrLine)) + } + + private fun formatClassHeaderExtends(node: Node): FormatNode { + return indent(Group(newId(), formatGeneric(node.children, SpaceOrLine))) + } + + private fun formatClassBody(node: Node): FormatNode { + val children = node.children + if (children.size == 2) { + // no members + return Nodes(formatGeneric(children, null)) + } + return Group(newId(), formatGeneric(children, ForceLine)) + } + + private fun formatClassBodyElements(node: Node): FormatNode { + val nodes = + formatGeneric(node.children) { prev, next -> + val lineDiff = prev.linesBetween(next) + if (lineDiff > 1 || lineDiff == 0) TWO_NEWLINES else ForceLine + } + return Indent(nodes) + } + + private fun formatClassProperty(node: Node): FormatNode { + val sameLine = + node.children + .lastOrNull { it.isExpressionOrPropertyBody() } + ?.let { + if (it.type.isExpression) isSameLineExpr(it) else isSameLineExpr(it.children.last()) + } ?: false + val nodes = + groupNonPrefixes(node) { children -> + val nodes = + formatGenericWithGen(children, { _, _ -> if (sameLine) Space else SpaceOrLine }) { node, _ + -> + if ((node.isExpressionOrPropertyBody()) && !sameLine) { + indent(format(node)) + } else format(node) + } + groupOnSpace(nodes) + } + return Nodes(nodes) + } + + private fun Node.isExpressionOrPropertyBody(): Boolean = + type.isExpression || + type == NodeType.CLASS_PROPERTY_BODY || + type == NodeType.OBJECT_PROPERTY_BODY + + private fun formatClassPropertyHeader(node: Node): FormatNode { + return Group(newId(), formatGeneric(node.children, SpaceOrLine)) + } + + private fun formatClassPropertyHeaderBegin(node: Node): FormatNode { + return Group(newId(), formatGeneric(node.children, SpaceOrLine)) + } + + private fun formatClassPropertyBody(node: Node): FormatNode { + return Nodes(formatGeneric(node.children, null)) + } + + private fun formatClassMethod(node: Node): FormatNode { + val prefixes = mutableListOf() + val nodes = + if (node.children[0].type == NodeType.CLASS_METHOD_HEADER) node.children + else { + val idx = node.children.indexOfFirst { it.type == NodeType.CLASS_METHOD_HEADER } + val prefixNodes = node.children.subList(0, idx) + prefixes += formatGeneric(prefixNodes, null) + prefixes += getSeparator(prefixNodes.last(), node.children[idx], ForceLine) + node.children.subList(idx, node.children.size) + } + + // Separate header (before =) and body (= and after) + val bodyIdx = nodes.indexOfFirst { it.type == NodeType.CLASS_METHOD_BODY } - 1 + val header = if (bodyIdx < 0) nodes else nodes.subList(0, bodyIdx) + val headerGroupId = newId() + val methodGroupId = newId() + val headerNodes = + formatGenericWithGen(header, SpaceOrLine) { node, _ -> + if (node.type == NodeType.PARAMETER_LIST) { + formatParameterList(node, id = headerGroupId) + } else { + format(node) + } + } + if (bodyIdx < 0) { + // body is empty, return header + return if (prefixes.isEmpty()) { + Group(headerGroupId, headerNodes) + } else { + Nodes(prefixes + Group(headerGroupId, headerNodes)) + } + } + + val bodyNodes = nodes.subList(bodyIdx, nodes.size) + + val expr = bodyNodes.last().children[0] + val isSameLineBody = isSameLineExpr(expr) + + // Format body (= and expression) + val bodyFormat = + if (isSameLineBody) { + formatGeneric(bodyNodes, Space) + } else { + formatGenericWithGen(bodyNodes, SpaceOrLine) { node, next -> + if (next == null) indent(format(node)) else format(node) + } + } + + val headerGroup = Group(headerGroupId, headerNodes) + val bodyGroup = Group(newId(), bodyFormat) + val separator = getSeparator(header.last(), bodyNodes.first(), Space) + val allNodes = Group(methodGroupId, listOf(headerGroup, separator, bodyGroup)) + + return if (prefixes.isEmpty()) allNodes else Nodes(prefixes + allNodes) + } + + private fun formatClassMethodHeader(node: Node): FormatNode { + val nodes = formatGeneric(node.children, Space) + return Nodes(nodes) + } + + private fun formatClassMethodBody(node: Node): FormatNode { + return Group(newId(), formatGeneric(node.children, null)) + } + + private fun formatParameter(node: Node): FormatNode { + if (node.children.size == 1) return format(node.children[0]) // underscore + return Group(newId(), formatGeneric(node.children, SpaceOrLine)) + } + + private fun formatParameterList(node: Node, id: Int? = null): FormatNode { + if (node.children.size == 2) return Text("()") + val groupId = id ?: newId() + val nodes = + formatGeneric(node.children) { prev, next -> + if (prev.isTerminal("(") || next.isTerminal(")")) { + if (next.isTerminal(")")) { + // trailing comma + IfWrap(groupId, nodes(Text(","), Line), Line) + } else Line + } else SpaceOrLine + } + return if (id != null) Nodes(nodes) else Group(groupId, nodes) + } + + private fun formatArgumentList(node: Node, twoBy2: Boolean = false): FormatNode { + if (node.children.size == 2) return Text("()") + val hasTrailingLambda = hasTrailingLambda(node) + val groupId = newId() + val nodes = + formatGenericWithGen( + node.children, + { prev, next -> + if (prev.isTerminal("(") || next.isTerminal(")")) { + val node = if (hasTrailingLambda) Empty else Line + if (next.isTerminal(")") && !hasTrailingLambda) { + // trailing comma + IfWrap(groupId, nodes(Text(","), node), node) + } else node + } else SpaceOrLine + }, + ) { node, _ -> + if (node.type == NodeType.ARGUMENT_LIST_ELEMENTS) { + formatArgumentListElements(node, hasTrailingLambda, twoBy2 = twoBy2) + } else format(node) + } + return Group(groupId, nodes) + } + + private fun formatArgumentListElements( + node: Node, + hasTrailingLambda: Boolean = false, + twoBy2: Boolean = false, + ): FormatNode { + val children = node.children + return if (twoBy2) { + val pairs = pairArguments(children) + val nodes = + formatGenericWithGen(pairs, SpaceOrLine) { node, _ -> + if (node.type == NodeType.ARGUMENT_LIST_ELEMENTS) { + Group(newId(), formatGeneric(node.children, SpaceOrLine)) + } else { + format(node) + } + } + Indent(nodes) + } else if (hasTrailingLambda) { + // if the args have a trailing expression (lambda, new, amends) group them differently + val splitIndex = children.indexOfLast { it.type in SAME_LINE_EXPRS } + val normalParams = children.subList(0, splitIndex) + val lastParam = children.subList(splitIndex, children.size) + val trailingNode = if (endsWithClosingBracket(children[splitIndex])) Empty else Line + val lastNodes = formatGeneric(lastParam, SpaceOrLine) + if (normalParams.isEmpty()) { + nodes(Group(newId(), lastNodes), trailingNode) + } else { + val separator = getSeparator(normalParams.last(), lastParam[0], Space) + val paramNodes = formatGeneric(normalParams, SpaceOrLine) + nodes(Group(newId(), paramNodes), separator, Group(newId(), lastNodes), trailingNode) + } + } else { + Indent(formatGeneric(children, SpaceOrLine)) + } + } + + private tailrec fun endsWithClosingBracket(node: Node): Boolean { + return if (node.children.isNotEmpty()) { + endsWithClosingBracket(node.children.last()) + } else { + node.isTerminal("}") + } + } + + private fun hasTrailingLambda(argList: Node): Boolean { + val children = argList.firstProperChild()?.children ?: return false + for (i in children.lastIndex downTo 0) { + val child = children[i] + if (!child.isProper()) continue + return child.type in SAME_LINE_EXPRS + } + return false + } + + private fun pairArguments(nodes: List): List { + val res = mutableListOf() + var tmp = mutableListOf() + var commas = 0 + for (node in nodes) { + if (node.isTerminal(",")) { + commas++ + if (commas == 2) { + res += Node(NodeType.ARGUMENT_LIST_ELEMENTS, tmp) + res += node + commas = 0 + tmp = mutableListOf() + } else { + tmp += node + } + } else { + tmp += node + } + } + if (tmp.isNotEmpty()) { + res += Node(NodeType.ARGUMENT_LIST_ELEMENTS, tmp) + } + return res + } + + private fun formatParameterListElements(node: Node): FormatNode { + return Indent(formatGeneric(node.children, SpaceOrLine)) + } + + private fun formatTypeParameterList(node: Node): FormatNode { + if (node.children.size == 2) return Text("<>") + val id = newId() + val nodes = + formatGeneric(node.children) { prev, next -> + if (prev.isTerminal("<") || next.isTerminal(">")) { + if (next.isTerminal(">")) { + // trailing comma + IfWrap(id, nodes(Text(","), Line), Line) + } else Line + } else SpaceOrLine + } + return Group(id, nodes) + } + + private fun formatObjectParameterList(node: Node): FormatNode { + // object param lists don't have trailing commas, as they have a trailing -> + val groupId = newId() + val nonWrappingNodes = Nodes(formatGeneric(node.children, SpaceOrLine)) + // double indent the params if they wrap + val wrappingNodes = indent(Indent(listOf(Line) + nonWrappingNodes)) + return Group(groupId, listOf(IfWrap(groupId, wrappingNodes, nodes(Space, nonWrappingNodes)))) + } + + private fun formatObjectBody(node: Node): FormatNode { + if (node.children.size == 2) return Text("{}") + val groupId = newId() + val nodes = + formatGenericWithGen( + node.children, + { prev, next -> + if (next.type == NodeType.OBJECT_PARAMETER_LIST) Empty + else if (prev.isTerminal("{") || next.isTerminal("}")) { + val lines = prev.linesBetween(next) + if (lines == 0) SpaceOrLine else ForceLine + } else SpaceOrLine + }, + ) { node, _ -> + if (node.type == NodeType.OBJECT_MEMBER_LIST) { + formatObjectMemberList(node, groupId) + } else format(node) + } + return Group(groupId, nodes) + } + + private fun formatObjectMemberList(node: Node, groupId: Int): FormatNode { + val nodes = + formatGeneric(node.children) { prev, next -> + val lines = prev.linesBetween(next) + when (lines) { + 0 -> IfWrap(groupId, Line, Text("; ")) + 1 -> ForceLine + else -> TWO_NEWLINES + } + } + return Indent(nodes) + } + + private fun formatObjectEntryHeader(node: Node): FormatNode { + return Group(newId(), formatGeneric(node.children, SpaceOrLine)) + } + + private fun formatForGenerator(node: Node): FormatNode { + val nodes = + formatGeneric(node.children) { prev, next -> + if ( + prev.type == NodeType.FOR_GENERATOR_HEADER || next.type == NodeType.FOR_GENERATOR_HEADER + ) { + Space + } else SpaceOrLine + } + return Group(newId(), nodes) + } + + private fun formatForGeneratorHeader(node: Node): FormatNode { + val nodes = + formatGeneric(node.children) { prev, next -> + if (prev.isTerminal("(") || next.isTerminal(")")) Line else null + } + return Group(newId(), nodes) + } + + private fun formatForGeneratorHeaderDefinition(node: Node): FormatNode { + val nodes = + formatGenericWithGen( + node.children, + { _, next -> if (next.type in SAME_LINE_EXPRS) Space else SpaceOrLine }, + ) { node, _ -> + if (node.type.isExpression && node.type !in SAME_LINE_EXPRS) indent(format(node)) + else format(node) + } + return indent(Group(newId(), nodes)) + } + + private fun formatForGeneratorHeaderDefinitionHeader(node: Node): FormatNode { + val nodes = formatGeneric(node.children, SpaceOrLine) + return Group(newId(), nodes) + } + + private fun formatWhenGenerator(node: Node): FormatNode { + val nodes = + formatGeneric(node.children) { prev, next -> + if ( + prev.type == NodeType.WHEN_GENERATOR_HEADER || + prev.isTerminal("when", "else") || + next.isTerminal("else") + ) { + Space + } else { + SpaceOrLine + } + } + return Group(newId(), nodes) + } + + private fun formatWhenGeneratorHeader(node: Node): FormatNode { + val nodes = + formatGenericWithGen( + node.children, + { prev, next -> if (prev.isTerminal("(") || next.isTerminal(")")) Line else SpaceOrLine }, + ) { node, _ -> + if (!node.type.isAffix && node.type != NodeType.TERMINAL) { + indent(format(node)) + } else format(node) + } + return Group(newId(), nodes) + } + + private fun formatMemberPredicate(node: Node): FormatNode { + val nodes = + formatGenericWithGen(node.children, SpaceOrLine) { node, next -> + if (next == null && node.type != NodeType.OBJECT_BODY) { + indent(format(node)) + } else format(node) + } + return Group(newId(), nodes) + } + + private fun formatMultilineString(node: Node): FormatNode { + val nodes = formatGeneric(node.children, null) + return MultilineStringGroup(node.children.last().span.colBegin, nodes) + } + + private fun formatIf(node: Node): FormatNode { + val nodes = + formatGeneric(node.children) { _, next -> + if (next.type == NodeType.IF_ELSE_EXPR && next.children[0].type == NodeType.IF_EXPR) { + Space + } else SpaceOrLine + } + return Group(newId(), nodes) + } + + private fun formatIfHeader(node: Node): FormatNode { + val nodes = + formatGeneric(node.children) { _, next -> + if (next.type == NodeType.IF_CONDITION) Space else SpaceOrLine + } + return Group(newId(), nodes) + } + + private fun formatIfCondition(node: Node): FormatNode { + val nodes = + formatGeneric(node.children) { prev, next -> + if (prev.isTerminal("(") || next.isTerminal(")")) Line else SpaceOrLine + } + return Group(newId(), nodes) + } + + private fun formatIfThen(node: Node): FormatNode { + return Indent(formatGeneric(node.children, null)) + } + + private fun formatIfElse(node: Node): FormatNode { + val children = node.children + if (children.size == 1) { + val expr = children[0] + return if (expr.type == NodeType.IF_EXPR) { + // unpack the group + val group = formatIf(expr) as Group + Nodes(group.nodes) + } else { + indent(format(expr)) + } + } + return Indent(formatGeneric(node.children, null)) + } + + private fun formatNewExpr(node: Node): FormatNode { + val nodes = formatGeneric(node.children, SpaceOrLine) + return Group(newId(), nodes) + } + + private fun formatNewHeader(node: Node): FormatNode { + val nodes = formatGeneric(node.children, SpaceOrLine) + return Group(newId(), nodes) + } + + private fun formatParenthesizedExpr(node: Node): FormatNode { + if (node.children.size == 2) return Text("()") + val nodes = + formatGenericWithGen( + node.children, + { prev, next -> if (prev.isTerminal("(") || next.isTerminal(")")) Line else SpaceOrLine }, + ) { node, _ -> + if (node.type.isExpression) indent(format(node)) else format(node) + } + return Group(newId(), nodes) + } + + private fun formatParenthesizedExprElements(node: Node): FormatNode { + return indent(Group(newId(), formatGeneric(node.children, null))) + } + + private fun formatFunctionLiteralExpr(node: Node): FormatNode { + val (params, rest) = node.children.splitOn { it.isTerminal("->") } + val sameLine = + node.children + .last { it.type == NodeType.FUNCTION_LITERAL_BODY } + .let { body -> + val expr = body.children.find { it.type.isExpression }!! + isSameLineExpr(expr) + } + val sep = if (sameLine) Space else SpaceOrLine + val bodySep = getSeparator(params.last(), rest.first(), sep) + + val nodes = formatGeneric(params, sep) + val restNodes = listOf(bodySep) + formatGeneric(rest, sep) + return Group(newId(), nodes + listOf(Group(newId(), restNodes))) + } + + private fun formatFunctionLiteralBody(node: Node): FormatNode { + val expr = node.children.find { it.type.isExpression }!! + val nodes = formatGeneric(node.children, null) + return if (isSameLineExpr(expr)) Group(newId(), nodes) else Indent(nodes) + } + + private fun formatLetExpr(node: Node): FormatNode { + val nodes = + formatGenericWithGen( + node.children, + { _, next -> if (next.type == NodeType.LET_PARAMETER_DEFINITION) Space else SpaceOrLine }, + ) { node, next -> + if (next == null) { + if (node.type == NodeType.LET_EXPR) { + // unpack the lets + val group = formatLetExpr(node) as Group + Nodes(group.nodes) + } else indent(format(node)) + } else format(node) + } + return Group(newId(), nodes) + } + + private fun formatLetParameterDefinition(node: Node): FormatNode { + val nodes = + formatGeneric(node.children) { prev, next -> + if (prev.isTerminal("(")) null else if (next.isTerminal(")")) Line else SpaceOrLine + } + return Group(newId(), nodes) + } + + private fun formatLetParameter(node: Node): FormatNode { + return indent(formatClassProperty(node)) + } + + private fun formatBinaryOpExpr(node: Node): FormatNode { + val flat = flattenBinaryOperatorExprs(node) + val callChainSize = flat.count { it.isOperator(".", "?.") } + val hasLambda = callChainSize > 1 && flat.any { hasFunctionLiteral(it, 2) } + val nodes = + formatGeneric(flat) { prev, next -> + if (prev.type == NodeType.OPERATOR) { + when (prev.text()) { + ".", + "?." -> null + "-" -> SpaceOrLine + else -> Space + } + } else if (next.type == NodeType.OPERATOR) { + when (next.text()) { + ".", + "?." -> if (hasLambda) ForceLine else Line + "-" -> Space + else -> SpaceOrLine + } + } else SpaceOrLine + } + val shouldGroup = node.children.size == flat.size + return Group(newId(), indentAfterFirstNewline(nodes, shouldGroup)) + } + + private fun hasFunctionLiteral(node: Node, depth: Int): Boolean { + if (node.type == NodeType.FUNCTION_LITERAL_EXPR) return true + for (child in node.children) { + if (child.type == NodeType.FUNCTION_LITERAL_EXPR) return true + if (depth > 0 && hasFunctionLiteral(child, depth - 1)) return true + } + return false + } + + private fun formatSubscriptExpr(node: Node): FormatNode { + return Nodes(formatGeneric(node.children, null)) + } + + private fun formatTraceThrowReadExpr(node: Node): FormatNode { + val nodes = + formatGenericWithGen( + node.children, + { prev, next -> if (prev.isTerminal("(") || next.isTerminal(")")) Line else null }, + ) { node, _ -> + if (node.type.isExpression) indent(format(node)) else format(node) + } + return Group(newId(), nodes) + } + + private fun formatDeclaredType(node: Node): FormatNode { + return Nodes(formatGeneric(node.children, SpaceOrLine)) + } + + private fun formatConstrainedType(node: Node): FormatNode { + val nodes = + formatGeneric(node.children) { _, next -> + if (next.type == NodeType.CONSTRAINED_TYPE_CONSTRAINT) null else SpaceOrLine + } + return Group(newId(), nodes) + } + + private fun formatUnionType(node: Node): FormatNode { + val nodes = + formatGeneric(node.children) { prev, next -> + when { + next.isTerminal("|") -> SpaceOrLine + prev.isTerminal("|") -> Space + else -> null + } + } + return Group(newId(), indentAfterFirstNewline(nodes)) + } + + private fun formatFunctionType(node: Node): FormatNode { + val nodes = + formatGenericWithGen( + node.children, + { prev, next -> if (prev.isTerminal("(") || next.isTerminal(")")) Line else SpaceOrLine }, + ) { node, next -> + if (next == null) indent(format(node)) else format(node) + } + return Group(newId(), nodes) + } + + private fun formatParenthesizedType(node: Node): FormatNode { + if (node.children.size == 2) return Text("()") + val groupId = newId() + val nodes = + formatGeneric(node.children) { prev, next -> + if (prev.isTerminal("(") || next.isTerminal(")")) Line else SpaceOrLine + } + return Group(groupId, nodes) + } + + private fun formatParenthesizedTypeElements(node: Node): FormatNode { + return indent(Group(newId(), formatGeneric(node.children, SpaceOrLine))) + } + + private fun formatTypeAnnotation(node: Node): FormatNode { + return Group(newId(), formatGeneric(node.children, Space)) + } + + private fun formatModifierList(node: Node): FormatNode { + val nodes = mutableListOf() + val children = node.children.groupBy { it.type.isAffix } + if (children[true] != null) { + nodes += formatGeneric(children[true]!!, SpaceOrLine) + } + val modifiers = children[false]!!.sortedBy(::modifierPrecedence) + nodes += formatGeneric(modifiers, Space) + return Nodes(nodes) + } + + private fun formatImportList(node: Node): FormatNode { + val nodes = mutableListOf() + val children = node.children.groupBy { it.type.isAffix } + if (children[true] != null) { + nodes += formatGeneric(children[true]!!, SpaceOrLine) + nodes += ForceLine + } + + val allImports = children[false]!! + val imports = allImports.groupBy { it.findChildByType(NodeType.TERMINAL)?.text(source) } + if (imports["import"] != null) { + formatImportListHelper(imports["import"]!!, nodes) + if (imports["import*"] != null) nodes += TWO_NEWLINES + } + if (imports["import*"] != null) { + formatImportListHelper(imports["import*"]!!, nodes) + } + + return Nodes(nodes) + } + + private fun formatImportListHelper(allImports: List, nodes: MutableList) { + val imports = + allImports.groupBy { imp -> + val url = getImportUrl(imp) + when { + ABSOLUTE_URL_REGEX.matches(url) -> 0 + url.startsWith('@') -> 1 + else -> 2 + } + } + val absolute = imports[0]?.sortedWith(ImportComparator(source)) + val projects = imports[1]?.sortedWith(ImportComparator(source)) + val relatives = imports[2]?.sortedWith(ImportComparator(source)) + var shouldNewline = false + + if (absolute != null) { + for ((i, imp) in absolute.withIndex()) { + if (i > 0) nodes += ForceLine + nodes += format(imp) + } + if (projects != null || relatives != null) nodes += ForceLine + shouldNewline = true + } + + if (projects != null) { + if (shouldNewline) nodes += ForceLine + for ((i, imp) in projects.withIndex()) { + if (i > 0) nodes += ForceLine + nodes += format(imp) + } + if (relatives != null) nodes += ForceLine + shouldNewline = true + } + + if (relatives != null) { + if (shouldNewline) nodes += ForceLine + for ((i, imp) in relatives.withIndex()) { + if (i > 0) nodes += ForceLine + nodes += format(imp) + } + } + } + + private fun formatGeneric(children: List, separator: FormatNode?): List { + return formatGeneric(children) { _, _ -> separator } + } + + private fun formatGeneric( + children: List, + separatorFn: (Node, Node) -> FormatNode?, + ): List { + return formatGenericWithGen(children, separatorFn, null) + } + + private fun formatGenericWithGen( + children: List, + separator: FormatNode?, + generatorFn: ((Node, Node?) -> FormatNode)?, + ): List { + return formatGenericWithGen(children, { _, _ -> separator }, generatorFn) + } + + private fun formatGenericWithGen( + children: List, + separatorFn: (Node, Node) -> FormatNode?, + generatorFn: ((Node, Node?) -> FormatNode)?, + ): List { + // skip semicolons + val children = children.filter { !it.isSemicolon() } + // short circuit + if (children.isEmpty()) return listOf(SpaceOrLine) + if (children.size == 1) return listOf(format(children[0])) + + val nodes = mutableListOf() + var prev = children[0] + for (child in children.drop(1)) { + nodes += + if (generatorFn != null) { + generatorFn(prev, child) + } else { + format(prev) + } + val separator = getSeparator(prev, child, separatorFn) + if (separator != null) nodes += separator + prev = child + } + nodes += + if (generatorFn != null) { + generatorFn(children.last(), null) + } else { + format(children.last()) + } + return nodes + } + + private fun Node.isSemicolon(): Boolean = type.isAffix && text() == ";" + + /** Groups all non prefixes (comments, doc comments, annotations) of this node together. */ + private fun groupNonPrefixes(node: Node, groupFn: (List) -> FormatNode): List { + val children = node.children + val index = + children.indexOfFirst { + !it.type.isAffix && it.type != NodeType.DOC_COMMENT && it.type != NodeType.ANNOTATION + } + if (index <= 0) { + // no prefixes + return listOf(groupFn(children)) + } + val prefixes = children.subList(0, index) + val nodes = children.subList(index, children.size) + val res = mutableListOf() + res += formatGeneric(prefixes, SpaceOrLine) + res += getSeparator(prefixes.last(), nodes.first()) + res += groupFn(nodes) + return res + } + + private fun getImportUrl(node: Node): String = + node.findChildByType(NodeType.STRING_CONSTANT)!!.text().drop(1).dropLast(1) + + private fun getSeparator( + prev: Node, + next: Node, + separator: FormatNode = SpaceOrLine, + ): FormatNode { + return getSeparator(prev, next) { _, _ -> separator }!! + } + + private fun getSeparator( + prev: Node, + next: Node, + separatorFn: (Node, Node) -> FormatNode?, + ): FormatNode? { + return when { + prevNode?.type == NodeType.LINE_COMMENT -> { + if (prev.linesBetween(next) > 1) { + TWO_NEWLINES + } else { + ForceLine + } + } + hasTrailingAffix(prev, next) -> Space + prev.type == NodeType.DOC_COMMENT || prev.type == NodeType.ANNOTATION -> ForceLine + prev.type in FORCE_LINE_AFFIXES || next.type.isAffix -> { + if (prev.linesBetween(next) > 1) { + TWO_NEWLINES + } else { + ForceLine + } + } + prev.type == NodeType.BLOCK_COMMENT -> if (prev.linesBetween(next) > 0) ForceLine else Space + next.type in EMPTY_SUFFIXES || + prev.isTerminal("[", "!", "@", "[[") || + next.isTerminal("]", "?", ",") -> null + prev.isTerminal("class", "function", "new") || + next.isTerminal("=", "{", "->", "class", "function") || + next.type == NodeType.OBJECT_BODY || + prev.type == NodeType.MODIFIER_LIST -> Space + next.type == NodeType.DOC_COMMENT -> TWO_NEWLINES + else -> separatorFn(prev, next) + } + } + + private fun hasTrailingAffix(node: Node, next: Node): Boolean { + if (node.span.lineEnd < next.span.lineBegin) return false + var n: Node? = next + while (n != null) { + if (n.type.isAffix && node.span.lineEnd == n.span.lineBegin) return true + n = n.children.getOrNull(0) + } + return false + } + + private fun modifierPrecedence(modifier: Node): Int { + return when (val text = modifier.text()) { + "abstract", + "open" -> 0 + "external" -> 1 + "local", + "hidden" -> 2 + "fixed", + "const" -> 3 + else -> throw RuntimeException("Unknown modifier `$text`") + } + } + + private fun isSameLineExpr(node: Node): Boolean { + return node.type in SAME_LINE_EXPRS + } + + private fun splitPrefixes(nodes: List): Pair, List> { + val splitPoint = nodes.indexOfFirst { !it.type.isAffix && it.type != NodeType.DOC_COMMENT } + return nodes.subList(0, splitPoint) to nodes.subList(splitPoint, nodes.size) + } + + private fun indentAfterFirstNewline( + nodes: List, + group: Boolean = false, + ): List { + val index = nodes.indexOfFirst { it is SpaceOrLine || it is ForceLine || it is Line } + if (index <= 0) return nodes + val indented = + if (group) { + group(Indent(nodes.subList(index, nodes.size))) + } else { + Indent(nodes.subList(index, nodes.size)) + } + + return nodes.subList(0, index) + listOf(indented) + } + + private fun groupOnSpace(fnodes: List): FormatNode { + val res = mutableListOf() + for ((i, node) in fnodes.withIndex()) { + if (i > 0 && (node is SpaceOrLine || node is Space)) { + res += groupOnSpace(fnodes.subList(i, fnodes.size)) + break + } else { + res += node + } + } + return Group(newId(), res) + } + + /** Flatten binary operators by precedence */ + private fun flattenBinaryOperatorExprs(node: Node): List { + val op = node.children.first { it.type == NodeType.OPERATOR }.text() + return flattenBinaryOperatorExprs(node, Operator.byName(op).prec) + } + + private fun flattenBinaryOperatorExprs(node: Node, prec: Int): List { + val actualOp = node.children.first { it.type == NodeType.OPERATOR }.text() + if (prec != Operator.byName(actualOp).prec) return listOf(node) + val res = mutableListOf() + for (child in node.children) { + if (child.type == NodeType.BINARY_OP_EXPR) { + res += flattenBinaryOperatorExprs(child, prec) + } else { + res += child + } + } + return res + } + + private fun Node.linesBetween(next: Node): Int = next.span.lineBegin - span.lineEnd + + private fun Node.text() = text(source) + + private fun Node.isTerminal(vararg texts: String): Boolean = + type == NodeType.TERMINAL && text(source) in texts + + private fun Node.isOperator(vararg texts: String): Boolean = + type == NodeType.OPERATOR && text(source) in texts + + private fun newId(): Int { + return id++ + } + + private fun nodes(vararg nodes: FormatNode) = Nodes(nodes.toList()) + + private fun group(vararg nodes: FormatNode) = Group(newId(), nodes.toList()) + + private fun indent(vararg nodes: FormatNode) = Indent(nodes.toList()) + + private class ImportComparator(private val source: CharArray) : Comparator { + override fun compare(o1: Node, o2: Node): Int { + val import1 = o1.findChildByType(NodeType.STRING_CONSTANT)?.text(source) + val import2 = o2.findChildByType(NodeType.STRING_CONSTANT)?.text(source) + if (import1 == null || import2 == null) { + // should never happen + throw RuntimeException("ImportComparator: not an import") + } + + return NaturalOrderComparator(ignoreCase = true).compare(import1, import2) + } + } + + private fun Node.firstProperChild(): Node? { + for (child in children) { + if (child.isProper()) return child + } + return null + } + + // returns true if this node is not an affix or terminal + private fun Node.isProper(): Boolean = !type.isAffix && type != NodeType.TERMINAL + + private fun List.splitOn(pred: (T) -> Boolean): Pair, List> { + val index = indexOfFirst { pred(it) } + return if (index == -1) { + Pair(this, emptyList()) + } else { + Pair(take(index), drop(index)) + } + } + + companion object { + private val ABSOLUTE_URL_REGEX = Regex("""\w+:.*""") + + private val TWO_NEWLINES = Nodes(listOf(ForceLine, ForceLine)) + + private val FORCE_LINE_AFFIXES = + EnumSet.of( + NodeType.DOC_COMMENT_LINE, + NodeType.LINE_COMMENT, + NodeType.SEMICOLON, + NodeType.SHEBANG, + ) + + private val EMPTY_SUFFIXES = + EnumSet.of( + NodeType.TYPE_ARGUMENT_LIST, + NodeType.TYPE_ANNOTATION, + NodeType.TYPE_PARAMETER_LIST, + NodeType.PARAMETER_LIST, + ) + + private val SAME_LINE_EXPRS = + EnumSet.of(NodeType.NEW_EXPR, NodeType.AMENDS_EXPR, NodeType.FUNCTION_LITERAL_EXPR) + } +} diff --git a/pkl-formatter/src/main/kotlin/org/pkl/formatter/Formatter.kt b/pkl-formatter/src/main/kotlin/org/pkl/formatter/Formatter.kt new file mode 100644 index 00000000..47dd9982 --- /dev/null +++ b/pkl-formatter/src/main/kotlin/org/pkl/formatter/Formatter.kt @@ -0,0 +1,53 @@ +/* + * Copyright © 2025 Apple Inc. and the Pkl project authors. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.pkl.formatter + +import java.nio.file.Files +import java.nio.file.Path +import org.pkl.formatter.ast.ForceLine +import org.pkl.formatter.ast.Nodes +import org.pkl.parser.GenericParser + +/** A formatter for Pkl files that applies canonical formatting rules. */ +class Formatter { + /** + * Formats a Pkl file from the given file path. + * + * @param path the path to the Pkl file to format + * @return the formatted Pkl source code as a string + * @throws java.io.IOException if the file cannot be read + */ + fun format(path: Path): String { + return format(Files.readString(path)) + } + + /** + * Formats the given Pkl source code text. + * + * @param text the Pkl source code to format + * @return the formatted Pkl source code as a string + */ + fun format(text: String): String { + val parser = GenericParser() + val builder = Builder(text) + val gen = Generator() + val ast = parser.parseModule(text) + val formatAst = builder.format(ast) + // force a line at the end of the file + gen.generate(Nodes(listOf(formatAst, ForceLine))) + return gen.toString() + } +} diff --git a/pkl-formatter/src/main/kotlin/org/pkl/formatter/Generator.kt b/pkl-formatter/src/main/kotlin/org/pkl/formatter/Generator.kt new file mode 100644 index 00000000..e5c9f8f2 --- /dev/null +++ b/pkl-formatter/src/main/kotlin/org/pkl/formatter/Generator.kt @@ -0,0 +1,149 @@ +/* + * Copyright © 2025 Apple Inc. and the Pkl project authors. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.pkl.formatter + +import org.pkl.formatter.ast.Empty +import org.pkl.formatter.ast.ForceLine +import org.pkl.formatter.ast.ForceWrap +import org.pkl.formatter.ast.FormatNode +import org.pkl.formatter.ast.Group +import org.pkl.formatter.ast.IfWrap +import org.pkl.formatter.ast.Indent +import org.pkl.formatter.ast.Line +import org.pkl.formatter.ast.MultilineStringGroup +import org.pkl.formatter.ast.Nodes +import org.pkl.formatter.ast.Space +import org.pkl.formatter.ast.SpaceOrLine +import org.pkl.formatter.ast.Text +import org.pkl.formatter.ast.Wrap + +internal class Generator { + private val buf: StringBuilder = StringBuilder() + private var indent: Int = 0 + private var size: Int = 0 + private val wrapped: MutableSet = mutableSetOf() + private var shouldAddIndent = false + + fun generate(node: FormatNode) { + node(node, Wrap.DETECT) + } + + private fun node(node: FormatNode, wrap: Wrap) { + when (node) { + is Empty -> {} + is Nodes -> node.nodes.forEach { node(it, wrap) } + is Group -> { + val width = node.nodes.sumOf { it.width(wrapped) } + val wrap = + if (size + width > MAX) { + wrapped += node.id + Wrap.ENABLED + } else { + Wrap.DETECT + } + node.nodes.forEach { node(it, wrap) } + } + is ForceWrap -> { + wrapped += node.id + val wrap = Wrap.ENABLED + node.nodes.forEach { node(it, wrap) } + } + is IfWrap -> { + if (wrapped.contains(node.id)) { + node(node.ifWrap, Wrap.ENABLED) + } else { + node(node.ifNotWrap, wrap) + } + } + is Text -> text(node.text) + is Line -> { + if (wrap.isEnabled()) { + newline() + } + } + is ForceLine -> newline() + is SpaceOrLine -> { + if (wrap.isEnabled()) { + newline() + } else { + text(" ") + } + } + is Space -> text(" ") + is Indent -> { + if (wrap.isEnabled() && node.nodes.isNotEmpty()) { + size += INDENT.length + indent++ + node.nodes.forEach { node(it, wrap) } + indent-- + } else { + node.nodes.forEach { node(it, wrap) } + } + } + is MultilineStringGroup -> { + val indentLength = indent * INDENT.length + val offset = (indentLength + 1) - node.endQuoteCol + var previousNewline = false + for ((i, child) in node.nodes.withIndex()) { + when { + child is ForceLine -> newline(shouldIndent = false) // don't indent + child is Text && + previousNewline && + child.text.isBlank() && + child.text.length == indentLength && + node.nodes[i + 1] is ForceLine -> {} + child is Text && previousNewline && offset != 0 -> text(reposition(child.text, offset)) + else -> node(child, Wrap.DETECT) // always detect wrapping + } + previousNewline = child is ForceLine + } + } + } + } + + private fun text(value: String) { + if (shouldAddIndent) { + repeat(times = indent) { buf.append(INDENT) } + shouldAddIndent = false + } + size += value.length + buf.append(value) + } + + private fun newline(shouldIndent: Boolean = true) { + size = INDENT.length * indent + buf.append('\n') + shouldAddIndent = shouldIndent + } + + private fun reposition(text: String, offset: Int): String { + return if (offset > 0) { + " ".repeat(offset) + text + } else { + text.drop(-offset) + } + } + + override fun toString(): String { + return buf.toString() + } + + companion object { + // max line length + const val MAX = 100 + private const val INDENT = " " + } +} diff --git a/pkl-formatter/src/main/kotlin/org/pkl/formatter/NaturalOrderComparator.kt b/pkl-formatter/src/main/kotlin/org/pkl/formatter/NaturalOrderComparator.kt new file mode 100644 index 00000000..ab5e70fa --- /dev/null +++ b/pkl-formatter/src/main/kotlin/org/pkl/formatter/NaturalOrderComparator.kt @@ -0,0 +1,66 @@ +/* + * Copyright © 2025 Apple Inc. and the Pkl project authors. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.pkl.formatter + +internal class NaturalOrderComparator(private val ignoreCase: Boolean = false) : + Comparator { + + override fun compare(s1: String, s2: String): Int { + var i = 0 + var j = 0 + + while (i < s1.length && j < s2.length) { + val c1 = if (ignoreCase) s1[i].lowercaseChar() else s1[i] + val c2 = if (ignoreCase) s2[j].lowercaseChar() else s2[j] + + if (c1.isDigit() && c2.isDigit()) { + val (num1, nextI) = getNumber(s1, i) + val (num2, nextJ) = getNumber(s2, j) + + val numComparison = num1.compareTo(num2) + if (numComparison != 0) { + return numComparison + } + i = nextI + j = nextJ + } else { + val charComparison = c1.compareTo(c2) + if (charComparison != 0) { + return charComparison + } + i++ + j++ + } + } + + return s1.length.compareTo(s2.length) + } + + private fun getNumber(s: String, startIndex: Int): LongAndInt { + var i = startIndex + val start = i + + while (i < s.length && s[i].isDigit()) { + i++ + } + val numStr = s.substring(start, i) + val number = numStr.toLongOrNull() ?: 0L + return LongAndInt(number, i) + } + + // use this instead of Pair to avoid boxing + private data class LongAndInt(val l: Long, var i: Int) +} diff --git a/pkl-formatter/src/main/kotlin/org/pkl/formatter/ast/FormatNode.kt b/pkl-formatter/src/main/kotlin/org/pkl/formatter/ast/FormatNode.kt new file mode 100644 index 00000000..37361a4f --- /dev/null +++ b/pkl-formatter/src/main/kotlin/org/pkl/formatter/ast/FormatNode.kt @@ -0,0 +1,66 @@ +/* + * Copyright © 2025 Apple Inc. and the Pkl project authors. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.pkl.formatter.ast + +import org.pkl.formatter.Generator + +enum class Wrap { + ENABLED, + DETECT; + + fun isEnabled(): Boolean = this == ENABLED +} + +sealed interface FormatNode { + fun width(wrapped: Set): Int = + when (this) { + is Nodes -> nodes.sumOf { it.width(wrapped) } + is Group -> nodes.sumOf { it.width(wrapped) } + is Indent -> nodes.sumOf { it.width(wrapped) } + is ForceWrap -> nodes.sumOf { it.width(wrapped + id) } + is IfWrap -> if (id in wrapped) ifWrap.width(wrapped) else ifNotWrap.width(wrapped) + is Text -> text.length + is SpaceOrLine, + is Space -> 1 + is ForceLine, + is MultilineStringGroup -> Generator.MAX + else -> 0 + } +} + +data class Text(val text: String) : FormatNode + +object Empty : FormatNode + +object Line : FormatNode + +object ForceLine : FormatNode + +object SpaceOrLine : FormatNode + +object Space : FormatNode + +data class Indent(val nodes: List) : FormatNode + +data class Nodes(val nodes: List) : FormatNode + +data class Group(val id: Int, val nodes: List) : FormatNode + +data class ForceWrap(val id: Int, val nodes: List) : FormatNode + +data class MultilineStringGroup(val endQuoteCol: Int, val nodes: List) : FormatNode + +data class IfWrap(val id: Int, val ifWrap: FormatNode, val ifNotWrap: FormatNode) : FormatNode diff --git a/pkl-formatter/src/test/files/FormatterSnippetTests/input/class-bodies.pkl b/pkl-formatter/src/test/files/FormatterSnippetTests/input/class-bodies.pkl new file mode 100644 index 00000000..85c19c7f --- /dev/null +++ b/pkl-formatter/src/test/files/FormatterSnippetTests/input/class-bodies.pkl @@ -0,0 +1,10 @@ +class Foo { + +} + +class Bar +{ + qux = 1 +} + +class Baz { prop = 0; prop2 = 1 } diff --git a/pkl-formatter/src/test/files/FormatterSnippetTests/input/comma-termination.pkl b/pkl-formatter/src/test/files/FormatterSnippetTests/input/comma-termination.pkl new file mode 100644 index 00000000..a9637590 --- /dev/null +++ b/pkl-formatter/src/test/files/FormatterSnippetTests/input/comma-termination.pkl @@ -0,0 +1,16 @@ +function fun(a, b, c, d) = a + +noTrailingCommas = fun(1, 2, 3, 4,) + +trailingCommas = fun("loooooooooooooooooooongString", "loooooooooooooooongString", "looooooooooooooooongString", "notTooLong") + +noTrailingCommaObjParams { + fooooooooooooooooooooooooooooooooo, baaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaar, baaaaaaaaaaaaaaaaaaaaaz -> + 1 + 1 +} + +trailingCommaInLambdas = (fooooooooooooooooooooooooooooooooo, baaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaar, baaaaaaaaaaaaaaaaaaaaaz) -> 1 + +trailingCommaInConstraints: String(isSomethingSomethingSomething, isSomethingElse, isSomethingSomethingSomethingElse) + +trailingCommaInTypeParameters: Mapping diff --git a/pkl-formatter/src/test/files/FormatterSnippetTests/input/comment-interleaved.pkl b/pkl-formatter/src/test/files/FormatterSnippetTests/input/comment-interleaved.pkl new file mode 100644 index 00000000..95fad6ce --- /dev/null +++ b/pkl-formatter/src/test/files/FormatterSnippetTests/input/comment-interleaved.pkl @@ -0,0 +1,31 @@ +module comment.interleaved + +local test: Int|String = + if (true) + 1 // It's the same as "100%" + else + "8%" + +foo: ( // some comment + * String( + // if dtstart is defined and a datetime, until must also be a datetime if defined + true + ) + |Int + // trailing comment + )? + +foo2: (// some comment + Int, // other comment + String + ) ->Int + +bar = ( // some comment + 10 + 10 + // another comment + ) + +bar2 = ( // some comment + foo, bar + // another comment + ) -> foo + bar diff --git a/pkl-formatter/src/test/files/FormatterSnippetTests/input/dangling-doc-comment.pkl b/pkl-formatter/src/test/files/FormatterSnippetTests/input/dangling-doc-comment.pkl new file mode 100644 index 00000000..0ac2d494 --- /dev/null +++ b/pkl-formatter/src/test/files/FormatterSnippetTests/input/dangling-doc-comment.pkl @@ -0,0 +1,8 @@ +module dangling + +/// Doc comment. +/// +/// for this field +// sepearted by a stray comment +/// but continues here. +some: String diff --git a/pkl-formatter/src/test/files/FormatterSnippetTests/input/doc-comments.pkl b/pkl-formatter/src/test/files/FormatterSnippetTests/input/doc-comments.pkl new file mode 100644 index 00000000..44275387 --- /dev/null +++ b/pkl-formatter/src/test/files/FormatterSnippetTests/input/doc-comments.pkl @@ -0,0 +1,14 @@ + +//// line 1 +///// line 2 +foo = 1 + +/// line 1 +/// +/// +/// line 2 +bar = 1 + +///line 1 +///line 2 +baz = 1 diff --git a/pkl-formatter/src/test/files/FormatterSnippetTests/input/expr-binary.pkl b/pkl-formatter/src/test/files/FormatterSnippetTests/input/expr-binary.pkl new file mode 100644 index 00000000..866d05c5 --- /dev/null +++ b/pkl-formatter/src/test/files/FormatterSnippetTests/input/expr-binary.pkl @@ -0,0 +1,25 @@ +foo { + 123123123123 + 123123123123 * 123123123123 - 123123123123 / 123123123123 + 123123123123 ** 123123123123 + 2 + 3 + 4 +} + +bar = clazz.superclass == null || clazz.superclass.reflectee == Module || clazz.superclass.reflectee == Typed + +baz = +new Listing { + 1 + 2 +} |> mixin1 |> mixin2 + +qux = + (baz) { + 1 + 2 + } { + 3 + 4 + } + +minus = superLoooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooongVariable - 100000 diff --git a/pkl-formatter/src/test/files/FormatterSnippetTests/input/expr-chain.pkl b/pkl-formatter/src/test/files/FormatterSnippetTests/input/expr-chain.pkl new file mode 100644 index 00000000..1b9ff4b1 --- /dev/null +++ b/pkl-formatter/src/test/files/FormatterSnippetTests/input/expr-chain.pkl @@ -0,0 +1,5 @@ +res = myList.map((it) -> it.partition).filter((it) -> someList.contains(it)) + +res2 = myList.map(lambda1).filter(lambda2) + +res3 = myList.map((it) -> it.partition) diff --git a/pkl-formatter/src/test/files/FormatterSnippetTests/input/expr-if.pkl b/pkl-formatter/src/test/files/FormatterSnippetTests/input/expr-if.pkl new file mode 100644 index 00000000..5263672e --- /dev/null +++ b/pkl-formatter/src/test/files/FormatterSnippetTests/input/expr-if.pkl @@ -0,0 +1,7 @@ +foo = + if (someCondition) 10000 + else + if (someOtherCondition) 20000 + else + if (someAnotherCondition) 30000 + else 4 diff --git a/pkl-formatter/src/test/files/FormatterSnippetTests/input/expr-let.pkl b/pkl-formatter/src/test/files/FormatterSnippetTests/input/expr-let.pkl new file mode 100644 index 00000000..1cd0af5c --- /dev/null +++ b/pkl-formatter/src/test/files/FormatterSnippetTests/input/expr-let.pkl @@ -0,0 +1,11 @@ + +foo = let (vaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaariable = 10) 1 * 1 + +bar = let (someVariable = new Listing { + 1 + }) 1 * 1 + +baz = + let (someVariable = 10000000) + let (someOtherVariable = 2000000) + let (someAnotherVariable = 3000000) someVariable + someOtherVariable + someAnotherVariable diff --git a/pkl-formatter/src/test/files/FormatterSnippetTests/input/imports.pkl b/pkl-formatter/src/test/files/FormatterSnippetTests/input/imports.pkl new file mode 100644 index 00000000..a5ce641d --- /dev/null +++ b/pkl-formatter/src/test/files/FormatterSnippetTests/input/imports.pkl @@ -0,0 +1,19 @@ +// top level comment + +import "@foo/Foo.pkl" as foo +import* "**.pkl" +import "pkl:math" + +import "package://example.com/myPackage@1.0.0#/Qux.pkl" +import* "file:///tmp/*.pkl" + + +import "https://example.com/baz.pkl" +import "module2.pkl" +import "pkl:reflect" +import "..." +import* "@foo/**.pkl" +import "@bar/Bar.pkl" +import "Module12.pkl" +import "module11.pkl" +import "module1.pkl" diff --git a/pkl-formatter/src/test/files/FormatterSnippetTests/input/indentation.pkl b/pkl-formatter/src/test/files/FormatterSnippetTests/input/indentation.pkl new file mode 100644 index 00000000..e82fe3d7 --- /dev/null +++ b/pkl-formatter/src/test/files/FormatterSnippetTests/input/indentation.pkl @@ -0,0 +1,51 @@ +foo = new { +bar = 1 +} + +class Foo { + baz: Int +} + +bar = + new Listing { + 1 + 2 + } + +baz = +(foo) { + 2 + 3 +} + +qux = + (x, y) -> new Listing { + x + y + } + +forGen = new Listing { + for (someVar in new Listing { + 1 + 2 + }) { + [someVar] = someVar + } +} + +objParams: Listing = + new { + default { + key -> key + 1 + } +} + +objParams2: Listing = new { + default { someVeryLongParameter1, someVeryLongParameter2, someVeryLongParameter3, someVeryLongParameter4 -> + 1 + } +} + +parenType: (ReaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaalyLongType(reaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaalylongConstraint)) + +functionType: (ReaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaalyLongType,ReaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaalyLongType2) -> String diff --git a/pkl-formatter/src/test/files/FormatterSnippetTests/input/line-breaks.pkl b/pkl-formatter/src/test/files/FormatterSnippetTests/input/line-breaks.pkl new file mode 100644 index 00000000..836c9dfe --- /dev/null +++ b/pkl-formatter/src/test/files/FormatterSnippetTests/input/line-breaks.pkl @@ -0,0 +1,64 @@ +module +foo.bar.baz +amends +"bar.pkl" + +import +"@foo/Foo.pkl" +as foo + +local +open +class Bar {} + +const +local +baz = 10 + +local +function +fun(x) = + x + +const local prooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooperty: String = "foo" + +local function function2(parameter1: Parameter1Type, parameter2: Parameter2Type, parameter3: Parameter3Type, parameter4: Parameter4Type): String = "" + +local const function function3(parameter1: String|Int, parameter2: String|Int): Mapping = + new {} + +prop = function2(loooooooooooooooooogParameter1, loooooooooooooooooogParameter2, loooooooooooooooooogParameter3, loooooooooooooooooogParameter4) + +prop2: String + |Int + |Boolean + +funcParam = fun( + (x, y) -> new Listing { + x + y +}) + +funcParam2 = aFun(foo, 10 * 10, anotherVariable, if (true) 100000 else 200000, (param1, param2) -> param1 * param2) + +funcParam3 = aFun(foo, 10 * 10, anotherVariable, 200000, (param1, param2) -> param1 * param2) + +funcParam4 = aFun(foo, 10 * 10, anotherVariable, if (true) 100000 else 200000, new Listing { + 1 + 2 +}) + +open local class SomeReallyInterestingClassName extends AnotherInterestingClassName { + foo: Int +} + +local function resourceMapping(type): Mapping = + new Mapping { default = (key) -> (type) { metadata { name = key } } } + +local const function biiiiiiiiiiiiiiiiiiiiiiiiigFunction(param1: String, param2: String(!isBlank)): Boolean + +local const function someFunction(param1: String, param2: String(!isBlank)): Boolean + +local function render(currentIndent: String) = + "\(currentIndent)@\(identifier.render(currentIndent))" + + if (body == null) "" else " " + body.render(currentIndent) diff --git a/pkl-formatter/src/test/files/FormatterSnippetTests/input/line-breaks2.pkl b/pkl-formatter/src/test/files/FormatterSnippetTests/input/line-breaks2.pkl new file mode 100644 index 00000000..e6c4f387 --- /dev/null +++ b/pkl-formatter/src/test/files/FormatterSnippetTests/input/line-breaks2.pkl @@ -0,0 +1,20 @@ +module reaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaly.loooooooooooooooooooooooooooooooog.naaaaaaaaaaaaaaaaaaaaaaaaaame +extends "reaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaalyLongModule.pkl" + +import "reaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaalyLongModule.pkl" as foo + +local open class LoooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooongName {} + +local open class ReaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaalyLongName {} + +const hidden loooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooogName = 99 + +const hidden reallyLoooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooogName = 99 + +const local function looooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooogName(x: Int, y) = x + y + +typealias Foo = LooooooooooooooooooooooooongTypeName|AnotherLooooooooooooooooooooooooongTypeName|OtherLooooooooooooooooooooooooongTypeName + +bar: Boolean|Mapping(loooooooooooooooooooogConstraint) + +hidden foobar: LongType(someVeryyyyyyyloooong, requirements.might.be.even_loooooooooooooooooooooooooooooooooooooooooooooooooooooooooonger) diff --git a/pkl-formatter/src/test/files/FormatterSnippetTests/input/line-width.pkl b/pkl-formatter/src/test/files/FormatterSnippetTests/input/line-width.pkl new file mode 100644 index 00000000..74cb3de9 --- /dev/null +++ b/pkl-formatter/src/test/files/FormatterSnippetTests/input/line-width.pkl @@ -0,0 +1,18 @@ +module reaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaly.loooooooooooooooooooooooooooooooog.naaaaaaaaaaaaaaaaaaaaaaaaaaaaame +extends "reaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaalyLongModule.pkl" + +local reallyLongVariableName = true +local anotherReallyLongName = 1 +local evenLongerVariableName = 2 + +/// this property has a reaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaly long doc comment +property = if (reallyLongVariableName) anotherReallyLongName else evenLongerVariableName + anotherReallyLongName + +longString = "reaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaly long string" + +shortProperty = 1 + +reaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaalyLongVariableName = 10 + +reaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaalyLongVariableName2 = + if (reallyLongVariableName) anotherReallyLongName else evenLongerVariableName + anotherReallyLongName diff --git a/pkl-formatter/src/test/files/FormatterSnippetTests/input/map-function.pkl b/pkl-formatter/src/test/files/FormatterSnippetTests/input/map-function.pkl new file mode 100644 index 00000000..07f987a9 --- /dev/null +++ b/pkl-formatter/src/test/files/FormatterSnippetTests/input/map-function.pkl @@ -0,0 +1,4 @@ + +foo = Map(1000, "some random string", 20000, "another random string", 30000, "yet another random string") + +incorrect = Map("This has", 1000000, "an incorrect number", 2000000, "of parameters", 30000000, "passed to Map") diff --git a/pkl-formatter/src/test/files/FormatterSnippetTests/input/modifiers.pkl b/pkl-formatter/src/test/files/FormatterSnippetTests/input/modifiers.pkl new file mode 100644 index 00000000..86806a17 --- /dev/null +++ b/pkl-formatter/src/test/files/FormatterSnippetTests/input/modifiers.pkl @@ -0,0 +1,3 @@ +hidden const foo = 1 + +open local class Foo {} diff --git a/pkl-formatter/src/test/files/FormatterSnippetTests/input/module-definitions.pkl b/pkl-formatter/src/test/files/FormatterSnippetTests/input/module-definitions.pkl new file mode 100644 index 00000000..8fe82fda --- /dev/null +++ b/pkl-formatter/src/test/files/FormatterSnippetTests/input/module-definitions.pkl @@ -0,0 +1,13 @@ +// comment +// one + + +/// doc comment + +open +module +foo + .bar + +amends +"baz.pkl" diff --git a/pkl-formatter/src/test/files/FormatterSnippetTests/input/multi-line-strings.pkl b/pkl-formatter/src/test/files/FormatterSnippetTests/input/multi-line-strings.pkl new file mode 100644 index 00000000..53bdabc7 --- /dev/null +++ b/pkl-formatter/src/test/files/FormatterSnippetTests/input/multi-line-strings.pkl @@ -0,0 +1,29 @@ +foo = """ + asd \(new { + bar = 1 +}) asd + """ + +bar = """ + line 1 + line 2 + line3 + """ + +baz = """ +\n +\(bar) +line +\u{123} +""" + +// remove unneeded spaces +qux = """ + foo + + bar + + baz + + \(foo) + """ diff --git a/pkl-formatter/src/test/files/FormatterSnippetTests/input/object-members.pkl b/pkl-formatter/src/test/files/FormatterSnippetTests/input/object-members.pkl new file mode 100644 index 00000000..d0dc1b4d --- /dev/null +++ b/pkl-formatter/src/test/files/FormatterSnippetTests/input/object-members.pkl @@ -0,0 +1,22 @@ +foo: Listing = new {1 2 3; 4;5 6 7} + +bar: Listing = new { 1 2 + 3 + 4 +} + +lineIsTooBig: Listing = new { 999999; 1000000; 1000001; 1000002; 1000003; 1000004; 1000005; 1000006; 1000007; 1000008; 1000009 } + +lineIsTooBig2 { 999999; 1000000; 1000001; 1000002; 1000003; 1000004; 1000005; 1000006; 1000007; 1000008; 1000009; 1000010 } + +baz = new Dynamic { + 1 2 + 3 4 + + ["foo"] = 3; bar = 30 + + + baz = true +} + +qux = new Dynamic {1 2 3 prop="prop"; [0]=9} diff --git a/pkl-formatter/src/test/files/FormatterSnippetTests/input/prefixes.pkl b/pkl-formatter/src/test/files/FormatterSnippetTests/input/prefixes.pkl new file mode 100644 index 00000000..4ee29819 --- /dev/null +++ b/pkl-formatter/src/test/files/FormatterSnippetTests/input/prefixes.pkl @@ -0,0 +1,12 @@ + +/// Looooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooong doc comment +@Annotation { looooooooooooooooooooooooooooooooooooooooooooooooooongVariableName = 10 } +typealias Foo = String + +/// Looooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooong doc comment +@Annotation { looooooooooooooooooooooooooooooooooooooooooooooooooongVariableName = 10 } +foo = "should fit in a single line" + +/// Looooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooong doc comment +@Annotation { looooooooooooooooooooooooooooooooooooooooooooooooooongVariableName = 10 } +function bar(x): String = "result: \(x)" diff --git a/pkl-formatter/src/test/files/FormatterSnippetTests/input/spaces.pkl b/pkl-formatter/src/test/files/FormatterSnippetTests/input/spaces.pkl new file mode 100644 index 00000000..9e390e56 --- /dev/null +++ b/pkl-formatter/src/test/files/FormatterSnippetTests/input/spaces.pkl @@ -0,0 +1,23 @@ +import"foo.pkl" + +bar=new Listing < Int > ( !isEmpty ){1;2}//a bar + +typealias Typealias=(String,Int)->Boolean + +///a baz +///returns its parameter +baz = (x,y,z)->x+y+z + +function fun ( x:Int ? ,b :Boolean )= if(b)/***return x**/x else x+bar [0 ] + +prop = trace (1) + super . foo + module .foo + +prop2 { + for(x in List(1)) { + when(x == 1){ + x + } + } +} + +choices: "foo" | * "bar" | String diff --git a/pkl-formatter/src/test/files/FormatterSnippetTests/input/type-aliases.pkl b/pkl-formatter/src/test/files/FormatterSnippetTests/input/type-aliases.pkl new file mode 100644 index 00000000..cd6629be --- /dev/null +++ b/pkl-formatter/src/test/files/FormatterSnippetTests/input/type-aliases.pkl @@ -0,0 +1,6 @@ +typealias + Foo + = + String + +typealias VeryLoooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooongTypeAlias = String(!isEmpty) diff --git a/pkl-formatter/src/test/files/FormatterSnippetTests/input/when.pkl b/pkl-formatter/src/test/files/FormatterSnippetTests/input/when.pkl new file mode 100644 index 00000000..910fc445 --- /dev/null +++ b/pkl-formatter/src/test/files/FormatterSnippetTests/input/when.pkl @@ -0,0 +1,9 @@ +foo { + when (true) + { + bar = 1 + } + else { + bar = 2 + } +} diff --git a/pkl-formatter/src/test/files/FormatterSnippetTests/output/class-bodies.pkl b/pkl-formatter/src/test/files/FormatterSnippetTests/output/class-bodies.pkl new file mode 100644 index 00000000..3b3f06d4 --- /dev/null +++ b/pkl-formatter/src/test/files/FormatterSnippetTests/output/class-bodies.pkl @@ -0,0 +1,11 @@ +class Foo {} + +class Bar { + qux = 1 +} + +class Baz { + prop = 0 + + prop2 = 1 +} diff --git a/pkl-formatter/src/test/files/FormatterSnippetTests/output/comma-termination.pkl b/pkl-formatter/src/test/files/FormatterSnippetTests/output/comma-termination.pkl new file mode 100644 index 00000000..ef383e42 --- /dev/null +++ b/pkl-formatter/src/test/files/FormatterSnippetTests/output/comma-termination.pkl @@ -0,0 +1,35 @@ +function fun(a, b, c, d) = a + +noTrailingCommas = fun(1, 2, 3, 4) + +trailingCommas = + fun( + "loooooooooooooooooooongString", + "loooooooooooooooongString", + "looooooooooooooooongString", + "notTooLong", + ) + +noTrailingCommaObjParams { + fooooooooooooooooooooooooooooooooo, + baaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaar, + baaaaaaaaaaaaaaaaaaaaaz -> + 1 + 1 +} + +trailingCommaInLambdas = ( + fooooooooooooooooooooooooooooooooo, + baaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaar, + baaaaaaaaaaaaaaaaaaaaaz, +) -> 1 + +trailingCommaInConstraints: String( + isSomethingSomethingSomething, + isSomethingElse, + isSomethingSomethingSomethingElse, +) + +trailingCommaInTypeParameters: Mapping< + SomethingSomethingSomethingSomething, + SomethingSomething | SomethingElse, +> diff --git a/pkl-formatter/src/test/files/FormatterSnippetTests/output/comment-interleaved.pkl b/pkl-formatter/src/test/files/FormatterSnippetTests/output/comment-interleaved.pkl new file mode 100644 index 00000000..876859ff --- /dev/null +++ b/pkl-formatter/src/test/files/FormatterSnippetTests/output/comment-interleaved.pkl @@ -0,0 +1,34 @@ +module comment.interleaved + +local test: Int | String = + if (true) + 1 // It's the same as "100%" + else + "8%" + +foo: ( // some comment + *String( + // if dtstart is defined and a datetime, until must also be a datetime if defined + true, + ) + | Int + // trailing comment +)? + +foo2: ( // some comment + Int, // other comment + String, +) -> + Int + +bar = + ( // some comment + 10 + 10 + // another comment + ) + +bar2 = ( // some comment + foo, + bar + // another comment +) -> foo + bar diff --git a/pkl-formatter/src/test/files/FormatterSnippetTests/output/dangling-doc-comment.pkl b/pkl-formatter/src/test/files/FormatterSnippetTests/output/dangling-doc-comment.pkl new file mode 100644 index 00000000..0ac2d494 --- /dev/null +++ b/pkl-formatter/src/test/files/FormatterSnippetTests/output/dangling-doc-comment.pkl @@ -0,0 +1,8 @@ +module dangling + +/// Doc comment. +/// +/// for this field +// sepearted by a stray comment +/// but continues here. +some: String diff --git a/pkl-formatter/src/test/files/FormatterSnippetTests/output/doc-comments.pkl b/pkl-formatter/src/test/files/FormatterSnippetTests/output/doc-comments.pkl new file mode 100644 index 00000000..49a7bea3 --- /dev/null +++ b/pkl-formatter/src/test/files/FormatterSnippetTests/output/doc-comments.pkl @@ -0,0 +1,13 @@ +/// / line 1 +/// // line 2 +foo = 1 + +/// line 1 +/// +/// +/// line 2 +bar = 1 + +/// line 1 +/// line 2 +baz = 1 diff --git a/pkl-formatter/src/test/files/FormatterSnippetTests/output/expr-binary.pkl b/pkl-formatter/src/test/files/FormatterSnippetTests/output/expr-binary.pkl new file mode 100644 index 00000000..d025d63c --- /dev/null +++ b/pkl-formatter/src/test/files/FormatterSnippetTests/output/expr-binary.pkl @@ -0,0 +1,34 @@ +foo { + 123123123123 + + 123123123123 * 123123123123 - + 123123123123 / 123123123123 + + 123123123123 ** 123123123123 + 2 + 3 + 4 +} + +bar = + clazz.superclass == null + || clazz.superclass.reflectee == Module + || clazz.superclass.reflectee == Typed + +baz = + new Listing { + 1 + 2 + } + |> mixin1 + |> mixin2 + +qux = (baz) { + 1 + 2 +} { + 3 + 4 +} + +minus = + superLoooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooongVariable - + 100000 diff --git a/pkl-formatter/src/test/files/FormatterSnippetTests/output/expr-chain.pkl b/pkl-formatter/src/test/files/FormatterSnippetTests/output/expr-chain.pkl new file mode 100644 index 00000000..ef7be664 --- /dev/null +++ b/pkl-formatter/src/test/files/FormatterSnippetTests/output/expr-chain.pkl @@ -0,0 +1,8 @@ +res = + myList + .map((it) -> it.partition) + .filter((it) -> someList.contains(it)) + +res2 = myList.map(lambda1).filter(lambda2) + +res3 = myList.map((it) -> it.partition) diff --git a/pkl-formatter/src/test/files/FormatterSnippetTests/output/expr-if.pkl b/pkl-formatter/src/test/files/FormatterSnippetTests/output/expr-if.pkl new file mode 100644 index 00000000..18bbbf10 --- /dev/null +++ b/pkl-formatter/src/test/files/FormatterSnippetTests/output/expr-if.pkl @@ -0,0 +1,9 @@ +foo = + if (someCondition) + 10000 + else if (someOtherCondition) + 20000 + else if (someAnotherCondition) + 30000 + else + 4 diff --git a/pkl-formatter/src/test/files/FormatterSnippetTests/output/expr-let.pkl b/pkl-formatter/src/test/files/FormatterSnippetTests/output/expr-let.pkl new file mode 100644 index 00000000..838154a4 --- /dev/null +++ b/pkl-formatter/src/test/files/FormatterSnippetTests/output/expr-let.pkl @@ -0,0 +1,18 @@ +foo = + let (vaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaariable = + 10 + ) + 1 * 1 + +bar = + let (someVariable = new Listing { + 1 + } + ) + 1 * 1 + +baz = + let (someVariable = 10000000) + let (someOtherVariable = 2000000) + let (someAnotherVariable = 3000000) + someVariable + someOtherVariable + someAnotherVariable diff --git a/pkl-formatter/src/test/files/FormatterSnippetTests/output/imports.pkl b/pkl-formatter/src/test/files/FormatterSnippetTests/output/imports.pkl new file mode 100644 index 00000000..f75cda49 --- /dev/null +++ b/pkl-formatter/src/test/files/FormatterSnippetTests/output/imports.pkl @@ -0,0 +1,21 @@ +// top level comment + +import "https://example.com/baz.pkl" +import "package://example.com/myPackage@1.0.0#/Qux.pkl" +import "pkl:math" +import "pkl:reflect" + +import "@bar/Bar.pkl" +import "@foo/Foo.pkl" as foo + +import "..." +import "module1.pkl" +import "module2.pkl" +import "module11.pkl" +import "Module12.pkl" + +import* "file:///tmp/*.pkl" + +import* "@foo/**.pkl" + +import* "**.pkl" diff --git a/pkl-formatter/src/test/files/FormatterSnippetTests/output/indentation.pkl b/pkl-formatter/src/test/files/FormatterSnippetTests/output/indentation.pkl new file mode 100644 index 00000000..a7567b88 --- /dev/null +++ b/pkl-formatter/src/test/files/FormatterSnippetTests/output/indentation.pkl @@ -0,0 +1,61 @@ +foo = new { + bar = 1 +} + +class Foo { + baz: Int +} + +bar = new Listing { + 1 + 2 +} + +baz = (foo) { + 2 + 3 +} + +qux = (x, y) -> new Listing { + x + y +} + +forGen = new Listing { + for ( + someVar in new Listing { + 1 + 2 + } + ) { + [someVar] = someVar + } +} + +objParams: Listing = new { + default { key -> + key + 1 + } +} + +objParams2: Listing = new { + default { + someVeryLongParameter1, + someVeryLongParameter2, + someVeryLongParameter3, + someVeryLongParameter4 -> + 1 + } +} + +parenType: ( + ReaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaalyLongType( + reaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaalylongConstraint, + ) +) + +functionType: ( + ReaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaalyLongType, + ReaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaalyLongType2, +) -> + String diff --git a/pkl-formatter/src/test/files/FormatterSnippetTests/output/line-breaks.pkl b/pkl-formatter/src/test/files/FormatterSnippetTests/output/line-breaks.pkl new file mode 100644 index 00000000..dac13632 --- /dev/null +++ b/pkl-formatter/src/test/files/FormatterSnippetTests/output/line-breaks.pkl @@ -0,0 +1,75 @@ +module foo.bar.baz + +amends "bar.pkl" + +import "@foo/Foo.pkl" as foo + +open local class Bar {} + +local const baz = 10 + +local function fun(x) = x + +local const prooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooperty: String = + "foo" + +local function function2( + parameter1: Parameter1Type, + parameter2: Parameter2Type, + parameter3: Parameter3Type, + parameter4: Parameter4Type, +): String = "" + +local const function function3( + parameter1: String | Int, + parameter2: String | Int, +): Mapping = new {} + +prop = + function2( + loooooooooooooooooogParameter1, + loooooooooooooooooogParameter2, + loooooooooooooooooogParameter3, + loooooooooooooooooogParameter4, + ) + +prop2: String | Int | Boolean + +funcParam = + fun((x, y) -> new Listing { + x + y + }) + +funcParam2 = + aFun(foo, 10 * 10, anotherVariable, if (true) 100000 else 200000, (param1, param2) -> + param1 * param2 + ) + +funcParam3 = aFun(foo, 10 * 10, anotherVariable, 200000, (param1, param2) -> param1 * param2) + +funcParam4 = + aFun(foo, 10 * 10, anotherVariable, if (true) 100000 else 200000, new Listing { + 1 + 2 + }) + +open local class SomeReallyInterestingClassName + extends AnotherInterestingClassName { + foo: Int +} + +local function resourceMapping(type): Mapping = new Mapping { + default = (key) -> (type) { metadata { name = key } } +} + +local const function biiiiiiiiiiiiiiiiiiiiiiiiigFunction( + param1: String, + param2: String(!isBlank), +): Boolean + +local const function someFunction(param1: String, param2: String(!isBlank)): Boolean + +local function render(currentIndent: String) = + "\(currentIndent)@\(identifier.render(currentIndent))" + + if (body == null) "" else " " + body.render(currentIndent) diff --git a/pkl-formatter/src/test/files/FormatterSnippetTests/output/line-breaks2.pkl b/pkl-formatter/src/test/files/FormatterSnippetTests/output/line-breaks2.pkl new file mode 100644 index 00000000..e1fbb341 --- /dev/null +++ b/pkl-formatter/src/test/files/FormatterSnippetTests/output/line-breaks2.pkl @@ -0,0 +1,42 @@ +module + reaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaly + .loooooooooooooooooooooooooooooooog + .naaaaaaaaaaaaaaaaaaaaaaaaaame + +extends + "reaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaalyLongModule.pkl" + +import + "reaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaalyLongModule.pkl" + as foo + +open local class LoooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooongName {} + +open local class ReaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaalyLongName {} + +hidden const loooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooogName = + 99 + +hidden const reallyLoooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooogName = + 99 + +local const function looooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooogName( + x: Int, + y, +) = x + y + +typealias Foo = + LooooooooooooooooooooooooongTypeName + | AnotherLooooooooooooooooooooooooongTypeName + | OtherLooooooooooooooooooooooooongTypeName + +bar: Boolean + | Mapping< + LooooooooooooooooooooooooooooongTypeName, + AnotherLooooooooooooooooooooooooooooongTypeName, + >(loooooooooooooooooooogConstraint) + +hidden foobar: LongType( + someVeryyyyyyyloooong, + requirements.might.be.even_loooooooooooooooooooooooooooooooooooooooooooooooooooooooooonger, +) diff --git a/pkl-formatter/src/test/files/FormatterSnippetTests/output/line-width.pkl b/pkl-formatter/src/test/files/FormatterSnippetTests/output/line-width.pkl new file mode 100644 index 00000000..69ab9844 --- /dev/null +++ b/pkl-formatter/src/test/files/FormatterSnippetTests/output/line-width.pkl @@ -0,0 +1,32 @@ +module + reaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaly + .loooooooooooooooooooooooooooooooog + .naaaaaaaaaaaaaaaaaaaaaaaaaaaaame + +extends + "reaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaalyLongModule.pkl" + +local reallyLongVariableName = true +local anotherReallyLongName = 1 +local evenLongerVariableName = 2 + +/// this property has a reaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaly long doc comment +property = + if (reallyLongVariableName) + anotherReallyLongName + else + evenLongerVariableName + anotherReallyLongName + +longString = + "reaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaly long string" + +shortProperty = 1 + +reaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaalyLongVariableName = + 10 + +reaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaalyLongVariableName2 = + if (reallyLongVariableName) + anotherReallyLongName + else + evenLongerVariableName + anotherReallyLongName diff --git a/pkl-formatter/src/test/files/FormatterSnippetTests/output/map-function.pkl b/pkl-formatter/src/test/files/FormatterSnippetTests/output/map-function.pkl new file mode 100644 index 00000000..1d4f6e74 --- /dev/null +++ b/pkl-formatter/src/test/files/FormatterSnippetTests/output/map-function.pkl @@ -0,0 +1,14 @@ +foo = + Map( + 1000, "some random string", + 20000, "another random string", + 30000, "yet another random string", + ) + +incorrect = + Map( + "This has", 1000000, + "an incorrect number", 2000000, + "of parameters", 30000000, + "passed to Map", + ) diff --git a/pkl-formatter/src/test/files/FormatterSnippetTests/output/modifiers.pkl b/pkl-formatter/src/test/files/FormatterSnippetTests/output/modifiers.pkl new file mode 100644 index 00000000..86806a17 --- /dev/null +++ b/pkl-formatter/src/test/files/FormatterSnippetTests/output/modifiers.pkl @@ -0,0 +1,3 @@ +hidden const foo = 1 + +open local class Foo {} diff --git a/pkl-formatter/src/test/files/FormatterSnippetTests/output/module-definitions.pkl b/pkl-formatter/src/test/files/FormatterSnippetTests/output/module-definitions.pkl new file mode 100644 index 00000000..9cea366f --- /dev/null +++ b/pkl-formatter/src/test/files/FormatterSnippetTests/output/module-definitions.pkl @@ -0,0 +1,7 @@ +// comment +// one + +/// doc comment +open module foo.bar + +amends "baz.pkl" diff --git a/pkl-formatter/src/test/files/FormatterSnippetTests/output/multi-line-strings.pkl b/pkl-formatter/src/test/files/FormatterSnippetTests/output/multi-line-strings.pkl new file mode 100644 index 00000000..b4bc3b3c --- /dev/null +++ b/pkl-formatter/src/test/files/FormatterSnippetTests/output/multi-line-strings.pkl @@ -0,0 +1,33 @@ +foo = + """ + asd \(new { + bar = 1 + }) asd + """ + +bar = + """ + line 1 + line 2 + line3 + """ + +baz = + """ + \n + \(bar) + line + \u{123} + """ + +// remove unneeded spaces +qux = + """ + foo + + bar + + baz + + \(foo) + """ diff --git a/pkl-formatter/src/test/files/FormatterSnippetTests/output/object-members.pkl b/pkl-formatter/src/test/files/FormatterSnippetTests/output/object-members.pkl new file mode 100644 index 00000000..27dd0d26 --- /dev/null +++ b/pkl-formatter/src/test/files/FormatterSnippetTests/output/object-members.pkl @@ -0,0 +1,51 @@ +foo: Listing = new { 1; 2; 3; 4; 5; 6; 7 } + +bar: Listing = new { + 1 + 2 + 3 + 4 +} + +lineIsTooBig: Listing = new { + 999999 + 1000000 + 1000001 + 1000002 + 1000003 + 1000004 + 1000005 + 1000006 + 1000007 + 1000008 + 1000009 +} + +lineIsTooBig2 { + 999999 + 1000000 + 1000001 + 1000002 + 1000003 + 1000004 + 1000005 + 1000006 + 1000007 + 1000008 + 1000009 + 1000010 +} + +baz = new Dynamic { + 1 + 2 + 3 + 4 + + ["foo"] = 3 + bar = 30 + + baz = true +} + +qux = new Dynamic { 1; 2; 3; prop = "prop"; [0] = 9 } diff --git a/pkl-formatter/src/test/files/FormatterSnippetTests/output/prefixes.pkl b/pkl-formatter/src/test/files/FormatterSnippetTests/output/prefixes.pkl new file mode 100644 index 00000000..0c2ae084 --- /dev/null +++ b/pkl-formatter/src/test/files/FormatterSnippetTests/output/prefixes.pkl @@ -0,0 +1,11 @@ +/// Looooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooong doc comment +@Annotation { looooooooooooooooooooooooooooooooooooooooooooooooooongVariableName = 10 } +typealias Foo = String + +/// Looooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooong doc comment +@Annotation { looooooooooooooooooooooooooooooooooooooooooooooooooongVariableName = 10 } +foo = "should fit in a single line" + +/// Looooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooong doc comment +@Annotation { looooooooooooooooooooooooooooooooooooooooooooooooooongVariableName = 10 } +function bar(x): String = "result: \(x)" diff --git a/pkl-formatter/src/test/files/FormatterSnippetTests/output/spaces.pkl b/pkl-formatter/src/test/files/FormatterSnippetTests/output/spaces.pkl new file mode 100644 index 00000000..da6b06f9 --- /dev/null +++ b/pkl-formatter/src/test/files/FormatterSnippetTests/output/spaces.pkl @@ -0,0 +1,23 @@ +import "foo.pkl" + +bar = new Listing(!isEmpty) { 1; 2 } //a bar + +typealias Typealias = (String, Int) -> Boolean + +/// a baz +/// returns its parameter +baz = (x, y, z) -> x + y + z + +function fun(x: Int?, b: Boolean) = if (b) /***return x**/ x else x + bar[0] + +prop = trace(1) + super.foo + module.foo + +prop2 { + for (x in List(1)) { + when (x == 1) { + x + } + } +} + +choices: "foo" | *"bar" | String diff --git a/pkl-formatter/src/test/files/FormatterSnippetTests/output/type-aliases.pkl b/pkl-formatter/src/test/files/FormatterSnippetTests/output/type-aliases.pkl new file mode 100644 index 00000000..1206e611 --- /dev/null +++ b/pkl-formatter/src/test/files/FormatterSnippetTests/output/type-aliases.pkl @@ -0,0 +1,4 @@ +typealias Foo = String + +typealias VeryLoooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooongTypeAlias = + String(!isEmpty) diff --git a/pkl-formatter/src/test/files/FormatterSnippetTests/output/when.pkl b/pkl-formatter/src/test/files/FormatterSnippetTests/output/when.pkl new file mode 100644 index 00000000..2f0f6fd3 --- /dev/null +++ b/pkl-formatter/src/test/files/FormatterSnippetTests/output/when.pkl @@ -0,0 +1,7 @@ +foo { + when (true) { + bar = 1 + } else { + bar = 2 + } +} diff --git a/pkl-formatter/src/test/kotlin/org/pkl/formatter/FormatterSnippetTests.kt b/pkl-formatter/src/test/kotlin/org/pkl/formatter/FormatterSnippetTests.kt new file mode 100644 index 00000000..f00220c5 --- /dev/null +++ b/pkl-formatter/src/test/kotlin/org/pkl/formatter/FormatterSnippetTests.kt @@ -0,0 +1,20 @@ +/* + * Copyright © 2024-2025 Apple Inc. and the Pkl project authors. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.pkl.formatter + +import org.junit.platform.commons.annotation.Testable + +@Testable class FormatterSnippetTests diff --git a/pkl-formatter/src/test/kotlin/org/pkl/formatter/FormatterSnippetTestsEngine.kt b/pkl-formatter/src/test/kotlin/org/pkl/formatter/FormatterSnippetTestsEngine.kt new file mode 100644 index 00000000..d245a0ac --- /dev/null +++ b/pkl-formatter/src/test/kotlin/org/pkl/formatter/FormatterSnippetTestsEngine.kt @@ -0,0 +1,73 @@ +/* + * Copyright © 2025 Apple Inc. and the Pkl project authors. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.pkl.formatter + +import java.nio.file.Path +import kotlin.io.path.isRegularFile +import kotlin.reflect.KClass +import org.pkl.commons.test.InputOutputTestEngine +import org.pkl.parser.ParserError + +abstract class AbstractFormatterSnippetTestsEngine : InputOutputTestEngine() { + + private val snippetsDir: Path = + rootProjectDir.resolve("pkl-formatter/src/test/files/FormatterSnippetTests") + + private val expectedOutputDir: Path = snippetsDir.resolve("output") + + /** Convenience for development; this selects which snippet test(s) to run. */ + // language=regexp + internal val selection: String = "" + + override val includedTests: List = listOf(Regex(".*$selection\\.pkl")) + + override val inputDir: Path = snippetsDir.resolve("input") + + override val isInputFile: (Path) -> Boolean = { it.isRegularFile() } + + override fun expectedOutputFileFor(inputFile: Path): Path { + val relativePath = relativize(inputFile, inputDir).toString() + return expectedOutputDir.resolve(relativePath) + } + + companion object { + private fun relativize(path: Path, base: Path): Path { + if (System.getProperty("os.name").contains("Windows")) { + if (path.isAbsolute && base.isAbsolute && (path.root != base.root)) { + return path + } + } + return base.relativize(path) + } + } +} + +class FormatterSnippetTestsEngine : AbstractFormatterSnippetTestsEngine() { + override val testClass: KClass<*> = FormatterSnippetTests::class + + override fun generateOutputFor(inputFile: Path): Pair { + val formatter = Formatter() + val (success, output) = + try { + val res = formatter.format(inputFile) + true to res + } catch (_: ParserError) { + false to "" + } + + return success to output + } +} diff --git a/pkl-formatter/src/test/kotlin/org/pkl/formatter/FormatterTest.kt b/pkl-formatter/src/test/kotlin/org/pkl/formatter/FormatterTest.kt new file mode 100644 index 00000000..19845589 --- /dev/null +++ b/pkl-formatter/src/test/kotlin/org/pkl/formatter/FormatterTest.kt @@ -0,0 +1,79 @@ +/* + * Copyright © 2025 Apple Inc. and the Pkl project authors. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.pkl.formatter + +import org.assertj.core.api.Assertions.assertThat +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.assertThrows +import org.pkl.parser.GenericParserError + +class FormatterTest { + + @Test + fun `multiline string - wrong start quote`() { + val ex = + assertThrows { + format( + """ + foo = ""${'"'}line1 + line 2 + ""${'"'} + """ + ) + } + + assertThat(ex.message).contains("The content of a multi-line string must begin on a new line") + } + + @Test + fun `multiline string - wrong end quote`() { + val ex = + assertThrows { + format( + """ + foo = ""${'"'} + line1 + line 2""${'"'} + """ + ) + } + + assertThat(ex.message) + .contains("The closing delimiter of a multi-line string must begin on a new line") + } + + @Test + fun `multiline string - wrong indentation`() { + val ex = + assertThrows { + format( + """ + foo = ""${'"'} + line1 + line 2 + ""${'"'} + """ + ) + } + + assertThat(ex.message) + .contains("Line must match or exceed indentation of the String's last line.") + } + + private fun format(code: String): String { + return Formatter().format(code.trimIndent()) + } +} diff --git a/pkl-formatter/src/test/resources/META-INF/services/org.junit.platform.engine.TestEngine b/pkl-formatter/src/test/resources/META-INF/services/org.junit.platform.engine.TestEngine new file mode 100644 index 00000000..1c9c99bf --- /dev/null +++ b/pkl-formatter/src/test/resources/META-INF/services/org.junit.platform.engine.TestEngine @@ -0,0 +1 @@ +org.pkl.formatter.FormatterSnippetTestsEngine diff --git a/pkl-parser/src/main/java/org/pkl/parser/GenericParser.java b/pkl-parser/src/main/java/org/pkl/parser/GenericParser.java new file mode 100644 index 00000000..161024c4 --- /dev/null +++ b/pkl-parser/src/main/java/org/pkl/parser/GenericParser.java @@ -0,0 +1,1574 @@ +/* + * Copyright © 2025 Apple Inc. and the Pkl project authors. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.pkl.parser; + +import java.util.ArrayList; +import java.util.List; +import java.util.function.Supplier; +import org.pkl.parser.syntax.Operator; +import org.pkl.parser.syntax.generic.FullSpan; +import org.pkl.parser.syntax.generic.Node; +import org.pkl.parser.syntax.generic.NodeType; +import org.pkl.parser.util.ErrorMessages; +import org.pkl.parser.util.Nullable; + +@SuppressWarnings("DuplicatedCode") +public class GenericParser { + + private Lexer lexer; + private Token lookahead; + private FullSpan spanLookahead; + private FullToken _lookahead; + private int cursor = 0; + private final List tokens = new ArrayList<>(); + + private void init(String source) { + this.lexer = new Lexer(source); + cursor = 0; + while (true) { + var ft = new FullToken(lexer.next(), lexer.fullSpan(), lexer.newLinesBetween); + tokens.add(ft); + if (ft.token == Token.EOF) break; + } + _lookahead = tokens.get(cursor); + lookahead = _lookahead.token; + spanLookahead = _lookahead.span; + } + + public Node parseModule(String source) { + init(source); + if (lookahead == Token.EOF) { + return new Node(NodeType.MODULE, new FullSpan(0, 0, 1, 1, 1, 1), List.of()); + } + var children = new ArrayList(); + var nodes = new ArrayList(); + if (lookahead == Token.SHEBANG) { + nodes.add(makeAffix(next())); + } + ff(nodes); + + var res = parseMemberHeader(children); + + if (isModuleDecl()) { + nodes.add(parseModuleDecl(children)); + children.clear(); + res = new HeaderResult(false, false, false); + ff(nodes); + } + + // imports + var imports = new ArrayList(); + while (lookahead == Token.IMPORT || lookahead == Token.IMPORT_STAR) { + if (res.hasDocComment || res.hasAnnotations || res.hasModifiers) { + throw parserError("wrongHeaders", "Imports"); + } + var lastImport = parseImportDecl(); + imports.add(lastImport); + // keep trailling affixes as part of the import + while (lookahead.isAffix() && lastImport.span.sameLine(spanLookahead)) { + imports.add(makeAffix(next())); + } + if (!isImport()) break; + ff(imports); + } + if (!imports.isEmpty()) { + nodes.add(new Node(NodeType.IMPORT_LIST, imports)); + ff(nodes); + } + + // entries + if (res.hasDocComment || res.hasAnnotations || res.hasModifiers) { + nodes.add(parseModuleMember(children)); + ff(nodes); + } + + while (lookahead != Token.EOF) { + children.clear(); + parseMemberHeader(children); + nodes.add(parseModuleMember(children)); + ff(nodes); + } + return new Node(NodeType.MODULE, nodes); + } + + private Node parseModuleDecl(List preChildren) { + var headerParts = getHeaderParts(preChildren); + var children = new ArrayList<>(headerParts.preffixes); + var headers = new ArrayList(); + if (headerParts.modifierList != null) { + headers.add(headerParts.modifierList); + } + if (lookahead == Token.MODULE) { + var subChildren = new ArrayList<>(headers); + subChildren.add(makeTerminal(next())); + ff(subChildren); + subChildren.add(parseQualifiedIdentifier()); + children.add(new Node(NodeType.MODULE_DEFINITION, subChildren)); + } else { + children.addAll(headers); + if (headerParts.modifierList != null) { + throw parserError("wrongHeaders", "Amends or extends declaration"); + } + } + var looka = lookahead(); + if (looka == Token.AMENDS || looka == Token.EXTENDS) { + var type = looka == Token.AMENDS ? NodeType.AMENDS_CLAUSE : NodeType.EXTENDS_CLAUSE; + ff(children); + var subChildren = new ArrayList(); + subChildren.add(makeTerminal(next())); + ff(subChildren); + subChildren.add(parseStringConstant()); + children.add(new Node(type, subChildren)); + } + return new Node(NodeType.MODULE_DECLARATION, children); + } + + private Node parseQualifiedIdentifier() { + var children = new ArrayList(); + children.add(parseIdentifier()); + while (lookahead() == Token.DOT) { + ff(children); + children.add(new Node(NodeType.TERMINAL, next().span)); + ff(children); + children.add(parseIdentifier()); + } + return new Node(NodeType.QUALIFIED_IDENTIFIER, children); + } + + private Node parseImportDecl() { + var children = new ArrayList(); + children.add(makeTerminal(next())); + ff(children); + children.add(parseStringConstant()); + if (lookahead() == Token.AS) { + ff(children); + var alias = new ArrayList(); + alias.add(makeTerminal(next())); + ff(alias); + alias.add(parseIdentifier()); + children.add(new Node(NodeType.IMPORT_ALIAS, alias)); + } + return new Node(NodeType.IMPORT, children); + } + + private HeaderResult parseMemberHeader(List children) { + var hasDocComment = false; + var hasAnnotation = false; + var hasModifier = false; + var docs = new ArrayList(); + ff(children); + while (lookahead() == Token.DOC_COMMENT) { + ff(docs); + docs.add(new Node(NodeType.DOC_COMMENT_LINE, next().span)); + hasDocComment = true; + } + if (hasDocComment) { + children.add(new Node(NodeType.DOC_COMMENT, docs)); + } + ff(children); + while (lookahead == Token.AT) { + children.add(parseAnnotation()); + hasAnnotation = true; + ff(children); + } + var modifiers = new ArrayList(); + while (lookahead.isModifier()) { + modifiers.add(make(NodeType.MODIFIER, next().span)); + hasModifier = true; + ff(children); + } + if (hasModifier) children.add(new Node(NodeType.MODIFIER_LIST, modifiers)); + return new HeaderResult(hasDocComment, hasAnnotation, hasModifier); + } + + private Node parseModuleMember(List preChildren) { + return switch (lookahead) { + case IDENTIFIER -> parseClassProperty(preChildren); + case TYPE_ALIAS -> parseTypeAlias(preChildren); + case CLASS -> parseClass(preChildren); + case FUNCTION -> parseClassMethod(preChildren); + case EOF -> throw parserError("unexpectedEndOfFile"); + default -> { + if (lookahead.isKeyword()) { + throw parserError("keywordNotAllowedHere", lookahead.text()); + } + if (lookahead == Token.DOC_COMMENT) { + throw parserError("danglingDocComment"); + } + throw parserError("invalidTopLevelToken"); + } + }; + } + + private Node parseTypeAlias(List preChildren) { + var headerParts = getHeaderParts(preChildren); + var children = new ArrayList<>(headerParts.preffixes); + var headers = new ArrayList(); + if (headerParts.modifierList != null) { + headers.add(headerParts.modifierList); + } + // typealias keyword + headers.add(makeTerminal(next())); + ff(headers); + headers.add(parseIdentifier()); + ff(headers); + if (lookahead == Token.LT) { + headers.add(parseTypeParameterList()); + ff(headers); + } + expect(Token.ASSIGN, headers, "unexpectedToken", "="); + children.add(new Node(NodeType.TYPEALIAS_HEADER, headers)); + var body = new ArrayList(); + ff(body); + body.add(parseType()); + children.add(new Node(NodeType.TYPEALIAS_BODY, body)); + return new Node(NodeType.TYPEALIAS, children); + } + + private Node parseClass(List preChildren) { + var headerParts = getHeaderParts(preChildren); + var children = new ArrayList<>(headerParts.preffixes); + var headers = new ArrayList(); + if (headerParts.modifierList != null) { + headers.add(headerParts.modifierList); + } + // class keyword + headers.add(makeTerminal(next())); + ff(headers); + headers.add(parseIdentifier()); + if (lookahead() == Token.LT) { + ff(headers); + headers.add(parseTypeParameterList()); + } + if (lookahead() == Token.EXTENDS) { + var extend = new ArrayList(); + ff(extend); + extend.add(makeTerminal(next())); + ff(extend); + extend.add(parseType()); + headers.add(new Node(NodeType.CLASS_HEADER_EXTENDS, extend)); + } + children.add(new Node(NodeType.CLASS_HEADER, headers)); + if (lookahead() == Token.LBRACE) { + ff(children); + children.add(parseClassBody()); + } + return new Node(NodeType.CLASS, children); + } + + private Node parseClassBody() { + var children = new ArrayList(); + children.add(makeTerminal(next())); + var elements = new ArrayList(); + var hasElements = false; + ff(elements); + while (lookahead != Token.RBRACE && lookahead != Token.EOF) { + hasElements = true; + var preChildren = new ArrayList(); + parseMemberHeader(preChildren); + if (lookahead == Token.FUNCTION) { + elements.add(parseClassMethod(preChildren)); + } else { + elements.add(parseClassProperty(preChildren)); + } + ff(elements); + } + if (lookahead == Token.EOF) { + throw parserError(ErrorMessages.create("missingDelimiter", "}"), prev().span.stopSpan()); + } + if (hasElements) { + children.add(new Node(NodeType.CLASS_BODY_ELEMENTS, elements)); + } else if (!elements.isEmpty()) { + // add affixes + children.addAll(elements); + } + expect(Token.RBRACE, children, "missingDelimiter", "}"); + return new Node(NodeType.CLASS_BODY, children); + } + + private Node parseClassProperty(List preChildren) { + var headerParts = getHeaderParts(preChildren); + var children = new ArrayList<>(headerParts.preffixes); + var header = new ArrayList(); + var headerBegin = new ArrayList(); + if (headerParts.modifierList != null) { + headerBegin.add(headerParts.modifierList); + } + headerBegin.add(parseIdentifier()); + header.add(new Node(NodeType.CLASS_PROPERTY_HEADER_BEGIN, headerBegin)); + var hasTypeAnnotation = false; + if (lookahead() == Token.COLON) { + ff(header); + header.add(parseTypeAnnotation()); + hasTypeAnnotation = true; + } + children.add(new Node(NodeType.CLASS_PROPERTY_HEADER, header)); + if (lookahead() == Token.ASSIGN) { + ff(children); + children.add(makeTerminal(next())); + var body = new ArrayList(); + ff(body); + body.add(parseExpr()); + children.add(new Node(NodeType.CLASS_PROPERTY_BODY, body)); + } else if (lookahead() == Token.LBRACE) { + if (hasTypeAnnotation) { + throw parserError("typeAnnotationInAmends"); + } + while (lookahead() == Token.LBRACE) { + ff(children); + children.add(parseObjectBody()); + } + } + return new Node(NodeType.CLASS_PROPERTY, children); + } + + private Node parseClassMethod(List preChildren) { + var headerParts = getHeaderParts(preChildren); + var children = new ArrayList<>(headerParts.preffixes); + var headers = new ArrayList(); + if (headerParts.modifierList != null) { + headers.add(headerParts.modifierList); + } + expect(Token.FUNCTION, headers, "unexpectedToken", "function"); + ff(headers); + headers.add(parseIdentifier()); + children.add(new Node(NodeType.CLASS_METHOD_HEADER, headers)); + ff(children); + if (lookahead == Token.LT) { + children.add(parseTypeParameterList()); + ff(children); + } + children.add(parseParameterList()); + if (lookahead() == Token.COLON) { + ff(children); + children.add(parseTypeAnnotation()); + } + if (lookahead() == Token.ASSIGN) { + ff(children); + children.add(makeTerminal(next())); + var body = new ArrayList(); + ff(body); + body.add(parseExpr()); + children.add(new Node(NodeType.CLASS_METHOD_BODY, body)); + } + return new Node(NodeType.CLASS_METHOD, children); + } + + private Node parseObjectBody() { + var children = new ArrayList(); + expect(Token.LBRACE, children, "unexpectedToken", "{"); + if (lookahead() == Token.RBRACE) { + ff(children); + children.add(makeTerminal(next())); + return new Node(NodeType.OBJECT_BODY, children); + } + if (isParameter()) { + var params = new ArrayList(); + ff(params); + parseListOf(Token.ARROW, params, this::parseParameter); + expect(Token.ARROW, params, "unexpectedToken2", ",", "->"); + children.add(new Node(NodeType.OBJECT_PARAMETER_LIST, params)); + ff(children); + } + var members = new ArrayList(); + ff(members); + while (lookahead != Token.RBRACE) { + if (lookahead == Token.EOF) { + throw parserError(ErrorMessages.create("missingDelimiter", "}"), prev().span.stopSpan()); + } + members.add(parseObjectMember()); + ff(members); + } + if (!members.isEmpty()) { + children.add(new Node(NodeType.OBJECT_MEMBER_LIST, members)); + } + children.add(makeTerminal(next())); // RBRACE + return new Node(NodeType.OBJECT_BODY, children); + } + + /** Returns true if the lookahead is a parameter, false if it's a member. May have to backtrack */ + private boolean isParameter() { + if (lookahead == Token.UNDERSCORE) return true; + if (lookahead != Token.IDENTIFIER) return false; + // have to backtrack + var originalCursor = cursor; + var result = false; + next(); // identifier + ff(); + if (lookahead == Token.ARROW || lookahead == Token.COMMA) { + result = true; + } else if (lookahead == Token.COLON) { + next(); // colon + ff(); + parseType(); + ff(); + result = lookahead == Token.COMMA || lookahead == Token.ARROW; + } + backtrackTo(originalCursor); + return result; + } + + private Node parseObjectMember() { + return switch (lookahead) { + case IDENTIFIER -> { + var originalCursor = cursor; + next(); + ff(new ArrayList<>()); + if (lookahead == Token.LBRACE || lookahead == Token.COLON || lookahead == Token.ASSIGN) { + // it's an objectProperty + backtrackTo(originalCursor); + yield parseObjectProperty(null); + } else { + backtrackTo(originalCursor); + // it's an expression + yield parseObjectElement(); + } + } + case FUNCTION -> parseObjectMethod(List.of()); + case LPRED -> parseMemberPredicate(); + case LBRACK -> parseObjectEntry(); + case SPREAD, QSPREAD -> parseObjectSpread(); + case WHEN -> parseWhenGenerator(); + case FOR -> parseForGenerator(); + case TYPE_ALIAS, CLASS -> + throw parserError(ErrorMessages.create("missingDelimiter", "}"), prev().span.stopSpan()); + default -> { + var preChildren = new ArrayList(); + while (lookahead.isModifier()) { + preChildren.add(make(NodeType.MODIFIER, next().span)); + ff(preChildren); + } + if (!preChildren.isEmpty()) { + if (lookahead == Token.FUNCTION) { + yield parseObjectMethod(List.of(new Node(NodeType.MODIFIER_LIST, preChildren))); + } else { + yield parseObjectProperty(List.of(new Node(NodeType.MODIFIER_LIST, preChildren))); + } + } else { + yield parseObjectElement(); + } + } + }; + } + + private Node parseObjectElement() { + return new Node(NodeType.OBJECT_ELEMENT, List.of(parseExpr())); + } + + private Node parseObjectProperty(@Nullable List preChildren) { + var children = new ArrayList(); + var header = new ArrayList(); + var headerBegin = new ArrayList(); + if (preChildren != null) { + headerBegin.addAll(preChildren); + } + ff(headerBegin); + var modifierList = new ArrayList(); + while (lookahead.isModifier()) { + modifierList.add(make(NodeType.MODIFIER, next().span)); + ff(modifierList); + } + if (!modifierList.isEmpty()) { + headerBegin.add(new Node(NodeType.MODIFIER_LIST, modifierList)); + } + headerBegin.add(parseIdentifier()); + header.add(new Node(NodeType.OBJECT_PROPERTY_HEADER_BEGIN, headerBegin)); + var hasTypeAnnotation = false; + if (lookahead() == Token.COLON) { + ff(header); + header.add(parseTypeAnnotation()); + hasTypeAnnotation = true; + } + children.add(new Node(NodeType.OBJECT_PROPERTY_HEADER, header)); + if (hasTypeAnnotation || lookahead() == Token.ASSIGN) { + ff(children); + expect(Token.ASSIGN, children, "unexpectedToken", "="); + var body = new ArrayList(); + ff(body); + body.add(parseExpr("}")); + children.add(new Node(NodeType.OBJECT_PROPERTY_BODY, body)); + return new Node(NodeType.OBJECT_PROPERTY, children); + } + ff(children); + children.addAll(parseBodyList()); + return new Node(NodeType.OBJECT_PROPERTY, children); + } + + private Node parseObjectMethod(List preChildren) { + var headerParts = getHeaderParts(preChildren); + var children = new ArrayList<>(headerParts.preffixes); + var headers = new ArrayList(); + if (headerParts.modifierList != null) { + headers.add(headerParts.modifierList); + } + expect(Token.FUNCTION, headers, "unexpectedToken", "function"); + ff(headers); + headers.add(parseIdentifier()); + children.add(new Node(NodeType.CLASS_METHOD_HEADER, headers)); + ff(children); + if (lookahead == Token.LT) { + children.add(parseTypeParameterList()); + ff(children); + } + children.add(parseParameterList()); + ff(children); + if (lookahead == Token.COLON) { + children.add(parseTypeAnnotation()); + ff(children); + } + expect(Token.ASSIGN, children, "unexpectedToken", "="); + var body = new ArrayList(); + ff(body); + body.add(parseExpr()); + children.add(new Node(NodeType.CLASS_METHOD_BODY, body)); + return new Node(NodeType.OBJECT_METHOD, children); + } + + private Node parseMemberPredicate() { + var children = new ArrayList(); + children.add(makeTerminal(next())); + ff(children); + children.add(parseExpr()); + ff(children); + var firstBrack = expect(Token.RBRACK, "unexpectedToken", "]]"); + children.add(makeTerminal(firstBrack)); + var secondbrack = expect(Token.RBRACK, "unexpectedToken", "]]"); + children.add(makeTerminal(secondbrack)); + if (firstBrack.span.charIndex() != secondbrack.span.charIndex() - 1) { + // There shouldn't be any whitespace between the first and second ']'. + var span = firstBrack.span.endWith(secondbrack.span); + var text = lexer.textFor(span.charIndex(), span.length()); + throw parserError(ErrorMessages.create("unexpectedToken", text, "]]"), firstBrack.span); + } + ff(children); + if (lookahead == Token.ASSIGN) { + children.add(makeTerminal(next())); + ff(children); + children.add(parseExpr("}")); + return new Node(NodeType.MEMBER_PREDICATE, children); + } + children.addAll(parseBodyList()); + return new Node(NodeType.MEMBER_PREDICATE, children); + } + + private Node parseObjectEntry() { + var children = new ArrayList(); + var header = new ArrayList(); + expect(Token.LBRACK, header, "unexpectedToken", "["); + ff(header); + header.add(parseExpr()); + expect(Token.RBRACK, header, "unexpectedToken", "]"); + if (lookahead() == Token.ASSIGN) { + ff(header); + header.add(makeTerminal(next())); + children.add(new Node(NodeType.OBJECT_ENTRY_HEADER, header)); + ff(children); + children.add(parseExpr()); + return new Node(NodeType.OBJECT_ENTRY, children); + } + children.add(new Node(NodeType.OBJECT_ENTRY_HEADER, header)); + ff(children); + children.addAll(parseBodyList()); + return new Node(NodeType.OBJECT_ENTRY, children); + } + + private Node parseObjectSpread() { + var children = new ArrayList(); + children.add(makeTerminal(next())); + ff(children); + children.add(parseExpr()); + return new Node(NodeType.OBJECT_SPREAD, children); + } + + private Node parseWhenGenerator() { + var children = new ArrayList(); + var header = new ArrayList(); + children.add(makeTerminal(next())); + ff(children); + expect(Token.LPAREN, header, "unexpectedToken", "("); + ff(header); + header.add(parseExpr()); + ff(header); + expect(Token.RPAREN, header, "unexpectedToken", ")"); + children.add(new Node(NodeType.WHEN_GENERATOR_HEADER, header)); + ff(children); + children.add(parseObjectBody()); + if (lookahead() == Token.ELSE) { + ff(children); + children.add(makeTerminal(next())); + ff(children); + children.add(parseObjectBody()); + } + return new Node(NodeType.WHEN_GENERATOR, children); + } + + private Node parseForGenerator() { + var children = new ArrayList(); + children.add(makeTerminal(next())); + ff(children); + var header = new ArrayList(); + expect(Token.LPAREN, header, "unexpectedToken", "("); + var headerDefinition = new ArrayList(); + var headerDefinitionHeader = new ArrayList(); + ff(headerDefinitionHeader); + headerDefinitionHeader.add(parseParameter()); + ff(headerDefinitionHeader); + if (lookahead == Token.COMMA) { + headerDefinitionHeader.add(makeTerminal(next())); + ff(headerDefinitionHeader); + headerDefinitionHeader.add(parseParameter()); + ff(headerDefinitionHeader); + } + expect(Token.IN, headerDefinitionHeader, "unexpectedToken", "in"); + headerDefinition.add( + new Node(NodeType.FOR_GENERATOR_HEADER_DEFINITION_HEADER, headerDefinitionHeader)); + ff(headerDefinition); + headerDefinition.add(parseExpr()); + ff(headerDefinition); + header.add(new Node(NodeType.FOR_GENERATOR_HEADER_DEFINITION, headerDefinition)); + expect(Token.RPAREN, header, "unexpectedToken", ")"); + children.add(new Node(NodeType.FOR_GENERATOR_HEADER, header)); + ff(children); + children.add(parseObjectBody()); + return new Node(NodeType.FOR_GENERATOR, children); + } + + private Node parseExpr() { + return parseExpr(null, 1); + } + + private Node parseExpr(@Nullable String expectation) { + return parseExpr(expectation, 1); + } + + private Node parseExpr(@Nullable String expectation, int minPrecedence) { + var expr = parseExprAtom(expectation); + var fullOpToken = fullLookahead(); + var operator = getOperator(fullOpToken.tk); + while (operator != null) { + if (operator.getPrec() < minPrecedence) break; + // `-` and `[]` must be in the same line as the left operand and have no semicolons inbetween + if ((operator == Operator.MINUS || operator == Operator.SUBSCRIPT) + && (fullOpToken.hasSemicolon || !expr.span.sameLine(fullOpToken.tk.span))) break; + var children = new ArrayList(); + children.add(expr); + ff(children); + var op = next(); + children.add(make(NodeType.OPERATOR, op.span)); + ff(children); + var nodeType = NodeType.BINARY_OP_EXPR; + var nextMinPrec = operator.isLeftAssoc() ? operator.getPrec() + 1 : operator.getPrec(); + switch (op.token) { + case IS, AS -> children.add(parseType()); + case LBRACK -> { + nodeType = NodeType.SUBSCRIPT_EXPR; + children.add(parseExpr("]")); + ff(children); + expect(Token.RBRACK, children, "unexpectedToken", "]"); + } + case NON_NULL -> nodeType = NodeType.NON_NULL_EXPR; + default -> children.add(parseExpr(expectation, nextMinPrec)); + } + + expr = new Node(nodeType, children); + fullOpToken = fullLookahead(); + operator = getOperator(fullOpToken.tk); + } + return expr; + } + + private @Nullable Operator getOperator(FullToken tk) { + return switch (tk.token) { + case POW -> Operator.POW; + case STAR -> Operator.MULT; + case DIV -> Operator.DIV; + case INT_DIV -> Operator.INT_DIV; + case MOD -> Operator.MOD; + case PLUS -> Operator.PLUS; + case MINUS -> Operator.MINUS; + case GT -> Operator.GT; + case GTE -> Operator.GTE; + case LT -> Operator.LT; + case LTE -> Operator.LTE; + case IS -> Operator.IS; + case AS -> Operator.AS; + case EQUAL -> Operator.EQ_EQ; + case NOT_EQUAL -> Operator.NOT_EQ; + case AND -> Operator.AND; + case OR -> Operator.OR; + case PIPE -> Operator.PIPE; + case COALESCE -> Operator.NULL_COALESCE; + case DOT -> Operator.DOT; + case QDOT -> Operator.QDOT; + case LBRACK -> Operator.SUBSCRIPT; + case NON_NULL -> Operator.NON_NULL; + default -> null; + }; + } + + private Node parseExprAtom(@Nullable String expectation) { + var expr = + switch (lookahead) { + case THIS -> new Node(NodeType.THIS_EXPR, next().span); + case OUTER -> new Node(NodeType.OUTER_EXPR, next().span); + case MODULE -> new Node(NodeType.MODULE_EXPR, next().span); + case NULL -> new Node(NodeType.NULL_EXPR, next().span); + case THROW -> { + var children = new ArrayList(); + children.add(makeTerminal(next())); + ff(children); + expect(Token.LPAREN, children, "unexpectedToken", "("); + ff(children); + children.add(parseExpr(")")); + ff(children); + expect(Token.RPAREN, children, "unexpectedToken", ")"); + yield new Node(NodeType.THROW_EXPR, children); + } + case TRACE -> { + var children = new ArrayList(); + children.add(makeTerminal(next())); + ff(children); + expect(Token.LPAREN, children, "unexpectedToken", "("); + ff(children); + children.add(parseExpr(")")); + ff(children); + expect(Token.RPAREN, children, "unexpectedToken", ")"); + yield new Node(NodeType.TRACE_EXPR, children); + } + case IMPORT, IMPORT_STAR -> { + var children = new ArrayList(); + children.add(makeTerminal(next())); + ff(children); + expect(Token.LPAREN, children, "unexpectedToken", "("); + ff(children); + children.add(parseStringConstant()); + ff(children); + expect(Token.RPAREN, children, "unexpectedToken", ")"); + yield new Node(NodeType.IMPORT_EXPR, children); + } + case READ, READ_STAR, READ_QUESTION -> { + var children = new ArrayList(); + children.add(makeTerminal(next())); + ff(children); + expect(Token.LPAREN, children, "unexpectedToken", "("); + ff(children); + children.add(parseExpr(")")); + ff(children); + expect(Token.RPAREN, children, "unexpectedToken", ")"); + yield new Node(NodeType.READ_EXPR, children); + } + case NEW -> { + var children = new ArrayList(); + var header = new ArrayList(); + header.add(makeTerminal(next())); + ff(header); + if (lookahead != Token.LBRACE) { + header.add(parseType("{")); + children.add(new Node(NodeType.NEW_HEADER, header)); + ff(children); + } else { + children.add(new Node(NodeType.NEW_HEADER, header)); + } + children.add(parseObjectBody()); + yield new Node(NodeType.NEW_EXPR, children); + } + case MINUS -> { + var children = new ArrayList(); + children.add(makeTerminal(next())); + ff(children); + // unary minus has higher precendence than most binary operators + children.add(parseExpr(expectation, 12)); + yield new Node(NodeType.UNARY_MINUS_EXPR, children); + } + case NOT -> { + var children = new ArrayList(); + children.add(makeTerminal(next())); + ff(children); + // logical not has higher precendence than most binary operators + children.add(parseExpr(expectation, 11)); + yield new Node(NodeType.LOGICAL_NOT_EXPR, children); + } + case LPAREN -> { + // can be function literal or parenthesized expression + if (isFunctionLiteral()) { + yield parseFunctionLiteral(); + } else { + yield parseParenthesizedExpr(); + } + } + case SUPER -> { + var children = new ArrayList(); + children.add(makeTerminal(next())); + ff(children); + if (lookahead == Token.DOT) { + children.add(makeTerminal(next())); + ff(children); + children.add(parseIdentifier()); + if (lookahead() == Token.LPAREN) { + ff(children); + children.add(parseArgumentList()); + } + yield new Node(NodeType.SUPER_ACCESS_EXPR, children); + } else { + expect(Token.LBRACK, children, "unexpectedToken", "["); + ff(children); + children.add(parseExpr()); + ff(children); + expect(Token.RBRACK, children, "unexpectedToken", "]"); + yield new Node(NodeType.SUPER_SUBSCRIPT_EXPR, children); + } + } + case IF -> { + var children = new ArrayList(); + var header = new ArrayList(); + header.add(makeTerminal(next())); + ff(header); + var condition = new ArrayList(); + var conditionExpr = new ArrayList(); + expect(Token.LPAREN, condition, "unexpectedToken", "("); + ff(conditionExpr); + conditionExpr.add(parseExpr(")")); + ff(conditionExpr); + condition.add(new Node(NodeType.IF_CONDITION_EXPR, conditionExpr)); + expect(Token.RPAREN, condition, "unexpectedToken", ")"); + header.add(new Node(NodeType.IF_CONDITION, condition)); + children.add(new Node(NodeType.IF_HEADER, header)); + var thenExpr = new ArrayList(); + ff(thenExpr); + thenExpr.add(parseExpr("else")); + ff(thenExpr); + children.add(new Node(NodeType.IF_THEN_EXPR, thenExpr)); + expect(Token.ELSE, children, "unexpectedToken", "else"); + var elseExpr = new ArrayList(); + ff(elseExpr); + elseExpr.add(parseExpr(expectation)); + children.add(new Node(NodeType.IF_ELSE_EXPR, elseExpr)); + yield new Node(NodeType.IF_EXPR, children); + } + case LET -> { + var children = new ArrayList(); + children.add(makeTerminal(next())); + ff(children); + var paramDef = new ArrayList(); + expect(Token.LPAREN, paramDef, "unexpectedToken", "("); + ff(paramDef); + var param = new ArrayList(); + param.add(parseParameter()); + ff(param); + expect(Token.ASSIGN, param, "unexpectedToken", "="); + ff(param); + param.add(parseExpr(")")); + paramDef.add(new Node(NodeType.LET_PARAMETER, param)); + ff(paramDef); + expect(Token.RPAREN, paramDef, "unexpectedToken", ")"); + children.add(new Node(NodeType.LET_PARAMETER_DEFINITION, paramDef)); + ff(children); + children.add(parseExpr(expectation)); + yield new Node(NodeType.LET_EXPR, children); + } + case TRUE, FALSE -> new Node(NodeType.BOOL_LITERAL_EXPR, next().span); + case INT, HEX, BIN, OCT -> new Node(NodeType.INT_LITERAL_EXPR, next().span); + case FLOAT -> new Node(NodeType.FLOAT_LITERAL_EXPR, next().span); + case STRING_START -> parseSingleLineStringLiteralExpr(); + case STRING_MULTI_START -> parseMultiLineStringLiteralExpr(); + case IDENTIFIER -> { + var children = new ArrayList(); + children.add(parseIdentifier()); + if (lookahead == Token.LPAREN + && noSemicolonInbetween() + && _lookahead.newLinesBetween == 0) { + children.add(parseArgumentList()); + } + yield new Node(NodeType.UNQUALIFIED_ACCESS_EXPR, children); + } + case EOF -> + throw parserError( + ErrorMessages.create("unexpectedEndOfFile"), prev().span.stopSpan()); + default -> { + var text = _lookahead.text(lexer); + if (expectation != null) { + throw parserError("unexpectedToken", text, expectation); + } + throw parserError("unexpectedTokenForExpression", text); + } + }; + return parseExprRest(expr); + } + + @SuppressWarnings("DuplicatedCode") + private Node parseExprRest(Node expr) { + // amends + if (lookahead() == Token.LBRACE) { + var children = new ArrayList(); + children.add(expr); + ff(children); + if (expr.type == NodeType.PARENTHESIZED_EXPR + || expr.type == NodeType.AMENDS_EXPR + || expr.type == NodeType.NEW_EXPR) { + children.add(parseObjectBody()); + return parseExprRest(new Node(NodeType.AMENDS_EXPR, children)); + } + throw parserError("unexpectedCurlyProbablyAmendsExpression", expr.text(lexer.getSource())); + } + return expr; + } + + private boolean isFunctionLiteral() { + var originalCursor = cursor; + try { + next(); // open ( + ff(); + var token = next().token; + ff(); + if (token == Token.RPAREN) { + return lookahead == Token.ARROW; + } + if (token == Token.UNDERSCORE) { + return true; + } + if (token != Token.IDENTIFIER) { + return false; + } + if (lookahead == Token.COMMA || lookahead == Token.COLON) { + return true; + } + if (lookahead == Token.RPAREN) { + next(); + ff(); + return lookahead == Token.ARROW; + } + return false; + } finally { + backtrackTo(originalCursor); + } + } + + private Node parseSingleLineStringLiteralExpr() { + var children = new ArrayList(); + var start = next(); + children.add(makeTerminal(start)); // string start + while (lookahead != Token.STRING_END) { + switch (lookahead) { + case STRING_PART -> { + var tk = next(); + if (!tk.text(lexer).isEmpty()) { + children.add(make(NodeType.STRING_CONSTANT, tk.span)); + } + } + case STRING_ESCAPE_NEWLINE, + STRING_ESCAPE_TAB, + STRING_ESCAPE_QUOTE, + STRING_ESCAPE_BACKSLASH, + STRING_ESCAPE_RETURN, + STRING_ESCAPE_UNICODE -> + children.add(make(NodeType.STRING_ESCAPE, next().span)); + case INTERPOLATION_START -> { + children.add(makeTerminal(next())); + ff(children); + children.add(parseExpr(")")); + ff(children); + expect(Token.RPAREN, children, "unexpectedToken", ")"); + } + case EOF -> { + var delimiter = new StringBuilder(start.text(lexer)).reverse().toString(); + throw parserError("missingDelimiter", delimiter); + } + } + } + children.add(makeTerminal(next())); // string end + return new Node(NodeType.SINGLE_LINE_STRING_LITERAL_EXPR, children); + } + + private Node parseMultiLineStringLiteralExpr() { + var children = new ArrayList(); + var start = next(); + children.add(makeTerminal(start)); // string start + if (lookahead != Token.STRING_NEWLINE) { + throw parserError(ErrorMessages.create("stringContentMustBeginOnNewLine"), spanLookahead); + } + while (lookahead != Token.STRING_END) { + switch (lookahead) { + case STRING_PART -> { + var tk = next(); + if (!tk.text(lexer).isEmpty()) { + children.add(make(NodeType.STRING_CONSTANT, tk.span)); + } + } + case STRING_NEWLINE -> children.add(make(NodeType.STRING_NEWLINE, next().span)); + case STRING_ESCAPE_NEWLINE, + STRING_ESCAPE_TAB, + STRING_ESCAPE_QUOTE, + STRING_ESCAPE_BACKSLASH, + STRING_ESCAPE_RETURN, + STRING_ESCAPE_UNICODE -> + children.add(make(NodeType.STRING_ESCAPE, next().span)); + case INTERPOLATION_START -> { + children.add(makeTerminal(next())); + ff(children); + children.add(parseExpr(")")); + ff(children); + expect(Token.RPAREN, children, "unexpectedToken", ")"); + } + case EOF -> { + var delimiter = new StringBuilder(start.text(lexer)).reverse().toString(); + throw parserError("missingDelimiter", delimiter); + } + } + } + children.add(makeTerminal(next())); // string end + validateStringEndDelimiter(children); + validateStringIndentation(children); + return new Node(NodeType.MULTI_LINE_STRING_LITERAL_EXPR, children); + } + + private void validateStringEndDelimiter(List nodes) { + var beforeLast = nodes.get(nodes.size() - 2); + if (beforeLast.type == NodeType.STRING_NEWLINE) return; + var text = beforeLast.text(lexer.getSource()); + if (!text.isBlank()) { + throw parserError( + ErrorMessages.create("closingStringDelimiterMustBeginOnNewLine"), beforeLast.span); + } + } + + private void validateStringIndentation(List nodes) { + var indentNode = nodes.get(nodes.size() - 2); + if (indentNode.type == NodeType.STRING_NEWLINE) return; + var indent = indentNode.text(lexer.getSource()); + var previousNewline = false; + for (var i = 1; i < nodes.size() - 2; i++) { + var child = nodes.get(i); + if (child.type != NodeType.STRING_NEWLINE && previousNewline) { + var text = child.text(lexer.getSource()); + if (!text.startsWith(indent)) { + throw parserError(ErrorMessages.create("stringIndentationMustMatchLastLine"), child.span); + } + } + previousNewline = child.type == NodeType.STRING_NEWLINE; + } + } + + private Node parseParenthesizedExpr() { + var children = new ArrayList(); + expect(Token.LPAREN, children, "unexpectedToken", "("); + if (lookahead() == Token.RPAREN) { + ff(children); + children.add(makeTerminal(next())); + return new Node(NodeType.PARENTHESIZED_EXPR, children); + } + var elements = new ArrayList(); + ff(elements); + elements.add(parseExpr(")")); + ff(elements); + children.add(new Node(NodeType.PARENTHESIZED_EXPR_ELEMENTS, elements)); + expect(Token.RPAREN, children, "unexpectedToken", ")"); + return new Node(NodeType.PARENTHESIZED_EXPR, children); + } + + private Node parseFunctionLiteral() { + var paramListChildren = new ArrayList(); + expect(Token.LPAREN, paramListChildren, "unexpectedToken", "("); + if (lookahead() == Token.RPAREN) { + ff(paramListChildren); + paramListChildren.add(makeTerminal(next())); + } else { + var elements = new ArrayList(); + ff(elements); + parseListOf(Token.RPAREN, elements, this::parseParameter); + paramListChildren.add(new Node(NodeType.PARAMETER_LIST_ELEMENTS, elements)); + expect(Token.RPAREN, paramListChildren, "unexpectedToken2", ",", ")"); + } + var children = new ArrayList(); + children.add(new Node(NodeType.PARAMETER_LIST, paramListChildren)); + ff(children); + expect(Token.ARROW, children, "unexpectedToken", "->"); + var body = new ArrayList(); + ff(body); + body.add(parseExpr()); + children.add(new Node(NodeType.FUNCTION_LITERAL_BODY, body)); + return new Node(NodeType.FUNCTION_LITERAL_EXPR, children); + } + + private Node parseType() { + return parseType(null); + } + + private Node parseType(@Nullable String expectation) { + var children = new ArrayList(); + var hasDefault = false; + FullSpan start = null; + if (lookahead == Token.STAR) { + var tk = next(); + start = tk.span; + children.add(makeTerminal(tk)); + ff(children); + hasDefault = true; + } + var first = parseTypeAtom(expectation); + children.add(first); + + if (lookahead() != Token.UNION) { + if (hasDefault) { + throw parserError(ErrorMessages.create("notAUnion"), start.endWith(first.span)); + } + return first; + } + + while (lookahead() == Token.UNION) { + ff(children); + children.add(makeTerminal(next())); + ff(children); + if (lookahead == Token.STAR) { + if (hasDefault) { + throw parserError("multipleUnionDefaults"); + } + children.add(makeTerminal(next())); + ff(children); + hasDefault = true; + } + var type = parseTypeAtom(expectation); + children.add(type); + } + return new Node(NodeType.UNION_TYPE, children); + } + + private Node parseTypeAtom(@Nullable String expectation) { + var typ = + switch (lookahead) { + case UNKNOWN -> make(NodeType.UNKNOWN_TYPE, next().span); + case NOTHING -> make(NodeType.NOTHING_TYPE, next().span); + case MODULE -> make(NodeType.MODULE_TYPE, next().span); + case LPAREN -> { + var children = new ArrayList(); + children.add(makeTerminal(next())); + var totalTypes = 0; + if (lookahead() == Token.RPAREN) { + ff(children); + children.add(makeTerminal(next())); + } else { + var elements = new ArrayList(); + ff(elements); + elements.add(parseType(")")); + ff(elements); + while (lookahead == Token.COMMA) { + elements.add(makeTerminal(next())); + ff(elements); + elements.add(parseType(")")); + totalTypes++; + ff(elements); + } + children.add(new Node(NodeType.PARENTHESIZED_TYPE_ELEMENTS, elements)); + expect(Token.RPAREN, children, "unexpectedToken2", ",", ")"); + } + if (totalTypes > 1 || lookahead() == Token.ARROW) { + var actualChildren = new ArrayList(); + actualChildren.add(new Node(NodeType.FUNCTION_TYPE_PARAMETERS, children)); + ff(actualChildren); + expect(Token.ARROW, actualChildren, "unexpectedToken", "->"); + ff(actualChildren); + actualChildren.add(parseType(expectation)); + yield new Node(NodeType.FUNCTION_TYPE, actualChildren); + } else { + yield new Node(NodeType.PARENTHESIZED_TYPE, children); + } + } + case IDENTIFIER -> { + var children = new ArrayList(); + children.add(parseQualifiedIdentifier()); + if (lookahead() == Token.LT) { + ff(children); + children.add(parseTypeArgumentList()); + } + yield new Node(NodeType.DECLARED_TYPE, children); + } + case STRING_START -> + new Node(NodeType.STRING_CONSTANT_TYPE, List.of(parseStringConstant())); + default -> { + var text = _lookahead.text(lexer); + if (expectation != null) { + throw parserError("unexpectedTokenForType2", text, expectation); + } + throw parserError("unexpectedTokenForType", text); + } + }; + + if (typ.type == NodeType.FUNCTION_TYPE) return typ; + return parseTypeEnd(typ); + } + + private Node parseTypeEnd(Node type) { + var children = new ArrayList(); + children.add(type); + // nullable types + if (lookahead() == Token.QUESTION) { + ff(children); + children.add(makeTerminal(next())); + var res = new Node(NodeType.NULLABLE_TYPE, children); + return parseTypeEnd(res); + } + // constrained types: have to start in the same line as the type + var fla = fullLookahead(); + if (fla.tk.token == Token.LPAREN && noSemicolonInbetween() && fla.tk.newLinesBetween == 0) { + ff(children); + var constraint = new ArrayList(); + constraint.add(makeTerminal(next())); + var elements = new ArrayList(); + ff(elements); + parseListOf(Token.RPAREN, elements, () -> parseExpr(")")); + constraint.add(new Node(NodeType.CONSTRAINED_TYPE_ELEMENTS, elements)); + expect(Token.RPAREN, constraint, "unexpectedToken2", ",", ")"); + children.add(new Node(NodeType.CONSTRAINED_TYPE_CONSTRAINT, constraint)); + var res = new Node(NodeType.CONSTRAINED_TYPE, children); + return parseTypeEnd(res); + } + return type; + } + + private Node parseAnnotation() { + var children = new ArrayList(); + children.add(makeTerminal(next())); + children.add(parseType()); + if (lookahead() == Token.LBRACE) { + ff(children); + children.add(parseObjectBody()); + } + return new Node(NodeType.ANNOTATION, children); + } + + private Node parseParameter() { + if (lookahead == Token.UNDERSCORE) { + return new Node(NodeType.PARAMETER, List.of(makeTerminal(next()))); + } + return parseTypedIdentifier(); + } + + private Node parseTypedIdentifier() { + var children = new ArrayList(); + children.add(parseIdentifier()); + if (lookahead() == Token.COLON) { + ff(children); + children.add(parseTypeAnnotation()); + } + return new Node(NodeType.PARAMETER, children); + } + + private Node parseParameterList() { + var children = new ArrayList(); + expect(Token.LPAREN, children, "unexpectedToken", "("); + ff(children); + if (lookahead == Token.RPAREN) { + children.add(makeTerminal(next())); + } else { + var elements = new ArrayList(); + parseListOf(Token.RPAREN, elements, this::parseParameter); + children.add(new Node(NodeType.PARAMETER_LIST_ELEMENTS, elements)); + expect(Token.RPAREN, children, "unexpectedToken2", ",", ")"); + } + return new Node(NodeType.PARAMETER_LIST, children); + } + + private List parseBodyList() { + if (lookahead != Token.LBRACE) { + throw parserError("unexpectedToken2", _lookahead.text(lexer), "{", "="); + } + var bodies = new ArrayList(); + do { + bodies.add(parseObjectBody()); + } while (lookahead() == Token.LBRACE); + return bodies; + } + + private Node parseTypeParameterList() { + var children = new ArrayList(); + expect(Token.LT, children, "unexpectedToken", "<"); + ff(children); + var elements = new ArrayList(); + parseListOf(Token.GT, elements, this::parseTypeParameter); + children.add(new Node(NodeType.TYPE_PARAMETER_LIST_ELEMENTS, elements)); + expect(Token.GT, children, "unexpectedToken2", ",", ">"); + return new Node(NodeType.TYPE_PARAMETER_LIST, children); + } + + private Node parseTypeArgumentList() { + var children = new ArrayList(); + expect(Token.LT, children, "unexpectedToken", "<"); + ff(children); + var elements = new ArrayList(); + parseListOf(Token.GT, elements, () -> parseType(">")); + children.add(new Node(NodeType.TYPE_ARGUMENT_LIST_ELEMENTS, elements)); + expect(Token.GT, children, "unexpectedToken2", ",", ">"); + return new Node(NodeType.TYPE_ARGUMENT_LIST, children); + } + + private Node parseArgumentList() { + var children = new ArrayList(); + expect(Token.LPAREN, children, "unexpectedToken", "("); + if (lookahead() == Token.RPAREN) { + ff(children); + children.add(makeTerminal(next())); + return new Node(NodeType.ARGUMENT_LIST, children); + } + var elements = new ArrayList(); + ff(elements); + parseListOf(Token.RPAREN, elements, () -> parseExpr(")")); + ff(elements); + children.add(new Node(NodeType.ARGUMENT_LIST_ELEMENTS, elements)); + expect(Token.RPAREN, children, "unexpectedToken2", ",", ")"); + return new Node(NodeType.ARGUMENT_LIST, children); + } + + private Node parseTypeParameter() { + var children = new ArrayList(); + if (lookahead == Token.IN) { + children.add(makeTerminal(next())); + } else if (lookahead == Token.OUT) { + children.add(makeTerminal(next())); + } + children.add(parseIdentifier()); + return new Node(NodeType.TYPE_PARAMETER, children); + } + + private Node parseTypeAnnotation() { + var children = new ArrayList(); + expect(Token.COLON, children, "unexpectedToken", ":"); + ff(children); + children.add(parseType()); + return new Node(NodeType.TYPE_ANNOTATION, children); + } + + private Node parseIdentifier() { + if (lookahead != Token.IDENTIFIER) { + if (lookahead.isKeyword()) { + throw parserError("keywordNotAllowedHere", lookahead.text()); + } + throw parserError("unexpectedToken", _lookahead.text(lexer), "identifier"); + } + return new Node(NodeType.IDENTIFIER, next().span); + } + + private Node parseStringConstant() { + var children = new ArrayList(); + var startTk = expect(Token.STRING_START, "unexpectedToken", "\""); + children.add(makeTerminal(startTk)); + while (lookahead != Token.STRING_END) { + switch (lookahead) { + case STRING_PART, + STRING_ESCAPE_NEWLINE, + STRING_ESCAPE_TAB, + STRING_ESCAPE_QUOTE, + STRING_ESCAPE_BACKSLASH, + STRING_ESCAPE_RETURN, + STRING_ESCAPE_UNICODE -> + children.add(makeTerminal(next())); + case EOF -> { + var delimiter = new StringBuilder(startTk.text(lexer)).reverse().toString(); + throw parserError("missingDelimiter", delimiter); + } + case INTERPOLATION_START -> throw parserError("interpolationInConstant"); + // the lexer makes sure we only get the above tokens inside a string + default -> throw new RuntimeException("Unreacheable code"); + } + } + children.add(makeTerminal(next())); // string end + return new Node(NodeType.STRING_CONSTANT, children); + } + + private FullToken expect(Token type, String errorKey, Object... messageArgs) { + if (lookahead != type) { + var span = spanLookahead; + if (lookahead == Token.EOF || _lookahead.newLinesBetween > 0) { + // don't point at the EOF or the next line, but at the end of the last token + span = prev().span.stopSpan(); + } + var args = messageArgs; + if (errorKey.startsWith("unexpectedToken")) { + args = new Object[messageArgs.length + 1]; + args[0] = lookahead == Token.EOF ? "EOF" : _lookahead.text(lexer); + System.arraycopy(messageArgs, 0, args, 1, messageArgs.length); + } + throw parserError(ErrorMessages.create(errorKey, args), span); + } + return next(); + } + + private void expect(Token type, List children, String errorKey, Object... messageArgs) { + var tk = expect(type, errorKey, messageArgs); + children.add(makeTerminal(tk)); + } + + private void parseListOf(Token terminator, List children, Supplier parser) { + children.add(parser.get()); + ff(children); + while (lookahead == Token.COMMA) { + // don't store the last comma + var comma = makeTerminal(next()); + if (lookahead() == terminator) break; + children.add(comma); + ff(children); + children.add(parser.get()); + ff(children); + } + } + + private GenericParserError parserError(String messageKey, Object... args) { + return new GenericParserError(ErrorMessages.create(messageKey, args), spanLookahead); + } + + private GenericParserError parserError(String message, FullSpan span) { + return new GenericParserError(message, span); + } + + private boolean isModuleDecl() { + var _cursor = cursor; + var ftk = tokens.get(_cursor); + while (ftk.token.isAffix() || ftk.token.isModifier()) { + ftk = tokens.get(++_cursor); + } + var tk = ftk.token; + return tk == Token.MODULE || tk == Token.EXTENDS || tk == Token.AMENDS; + } + + private boolean isImport() { + var _cursor = cursor; + var ftk = tokens.get(_cursor); + while (ftk.token.isAffix()) { + ftk = tokens.get(++_cursor); + } + var tk = ftk.token; + return tk == Token.IMPORT || tk == Token.IMPORT_STAR; + } + + private FullToken next() { + var tmp = tokens.get(cursor++); + _lookahead = tokens.get(cursor); + lookahead = _lookahead.token; + spanLookahead = _lookahead.span; + return tmp; + } + + private boolean noSemicolonInbetween() { + return tokens.get(cursor - 1).token != Token.SEMICOLON; + } + + private void backtrack() { + var tmp = tokens.get(--cursor); + lookahead = tmp.token; + spanLookahead = tmp.span; + } + + private void backtrackTo(int point) { + cursor = point; + var tmp = tokens.get(cursor); + lookahead = tmp.token; + spanLookahead = tmp.span; + } + + private FullToken prev() { + return tokens.get(cursor - 1); + } + + // Jump over affixes and find the next token + private Token lookahead() { + var i = cursor; + var tmp = tokens.get(i); + while (tmp.token.isAffix() && tmp.token != Token.EOF) { + tmp = tokens.get(++i); + } + return tmp.token; + } + + // Jump over affixes and find the next token + private LookaheadSearch fullLookahead() { + var i = cursor; + var hasSemicolon = false; + var tmp = tokens.get(i); + while (tmp.token.isAffix() && tmp.token != Token.EOF) { + if (tmp.token == Token.SEMICOLON) { + hasSemicolon = true; + } + tmp = tokens.get(++i); + } + return new LookaheadSearch(tmp, hasSemicolon); + } + + private record LookaheadSearch(FullToken tk, boolean hasSemicolon) {} + + private record HeaderParts(List preffixes, @Nullable Node modifierList) {} + + private HeaderParts getHeaderParts(List nodes) { + if (nodes.isEmpty()) return new HeaderParts(nodes, null); + var last = nodes.get(nodes.size() - 1); + if (last.type == NodeType.MODIFIER_LIST) { + return new HeaderParts(nodes.subList(0, nodes.size() - 1), last); + } + return new HeaderParts(nodes, null); + } + + private Node make(NodeType type, FullSpan span) { + return new Node(type, span); + } + + private Node makeAffix(FullToken tk) { + return new Node(nodeTypeForAffix(tk.token), tk.span); + } + + private Node makeTerminal(FullToken tk) { + return new Node(NodeType.TERMINAL, tk.span); + } + + // fast-forward over affix tokens + // store children + private void ff(List children) { + var tmp = tokens.get(cursor); + while (tmp.token.isAffix()) { + children.add(makeAffix(tmp)); + tmp = tokens.get(++cursor); + } + _lookahead = tmp; + lookahead = _lookahead.token; + spanLookahead = _lookahead.span; + } + + // fast-forward over affix tokens + private void ff() { + var tmp = tokens.get(cursor); + while (tmp.token.isAffix()) { + tmp = tokens.get(++cursor); + } + _lookahead = tmp; + lookahead = _lookahead.token; + spanLookahead = _lookahead.span; + } + + private NodeType nodeTypeForAffix(Token token) { + return switch (token) { + case LINE_COMMENT -> NodeType.LINE_COMMENT; + case BLOCK_COMMENT -> NodeType.BLOCK_COMMENT; + case SHEBANG -> NodeType.SHEBANG; + case SEMICOLON -> NodeType.SEMICOLON; + default -> throw new RuntimeException("Unreacheable code"); + }; + } + + private record FullToken(Token token, FullSpan span, int newLinesBetween) { + String text(Lexer lexer) { + return lexer.textFor(span.charIndex(), span.length()); + } + } + + private record HeaderResult( + boolean hasDocComment, boolean hasAnnotations, boolean hasModifiers) {} +} diff --git a/pkl-parser/src/main/java/org/pkl/parser/GenericParserError.java b/pkl-parser/src/main/java/org/pkl/parser/GenericParserError.java new file mode 100644 index 00000000..33fa4037 --- /dev/null +++ b/pkl-parser/src/main/java/org/pkl/parser/GenericParserError.java @@ -0,0 +1,36 @@ +/* + * Copyright © 2025 Apple Inc. and the Pkl project authors. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.pkl.parser; + +import org.pkl.parser.syntax.generic.FullSpan; + +public class GenericParserError extends RuntimeException { + private final FullSpan span; + + public GenericParserError(String msg, FullSpan span) { + super(msg); + this.span = span; + } + + public FullSpan getSpan() { + return span; + } + + @Override + public String toString() { + return getMessage() + " at " + span; + } +} diff --git a/pkl-parser/src/main/java/org/pkl/parser/Lexer.java b/pkl-parser/src/main/java/org/pkl/parser/Lexer.java index 296ecd19..a127286e 100644 --- a/pkl-parser/src/main/java/org/pkl/parser/Lexer.java +++ b/pkl-parser/src/main/java/org/pkl/parser/Lexer.java @@ -17,6 +17,7 @@ package org.pkl.parser; import java.util.ArrayDeque; import java.util.Deque; +import org.pkl.parser.syntax.generic.FullSpan; import org.pkl.parser.util.ErrorMessages; public class Lexer { @@ -25,6 +26,10 @@ public class Lexer { private final int size; protected int cursor = 0; protected int sCursor = 0; + private int line = 1; + private int sLine = 1; + private int col = 1; + private int sCol = 1; private char lookahead; private State state = State.DEFAULT; private final Deque interpolationStack = new ArrayDeque<>(); @@ -50,6 +55,11 @@ public class Lexer { return new Span(sCursor, cursor - sCursor); } + // The full span of the last lexed token + public FullSpan fullSpan() { + return new FullSpan(sCursor, cursor - sCursor, sLine, sCol, line, col); + } + // The text of the last lexed token public String text() { return new String(source, sCursor, cursor - sCursor); @@ -65,6 +75,8 @@ public class Lexer { public Token next() { sCursor = cursor; + sLine = line; + sCol = col; newLinesBetween = 0; return switch (state) { case DEFAULT -> nextDefault(); @@ -79,7 +91,9 @@ public class Lexer { sCursor = cursor; if (ch == '\n') { newLinesBetween++; + sLine = line; } + sCol = col; ch = nextChar(); } return switch (ch) { @@ -678,15 +692,23 @@ public class Lexer { } else { lookahead = source[cursor]; } + if (tmp == '\n') { + line++; + col = 1; + } else { + col++; + } return tmp; } private void backup() { lookahead = source[--cursor]; + col--; } private void backup(int amount) { cursor -= amount; + col -= amount; lookahead = source[cursor]; } diff --git a/pkl-parser/src/main/java/org/pkl/parser/Parser.java b/pkl-parser/src/main/java/org/pkl/parser/Parser.java index b6c1cc1e..14aee97d 100644 --- a/pkl-parser/src/main/java/org/pkl/parser/Parser.java +++ b/pkl-parser/src/main/java/org/pkl/parser/Parser.java @@ -18,7 +18,6 @@ package org.pkl.parser; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; -import java.util.EnumSet; import java.util.List; import java.util.function.Supplier; import org.pkl.parser.syntax.Annotation; @@ -1809,16 +1808,13 @@ public class Parser { private FullToken forceNext() { var tk = lexer.next(); precededBySemicolon = false; - while (AFFIXES.contains(tk)) { + while (tk.isAffix()) { precededBySemicolon = precededBySemicolon || tk == Token.SEMICOLON; tk = lexer.next(); } return new FullToken(tk, lexer.span(), lexer.newLinesBetween); } - private static final EnumSet AFFIXES = - EnumSet.of(Token.LINE_COMMENT, Token.BLOCK_COMMENT, Token.SEMICOLON); - // Like next, but don't ignore comments private FullToken nextComment() { prev = _lookahead; diff --git a/pkl-parser/src/main/java/org/pkl/parser/Token.java b/pkl-parser/src/main/java/org/pkl/parser/Token.java index 034d02bc..b20d8130 100644 --- a/pkl-parser/src/main/java/org/pkl/parser/Token.java +++ b/pkl-parser/src/main/java/org/pkl/parser/Token.java @@ -191,6 +191,13 @@ public enum Token { }; } + public boolean isAffix() { + return switch (this) { + case LINE_COMMENT, BLOCK_COMMENT, SEMICOLON -> true; + default -> false; + }; + } + public String text() { if (this == UNDERSCORE) { return "_"; diff --git a/pkl-parser/src/main/java/org/pkl/parser/syntax/Operator.java b/pkl-parser/src/main/java/org/pkl/parser/syntax/Operator.java index 1dba3e99..a92a5a61 100644 --- a/pkl-parser/src/main/java/org/pkl/parser/syntax/Operator.java +++ b/pkl-parser/src/main/java/org/pkl/parser/syntax/Operator.java @@ -35,8 +35,10 @@ public enum Operator { INT_DIV(9, true), MOD(9, true), POW(10, false), - DOT(11, true), - QDOT(11, true); + NON_NULL(16, true), + SUBSCRIPT(18, true), + DOT(20, true), + QDOT(20, true); private final int prec; private final boolean isLeftAssoc; @@ -53,4 +55,33 @@ public enum Operator { public boolean isLeftAssoc() { return isLeftAssoc; } + + public static Operator byName(String name) { + return switch (name) { + case "??" -> NULL_COALESCE; + case "|>" -> PIPE; + case "||" -> OR; + case "&&" -> AND; + case "==" -> EQ_EQ; + case "!=" -> NOT_EQ; + case "is" -> IS; + case "as" -> AS; + case "<" -> LT; + case "<=" -> LTE; + case ">" -> GT; + case ">=" -> GTE; + case "+" -> PLUS; + case "-" -> MINUS; + case "*" -> MULT; + case "/" -> DIV; + case "~/" -> INT_DIV; + case "%" -> MOD; + case "**" -> POW; + case "!!" -> NON_NULL; + case "[" -> SUBSCRIPT; + case "." -> DOT; + case "?." -> QDOT; + default -> throw new RuntimeException("Unknown operator: " + name); + }; + } } diff --git a/pkl-parser/src/main/java/org/pkl/parser/syntax/generic/FullSpan.java b/pkl-parser/src/main/java/org/pkl/parser/syntax/generic/FullSpan.java new file mode 100644 index 00000000..319ba1f2 --- /dev/null +++ b/pkl-parser/src/main/java/org/pkl/parser/syntax/generic/FullSpan.java @@ -0,0 +1,49 @@ +/* + * Copyright © 2025 Apple Inc. and the Pkl project authors. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.pkl.parser.syntax.generic; + +import org.pkl.parser.Span; + +public record FullSpan( + int charIndex, int length, int lineBegin, int colBegin, int lineEnd, int colEnd) { + + public FullSpan endWith(FullSpan end) { + return new FullSpan( + charIndex, + end.charIndex - charIndex + end.length, + lineBegin, + colBegin, + end.lineEnd, + end.colEnd); + } + + public Span toSpan() { + return new Span(charIndex, length); + } + + public boolean sameLine(FullSpan other) { + return lineEnd == other.lineBegin; + } + + public FullSpan stopSpan() { + return new FullSpan(charIndex + length - 1, 1, lineEnd, colEnd, lineEnd, colEnd); + } + + @Override + public String toString() { + return "(%d:%d - %d:%d)".formatted(lineBegin, colBegin, lineEnd, colEnd); + } +} diff --git a/pkl-parser/src/main/java/org/pkl/parser/syntax/generic/Node.java b/pkl-parser/src/main/java/org/pkl/parser/syntax/generic/Node.java new file mode 100644 index 00000000..98ff85a1 --- /dev/null +++ b/pkl-parser/src/main/java/org/pkl/parser/syntax/generic/Node.java @@ -0,0 +1,85 @@ +/* + * Copyright © 2025 Apple Inc. and the Pkl project authors. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.pkl.parser.syntax.generic; + +import java.util.Collections; +import java.util.List; +import java.util.Objects; +import org.pkl.parser.util.Nullable; + +public class Node { + public final List children; + public final FullSpan span; + public final NodeType type; + private @Nullable String text; + + public Node(NodeType type, FullSpan span) { + this(type, span, Collections.emptyList()); + } + + public Node(NodeType type, FullSpan span, List children) { + this.type = type; + this.span = span; + this.children = Collections.unmodifiableList(children); + } + + public Node(NodeType type, List children) { + this.type = type; + if (children.isEmpty()) throw new RuntimeException("No children or span given for node"); + var end = children.get(children.size() - 1).span; + this.span = children.get(0).span.endWith(end); + this.children = Collections.unmodifiableList(children); + } + + public String text(char[] source) { + if (text == null) { + text = new String(source, span.charIndex(), span.length()); + } + return text; + } + + /** Returns the first child of type {@code type} or {@code null}. */ + public @Nullable Node findChildByType(NodeType type) { + for (var child : children) { + if (child.type == type) return child; + } + return null; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + Node node = (Node) o; + return Objects.equals(children, node.children) + && Objects.equals(span, node.span) + && Objects.equals(type, node.type); + } + + @Override + public int hashCode() { + return Objects.hash(children, span, type); + } + + @Override + public String toString() { + return "Node{type='" + type + "', span=" + span + ", children=" + children + '}'; + } +} diff --git a/pkl-parser/src/main/java/org/pkl/parser/syntax/generic/NodeType.java b/pkl-parser/src/main/java/org/pkl/parser/syntax/generic/NodeType.java new file mode 100644 index 00000000..c898fdab --- /dev/null +++ b/pkl-parser/src/main/java/org/pkl/parser/syntax/generic/NodeType.java @@ -0,0 +1,176 @@ +/* + * Copyright © 2025 Apple Inc. and the Pkl project authors. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.pkl.parser.syntax.generic; + +public enum NodeType { + TERMINAL, + SHEBANG, + // affixes, + LINE_COMMENT(NodeKind.AFFIX), + BLOCK_COMMENT(NodeKind.AFFIX), + SEMICOLON(NodeKind.AFFIX), + + MODULE, + DOC_COMMENT, + DOC_COMMENT_LINE, + MODIFIER, + MODIFIER_LIST, + AMENDS_CLAUSE, + EXTENDS_CLAUSE, + MODULE_DECLARATION, + MODULE_DEFINITION, + ANNOTATION, + IDENTIFIER, + QUALIFIED_IDENTIFIER, + IMPORT, + IMPORT_ALIAS, + IMPORT_LIST, + TYPEALIAS, + TYPEALIAS_HEADER, + TYPEALIAS_BODY, + CLASS, + CLASS_HEADER, + CLASS_HEADER_EXTENDS, + CLASS_BODY, + CLASS_BODY_ELEMENTS, + CLASS_METHOD, + CLASS_METHOD_HEADER, + CLASS_METHOD_BODY, + CLASS_PROPERTY, + CLASS_PROPERTY_HEADER, + CLASS_PROPERTY_HEADER_BEGIN, + CLASS_PROPERTY_BODY, + OBJECT_BODY, + OBJECT_MEMBER_LIST, + PARAMETER, + TYPE_ANNOTATION, + PARAMETER_LIST, + PARAMETER_LIST_ELEMENTS, + TYPE_PARAMETER_LIST, + TYPE_PARAMETER_LIST_ELEMENTS, + ARGUMENT_LIST, + ARGUMENT_LIST_ELEMENTS, + TYPE_ARGUMENT_LIST, + TYPE_ARGUMENT_LIST_ELEMENTS, + OBJECT_PARAMETER_LIST, + TYPE_PARAMETER, + STRING_CONSTANT, + OPERATOR, + STRING_NEWLINE, + STRING_ESCAPE, + + // members + OBJECT_ELEMENT, + OBJECT_PROPERTY, + OBJECT_PROPERTY_HEADER, + OBJECT_PROPERTY_HEADER_BEGIN, + OBJECT_PROPERTY_BODY, + OBJECT_METHOD, + MEMBER_PREDICATE, + OBJECT_ENTRY, + OBJECT_ENTRY_HEADER, + OBJECT_SPREAD, + WHEN_GENERATOR, + WHEN_GENERATOR_HEADER, + FOR_GENERATOR, + FOR_GENERATOR_HEADER, + FOR_GENERATOR_HEADER_DEFINITION, + FOR_GENERATOR_HEADER_DEFINITION_HEADER, + + // expressions + THIS_EXPR(NodeKind.EXPR), + OUTER_EXPR(NodeKind.EXPR), + MODULE_EXPR(NodeKind.EXPR), + NULL_EXPR(NodeKind.EXPR), + THROW_EXPR(NodeKind.EXPR), + TRACE_EXPR(NodeKind.EXPR), + IMPORT_EXPR(NodeKind.EXPR), + READ_EXPR(NodeKind.EXPR), + NEW_EXPR(NodeKind.EXPR), + NEW_HEADER, + UNARY_MINUS_EXPR(NodeKind.EXPR), + LOGICAL_NOT_EXPR(NodeKind.EXPR), + FUNCTION_LITERAL_EXPR(NodeKind.EXPR), + FUNCTION_LITERAL_BODY, + PARENTHESIZED_EXPR(NodeKind.EXPR), + PARENTHESIZED_EXPR_ELEMENTS, + SUPER_SUBSCRIPT_EXPR(NodeKind.EXPR), + SUPER_ACCESS_EXPR(NodeKind.EXPR), + SUBSCRIPT_EXPR(NodeKind.EXPR), + IF_EXPR(NodeKind.EXPR), + IF_HEADER, + IF_CONDITION, + IF_CONDITION_EXPR, + IF_THEN_EXPR, + IF_ELSE_EXPR, + LET_EXPR(NodeKind.EXPR), + LET_PARAMETER_DEFINITION, + LET_PARAMETER, + BOOL_LITERAL_EXPR(NodeKind.EXPR), + INT_LITERAL_EXPR(NodeKind.EXPR), + FLOAT_LITERAL_EXPR(NodeKind.EXPR), + SINGLE_LINE_STRING_LITERAL_EXPR(NodeKind.EXPR), + MULTI_LINE_STRING_LITERAL_EXPR(NodeKind.EXPR), + UNQUALIFIED_ACCESS_EXPR(NodeKind.EXPR), + NON_NULL_EXPR(NodeKind.EXPR), + AMENDS_EXPR(NodeKind.EXPR), + BINARY_OP_EXPR(NodeKind.EXPR), + + // types + UNKNOWN_TYPE(NodeKind.TYPE), + NOTHING_TYPE(NodeKind.TYPE), + MODULE_TYPE(NodeKind.TYPE), + UNION_TYPE(NodeKind.TYPE), + FUNCTION_TYPE(NodeKind.TYPE), + FUNCTION_TYPE_PARAMETERS, + PARENTHESIZED_TYPE(NodeKind.TYPE), + PARENTHESIZED_TYPE_ELEMENTS, + DECLARED_TYPE(NodeKind.TYPE), + NULLABLE_TYPE(NodeKind.TYPE), + STRING_CONSTANT_TYPE(NodeKind.TYPE), + CONSTRAINED_TYPE(NodeKind.TYPE), + CONSTRAINED_TYPE_CONSTRAINT, + CONSTRAINED_TYPE_ELEMENTS; + + private final NodeKind kind; + + NodeType() { + this.kind = NodeKind.NONE; + } + + NodeType(NodeKind kind) { + this.kind = kind; + } + + public boolean isAffix() { + return kind == NodeKind.AFFIX; + } + + public boolean isExpression() { + return kind == NodeKind.EXPR; + } + + public boolean isType() { + return kind == NodeKind.TYPE; + } + + private enum NodeKind { + TYPE, + EXPR, + AFFIX, + NONE; + } +} diff --git a/pkl-parser/src/main/java/org/pkl/parser/syntax/generic/package-info.java b/pkl-parser/src/main/java/org/pkl/parser/syntax/generic/package-info.java new file mode 100644 index 00000000..63e51082 --- /dev/null +++ b/pkl-parser/src/main/java/org/pkl/parser/syntax/generic/package-info.java @@ -0,0 +1,4 @@ +@NonnullByDefault +package org.pkl.parser.syntax.generic; + +import org.pkl.parser.util.NonnullByDefault; diff --git a/pkl-parser/src/test/kotlin/org/pkl/parser/GenericSexpRenderer.kt b/pkl-parser/src/test/kotlin/org/pkl/parser/GenericSexpRenderer.kt new file mode 100644 index 00000000..2537dfe2 --- /dev/null +++ b/pkl-parser/src/test/kotlin/org/pkl/parser/GenericSexpRenderer.kt @@ -0,0 +1,261 @@ +/* + * Copyright © 2025 Apple Inc. and the Pkl project authors. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.pkl.parser + +import java.util.EnumSet +import org.pkl.parser.syntax.generic.Node +import org.pkl.parser.syntax.generic.NodeType + +class GenericSexpRenderer(code: String) { + private var tab = "" + private var buf = StringBuilder() + private val source = code.toCharArray() + + fun render(node: Node): String { + innerRender(node) + return buf.toString() + } + + private fun innerRender(node: Node) { + if (node.type == NodeType.UNION_TYPE) { + renderUnionType(node) + return + } + if (node.type == NodeType.BINARY_OP_EXPR && binopName(node).endsWith("ualifiedAccessExpr")) { + renderQualifiedAccess(node) + return + } + doRender(name(node), collectChildren(node)) + } + + private fun doRender(name: String, children: List) { + buf.append(tab) + buf.append("(") + buf.append(name) + val oldTab = increaseTab() + for (child in children) { + buf.append('\n') + innerRender(child) + } + tab = oldTab + buf.append(')') + } + + private fun renderUnionType(node: Node) { + buf.append(tab) + buf.append("(") + buf.append(name(node)) + val oldTab = increaseTab() + var previousTerminal: Node? = null + for (child in node.children) { + if (child.type == NodeType.TERMINAL) previousTerminal = child + if (child.type in IGNORED_CHILDREN) continue + buf.append('\n') + if (previousTerminal != null && previousTerminal.text(source) == "*") { + previousTerminal = null + renderDefaultUnionType(child) + } else { + innerRender(child) + } + } + tab = oldTab + buf.append(')') + } + + private fun renderQualifiedAccess(node: Node) { + var children = node.children + if (children.last().type == NodeType.UNQUALIFIED_ACCESS_EXPR) { + children = children.dropLast(1) + collectChildren(children.last()) + } + val toRender = mutableListOf() + for (child in children) { + if (child.type in IGNORED_CHILDREN || child.type == NodeType.OPERATOR) continue + toRender += child + } + doRender(name(node), toRender) + } + + private fun renderDefaultUnionType(node: Node) { + buf.append(tab) + buf.append("(defaultUnionType\n") + val oldTab = increaseTab() + innerRender(node) + tab = oldTab + buf.append(')') + } + + private fun collectChildren(node: Node): List = + when (node.type) { + NodeType.MULTI_LINE_STRING_LITERAL_EXPR -> + node.children.filter { it.type !in IGNORED_CHILDREN && !it.type.isStringData() } + NodeType.SINGLE_LINE_STRING_LITERAL_EXPR -> { + val children = node.children.filter { it.type !in IGNORED_CHILDREN } + val res = mutableListOf() + var prev: Node? = null + for (child in children) { + val inARow = child.type.isStringData() && (prev != null && prev.type.isStringData()) + if (!inARow) { + res += child + } + prev = child + } + res + } + NodeType.DOC_COMMENT -> listOf() + else -> { + val nodes = mutableListOf() + for (child in node.children) { + if (child.type in IGNORED_CHILDREN) continue + if (child.type in UNPACK_CHILDREN) { + nodes += collectChildren(child) + } else { + nodes += child + } + } + nodes + } + } + + private fun NodeType.isStringData(): Boolean = + this == NodeType.STRING_CONSTANT || this == NodeType.STRING_ESCAPE + + private fun name(node: Node): String = + when (node.type) { + NodeType.MODULE_DECLARATION -> "moduleHeader" + NodeType.IMPORT -> importName(node, isExpr = false) + NodeType.IMPORT_EXPR -> importName(node, isExpr = true) + NodeType.BINARY_OP_EXPR -> binopName(node) + NodeType.CLASS -> "clazz" + NodeType.EXTENDS_CLAUSE, + NodeType.AMENDS_CLAUSE -> "extendsOrAmendsClause" + NodeType.TYPEALIAS -> "typeAlias" + NodeType.STRING_ESCAPE -> "stringConstant" + NodeType.READ_EXPR -> { + val terminal = node.children.find { it.type == NodeType.TERMINAL }!!.text(source) + when (terminal) { + "read*" -> "readGlobExpr" + "read?" -> "readNullExpr" + else -> "readExpr" + } + } + else -> { + val names = node.type.name.split('_').map { it.lowercase() } + if (names.size > 1) { + val capitalized = names.drop(1).map { n -> n.replaceFirstChar { it.titlecase() } } + (listOf(names[0]) + capitalized).joinToString("") + } else names[0] + } + } + + private fun importName(node: Node, isExpr: Boolean): String { + val terminal = node.children.find { it.type == NodeType.TERMINAL }!!.text(source) + val suffix = if (isExpr) "Expr" else "Clause" + return if (terminal == "import*") "importGlob$suffix" else "import$suffix" + } + + private fun binopName(node: Node): String { + val op = node.children.find { it.type == NodeType.OPERATOR }!!.text(source) + return when (op) { + "**" -> "exponentiationExpr" + "*", + "/", + "~/", + "%" -> "multiplicativeExpr" + "+", + "-" -> "additiveExpr" + ">", + ">=", + "<", + "<=" -> "comparisonExpr" + "is" -> "typeCheckExpr" + "as" -> "typeCastExpr" + "==", + "!=" -> "equalityExpr" + "&&" -> "logicalAndExpr" + "||" -> "logicalOrExpr" + "|>" -> "pipeExpr" + "??" -> "nullCoalesceExpr" + "." -> "qualifiedAccessExpr" + "?." -> "nullableQualifiedAccessExpr" + else -> throw RuntimeException("Unknown operator: $op") + } + } + + private fun increaseTab(): String { + val old = tab + tab += " " + return old + } + + companion object { + private val IGNORED_CHILDREN = + EnumSet.of( + NodeType.LINE_COMMENT, + NodeType.BLOCK_COMMENT, + NodeType.SHEBANG, + NodeType.SEMICOLON, + NodeType.TERMINAL, + NodeType.OPERATOR, + NodeType.STRING_NEWLINE, + ) + + private val UNPACK_CHILDREN = + EnumSet.of( + NodeType.MODULE_DEFINITION, + NodeType.IMPORT_LIST, + NodeType.IMPORT_ALIAS, + NodeType.TYPEALIAS_HEADER, + NodeType.TYPEALIAS_BODY, + NodeType.CLASS_PROPERTY_HEADER, + NodeType.CLASS_PROPERTY_HEADER_BEGIN, + NodeType.CLASS_PROPERTY_BODY, + NodeType.CLASS_METHOD_HEADER, + NodeType.CLASS_METHOD_BODY, + NodeType.CLASS_HEADER, + NodeType.CLASS_HEADER_EXTENDS, + NodeType.CLASS_BODY_ELEMENTS, + NodeType.MODIFIER_LIST, + NodeType.NEW_HEADER, + NodeType.OBJECT_MEMBER_LIST, + NodeType.OBJECT_ENTRY_HEADER, + NodeType.OBJECT_PROPERTY_HEADER, + NodeType.OBJECT_PROPERTY_HEADER_BEGIN, + NodeType.OBJECT_PROPERTY_BODY, + NodeType.OBJECT_PARAMETER_LIST, + NodeType.FOR_GENERATOR_HEADER, + NodeType.FOR_GENERATOR_HEADER_DEFINITION, + NodeType.FOR_GENERATOR_HEADER_DEFINITION_HEADER, + NodeType.WHEN_GENERATOR_HEADER, + NodeType.IF_HEADER, + NodeType.IF_CONDITION, + NodeType.IF_CONDITION_EXPR, + NodeType.IF_THEN_EXPR, + NodeType.IF_ELSE_EXPR, + NodeType.FUNCTION_LITERAL_BODY, + NodeType.ARGUMENT_LIST_ELEMENTS, + NodeType.PARAMETER_LIST_ELEMENTS, + NodeType.CONSTRAINED_TYPE_CONSTRAINT, + NodeType.CONSTRAINED_TYPE_ELEMENTS, + NodeType.TYPE_PARAMETER_LIST_ELEMENTS, + NodeType.TYPE_ARGUMENT_LIST_ELEMENTS, + NodeType.LET_PARAMETER_DEFINITION, + NodeType.LET_PARAMETER, + NodeType.PARENTHESIZED_EXPR_ELEMENTS, + NodeType.PARENTHESIZED_TYPE_ELEMENTS, + NodeType.FUNCTION_TYPE_PARAMETERS, + ) + } +} diff --git a/pkl-parser/src/test/kotlin/org/pkl/parser/ParserComparisonTest.kt b/pkl-parser/src/test/kotlin/org/pkl/parser/ParserComparisonTest.kt new file mode 100644 index 00000000..0cb1450f --- /dev/null +++ b/pkl-parser/src/test/kotlin/org/pkl/parser/ParserComparisonTest.kt @@ -0,0 +1,118 @@ +/* + * Copyright © 2024-2025 Apple Inc. and the Pkl project authors. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.pkl.parser + +import java.nio.file.Path +import kotlin.io.path.Path +import kotlin.io.path.extension +import kotlin.io.path.pathString +import kotlin.io.path.readText +import org.assertj.core.api.Assertions.assertThat +import org.assertj.core.api.SoftAssertions +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.parallel.Execution +import org.junit.jupiter.api.parallel.ExecutionMode +import org.pkl.commons.walk + +@Execution(ExecutionMode.CONCURRENT) +class ParserComparisonTest { + + @Test + fun compareSnippetTests() { + SoftAssertions.assertSoftly { softly -> + getSnippets() + .parallelStream() + .map { Pair(it.pathString, it.readText()) } + .forEach { (path, snippet) -> + try { + compare(snippet, path, softly) + } catch (e: GenericParserError) { + softly.fail("path: $path. Message: ${e.message}", e) + } + } + } + } + + fun getSnippets(): List { + return Path("../pkl-core/src/test/files/LanguageSnippetTests/input") + .walk() + .filter { path -> + val pathStr = path.toString().replace("\\", "/") + path.extension == "pkl" && + !exceptions.any { pathStr.endsWith(it) } && + !regexExceptions.any { it.matches(pathStr) } + } + .toList() + } + + fun compare(code: String, path: String? = null, softly: SoftAssertions? = null) { + val (sexp, genSexp) = renderBoth(code) + when { + (path != null && softly != null) -> + softly.assertThat(genSexp).`as`("path: $path").isEqualTo(sexp) + else -> assertThat(genSexp).`as`("path: $path").isEqualTo(sexp) + } + } + + fun renderBoth(code: String): Pair = + Pair(renderCode(code), renderGenericCode(code)) + + companion object { + private fun renderCode(code: String): String { + val parser = Parser() + val mod = parser.parseModule(code) + val renderer = SexpRenderer() + return renderer.render(mod) + } + + private fun renderGenericCode(code: String): String { + val parser = GenericParser() + val mod = parser.parseModule(code) + val renderer = GenericSexpRenderer(code) + return renderer.render(mod) + } + + // tests that are not syntactically valid Pkl + private val exceptions = + setOf( + "stringError1.pkl", + "annotationIsNotExpression2.pkl", + "amendsRequiresParens.pkl", + "errors/parser18.pkl", + "errors/nested1.pkl", + "errors/invalidCharacterEscape.pkl", + "errors/invalidUnicodeEscape.pkl", + "errors/unterminatedUnicodeEscape.pkl", + "errors/keywordNotAllowedHere1.pkl", + "errors/keywordNotAllowedHere2.pkl", + "errors/keywordNotAllowedHere3.pkl", + "errors/keywordNotAllowedHere4.pkl", + "errors/moduleWithHighMinPklVersionAndParseErrors.pkl", + "errors/underscore.pkl", + "errors/shebang.pkl", + "notAUnionDefault.pkl", + "multipleDefaults.pkl", + "modules/invalidModule1.pkl", + ) + + private val regexExceptions = + setOf( + Regex(".*/errors/delimiters/.*"), + Regex(".*/errors/parser\\d+\\.pkl"), + Regex(".*/parser/.*"), + ) + } +} diff --git a/pkl-parser/src/test/kotlin/org/pkl/parser/SexpRenderer.kt b/pkl-parser/src/test/kotlin/org/pkl/parser/SexpRenderer.kt index fb9737b3..7d8e2cae 100644 --- a/pkl-parser/src/test/kotlin/org/pkl/parser/SexpRenderer.kt +++ b/pkl-parser/src/test/kotlin/org/pkl/parser/SexpRenderer.kt @@ -82,13 +82,22 @@ class SexpRenderer { } if (decl.extendsOrAmendsDecl !== null) { buf.append('\n') - buf.append(tab) - buf.append("(extendsOrAmendsClause)") + renderExtendsOrAmendsClause(decl.extendsOrAmendsDecl!!) } tab = oldTab buf.append(')') } + fun renderExtendsOrAmendsClause(clause: ExtendsOrAmendsClause) { + buf.append(tab) + buf.append("(extendsOrAmendsClause") + val oldTab = increaseTab() + buf.append('\n') + renderStringConstant(clause.url) + tab = oldTab + buf.append(')') + } + fun renderImport(imp: ImportClause) { buf.append(tab) if (imp.isGlob) { @@ -97,6 +106,8 @@ class SexpRenderer { buf.append("(importClause") } val oldTab = increaseTab() + buf.append('\n') + renderStringConstant(imp.importStr) if (imp.alias !== null) { buf.append('\n') buf.append(tab) @@ -178,6 +189,7 @@ class SexpRenderer { buf.append("(identifier)") val tparList = `typealias`.typeParameterList if (tparList !== null) { + buf.append('\n') renderTypeParameterList(tparList) } buf.append('\n') @@ -244,6 +256,7 @@ class SexpRenderer { buf.append("(identifier)") val tparList = classMethod.typeParameterList if (tparList !== null) { + buf.append('\n') renderTypeParameterList(tparList) } buf.append('\n') @@ -385,11 +398,7 @@ class SexpRenderer { is MultiLineStringLiteralExpr -> renderMultiLineStringLiteral(expr) is ThrowExpr -> renderThrowExpr(expr) is TraceExpr -> renderTraceExpr(expr) - is ImportExpr -> { - buf.append(tab) - val name = if (expr.isGlob) "(importGlobExpr)" else "(importExpr)" - buf.append(name) - } + is ImportExpr -> renderImportExpr(expr) is ReadExpr -> renderReadExpr(expr) is UnqualifiedAccessExpr -> renderUnqualifiedAccessExpr(expr) is QualifiedAccessExpr -> renderQualifiedAccessExpr(expr) @@ -399,7 +408,7 @@ class SexpRenderer { is IfExpr -> renderIfExpr(expr) is LetExpr -> renderLetExpr(expr) is FunctionLiteralExpr -> renderFunctionLiteralExpr(expr) - is ParenthesizedExpr -> renderParenthesisedExpr(expr) + is ParenthesizedExpr -> renderParenthesizedExpr(expr) is NewExpr -> renderNewExpr(expr) is AmendsExpr -> renderAmendsExpr(expr) is NonNullExpr -> renderNonNullExpr(expr) @@ -421,7 +430,7 @@ class SexpRenderer { renderExpr(part.expr) } else { buf.append('\n').append(tab) - buf.append("(stringConstantExpr)") + buf.append("(stringConstant)") } } buf.append(')') @@ -480,6 +489,17 @@ class SexpRenderer { tab = oldTab } + fun renderImportExpr(expr: ImportExpr) { + buf.append(tab) + val name = if (expr.isGlob) "(importGlobExpr" else "(importExpr" + buf.append(name) + val oldTab = increaseTab() + buf.append('\n') + renderStringConstant(expr.importStr) + buf.append(')') + tab = oldTab + } + fun renderUnqualifiedAccessExpr(expr: UnqualifiedAccessExpr) { buf.append(tab) buf.append("(unqualifiedAccessExpr") @@ -517,6 +537,7 @@ class SexpRenderer { buf.append("(superAccessExpr") val oldTab = increaseTab() buf.append('\n') + buf.append(tab) buf.append("(identifier)") if (expr.argumentList !== null) { buf.append('\n') @@ -588,7 +609,7 @@ class SexpRenderer { tab = oldTab } - fun renderParenthesisedExpr(expr: ParenthesizedExpr) { + fun renderParenthesizedExpr(expr: ParenthesizedExpr) { buf.append(tab) buf.append("(parenthesizedExpr") val oldTab = increaseTab() @@ -713,6 +734,11 @@ class SexpRenderer { tab = oldTab } + fun renderStringConstant(str: StringConstant) { + buf.append(tab) + buf.append("(stringConstant)") + } + fun renderTypeAnnotation(typeAnnotation: TypeAnnotation) { buf.append(tab) buf.append("(typeAnnotation") @@ -737,10 +763,7 @@ class SexpRenderer { buf.append(tab) buf.append("(moduleType)") } - is StringConstantType -> { - buf.append(tab) - buf.append("(stringConstantType)") - } + is StringConstantType -> renderStringConstantType(type) is DeclaredType -> renderDeclaredType(type) is ParenthesizedType -> renderParenthesizedType(type) is NullableType -> renderNullableType(type) @@ -750,6 +773,16 @@ class SexpRenderer { } } + fun renderStringConstantType(type: StringConstantType) { + buf.append(tab) + buf.append("(stringConstantType") + val oldTab = increaseTab() + buf.append('\n') + renderStringConstant(type.str) + buf.append(')') + tab = oldTab + } + fun renderDeclaredType(type: DeclaredType) { buf.append(tab) buf.append("(declaredType") @@ -778,7 +811,7 @@ class SexpRenderer { fun renderParenthesizedType(type: ParenthesizedType) { buf.append(tab) - buf.append("(parenthesisedType") + buf.append("(parenthesizedType") val oldTab = increaseTab() buf.append('\n') renderType(type.type) @@ -903,12 +936,11 @@ class SexpRenderer { buf.append(tab) buf.append("(objectMethod") val oldTab = increaseTab() - buf.append('\n') for (mod in method.modifiers) { buf.append('\n') renderModifier(mod) } - buf.append('\n') + buf.append('\n').append(tab) buf.append("(identifier)") val tparList = method.typeParameterList if (tparList !== null) { @@ -1012,19 +1044,20 @@ class SexpRenderer { fun renderTypeParameterList(typeParameterList: TypeParameterList) { buf.append(tab) - buf.append("(TypeParameterList\n") + buf.append("(typeParameterList") val oldTab = increaseTab() for (tpar in typeParameterList.parameters) { buf.append('\n') renderTypeParameter(tpar) } + buf.append(')') tab = oldTab } @Suppress("UNUSED_PARAMETER") fun renderTypeParameter(ignored: TypeParameter?) { buf.append(tab) - buf.append("(TypeParameter\n") + buf.append("(typeParameter\n") val oldTab = increaseTab() buf.append(tab) buf.append("(identifier))") diff --git a/settings.gradle.kts b/settings.gradle.kts index ccac20b8..bf344c26 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -41,6 +41,8 @@ include("pkl-doc") include("pkl-executor") +include("pkl-formatter") + include("pkl-gradle") include("pkl-parser")