From e07abb731115e2f023d1f73507b5f208fcaa01b6 Mon Sep 17 00:00:00 2001 From: Jen Basch Date: Tue, 21 Apr 2026 10:29:52 -0700 Subject: [PATCH] SPICE-0028: Add support for multi-line string line continuations (#1507) SPICE: https://github.com/apple/pkl-evolution/pull/31 --- .../language-reference/pages/index.adoc | 20 ++++++++++++++++-- .../org/pkl/core/util/SyntaxHighlighter.java | 4 +++- .../basic/stringMultilineContinuation.pkl | 21 +++++++++++++++++++ .../input/errors/invalidCharacterEscape2.pkl | 1 + .../input/errors/parser19.pkl | 5 +++++ .../input/errors/parser20.pkl | 2 ++ .../input/errors/parser21.pkl | 5 +++++ .../basic/stringMultilineContinuation.pcf | 7 +++++++ .../output/errors/invalidCharacterEscape.err | 2 +- .../output/errors/invalidCharacterEscape2.err | 8 +++++++ .../output/errors/parser19.err | 6 ++++++ .../output/errors/parser20.err | 8 +++++++ .../output/errors/parser21.err | 8 +++++++ .../main/java/org/pkl/formatter/Builder.java | 9 +++++++- .../input/multi-line-strings.pkl | 20 ++++++++++++++++++ .../output/multi-line-strings.pkl | 19 +++++++++++++++++ .../org/pkl/parser/GenericParserImpl.java | 7 ++++++- .../src/main/java/org/pkl/parser/Lexer.java | 16 ++++++++++++++ .../main/java/org/pkl/parser/ParserImpl.java | 7 ++++++- .../src/main/java/org/pkl/parser/Token.java | 1 + .../pkl/parser/syntax/generic/NodeType.java | 3 ++- .../org/pkl/parser/errorMessages.properties | 12 ++++++++++- .../org/pkl/parser/GenericSexpRenderer.kt | 1 + .../org/pkl/parser/ParserComparisonTest.kt | 3 ++- 24 files changed, 185 insertions(+), 10 deletions(-) create mode 100644 pkl-core/src/test/files/LanguageSnippetTests/input/basic/stringMultilineContinuation.pkl create mode 100644 pkl-core/src/test/files/LanguageSnippetTests/input/errors/invalidCharacterEscape2.pkl create mode 100644 pkl-core/src/test/files/LanguageSnippetTests/input/errors/parser19.pkl create mode 100644 pkl-core/src/test/files/LanguageSnippetTests/input/errors/parser20.pkl create mode 100644 pkl-core/src/test/files/LanguageSnippetTests/input/errors/parser21.pkl create mode 100644 pkl-core/src/test/files/LanguageSnippetTests/output/basic/stringMultilineContinuation.pcf create mode 100644 pkl-core/src/test/files/LanguageSnippetTests/output/errors/invalidCharacterEscape2.err create mode 100644 pkl-core/src/test/files/LanguageSnippetTests/output/errors/parser19.err create mode 100644 pkl-core/src/test/files/LanguageSnippetTests/output/errors/parser20.err create mode 100644 pkl-core/src/test/files/LanguageSnippetTests/output/errors/parser21.err diff --git a/docs/modules/language-reference/pages/index.adoc b/docs/modules/language-reference/pages/index.adoc index 60a94e68..2f2ff5b3 100644 --- a/docs/modules/language-reference/pages/index.adoc +++ b/docs/modules/language-reference/pages/index.adoc @@ -255,8 +255,7 @@ String literals are enclosed in double quotes: "Hello, World!" ---- -TIP: Except for a few minor differences footnote:[Pkl's string literals have fewer character escape sequences, -have stricter rules for line indentation in multiline strings, and do not have a line continuation character.], +TIP: Except for a few minor differences footnote:[Pkl's string literals have fewer character escape sequences and stricter rules for line indentation in multiline strings.], String literals have the same syntax and semantics as in Swift 5. Learn one of them, know both of them! Inside a string literal, the following character escape sequences have special meaning: @@ -362,6 +361,23 @@ str = """ """ ---- +To prevent line breaks from becoming part of the string's value, use a backslash (`\`) to end those lines. + +[source%tested,{pkl}] +---- +str = """ + Although the Dodo is extinct, \ + the species will be remembered. + """ +---- + +This multiline string is equivalent to the following single-line string: + +[source%parsed,{pkl-expr}] +---- +"Although the Dodo is extinct, the species will be remembered." +---- + [[custom-string-delimiters]] === Custom String Delimiters diff --git a/pkl-core/src/main/java/org/pkl/core/util/SyntaxHighlighter.java b/pkl-core/src/main/java/org/pkl/core/util/SyntaxHighlighter.java index 88786abb..28f60b72 100644 --- a/pkl-core/src/main/java/org/pkl/core/util/SyntaxHighlighter.java +++ b/pkl-core/src/main/java/org/pkl/core/util/SyntaxHighlighter.java @@ -18,6 +18,7 @@ package org.pkl.core.util; import static org.pkl.parser.Token.FALSE; import static org.pkl.parser.Token.NULL; import static org.pkl.parser.Token.STRING_ESCAPE_BACKSLASH; +import static org.pkl.parser.Token.STRING_ESCAPE_CONTINUATION; import static org.pkl.parser.Token.STRING_ESCAPE_NEWLINE; import static org.pkl.parser.Token.STRING_ESCAPE_QUOTE; import static org.pkl.parser.Token.STRING_ESCAPE_RETURN; @@ -41,7 +42,8 @@ public final class SyntaxHighlighter { STRING_ESCAPE_RETURN, STRING_ESCAPE_QUOTE, STRING_ESCAPE_BACKSLASH, - STRING_ESCAPE_UNICODE); + STRING_ESCAPE_UNICODE, + STRING_ESCAPE_CONTINUATION); private static final EnumSet constant = EnumSet.of(TRUE, FALSE, NULL); diff --git a/pkl-core/src/test/files/LanguageSnippetTests/input/basic/stringMultilineContinuation.pkl b/pkl-core/src/test/files/LanguageSnippetTests/input/basic/stringMultilineContinuation.pkl new file mode 100644 index 00000000..e0c08471 --- /dev/null +++ b/pkl-core/src/test/files/LanguageSnippetTests/input/basic/stringMultilineContinuation.pkl @@ -0,0 +1,21 @@ +amends "../snippetTest.pkl" + +examples { + ["string continuation"] { + """ + hello \ + world + """ + + #""" + hello \# + world + """# + + """ + hello \ + \ + world + """ + } +} diff --git a/pkl-core/src/test/files/LanguageSnippetTests/input/errors/invalidCharacterEscape2.pkl b/pkl-core/src/test/files/LanguageSnippetTests/input/errors/invalidCharacterEscape2.pkl new file mode 100644 index 00000000..354894db --- /dev/null +++ b/pkl-core/src/test/files/LanguageSnippetTests/input/errors/invalidCharacterEscape2.pkl @@ -0,0 +1 @@ +res1 = "xxx\ xxx" diff --git a/pkl-core/src/test/files/LanguageSnippetTests/input/errors/parser19.pkl b/pkl-core/src/test/files/LanguageSnippetTests/input/errors/parser19.pkl new file mode 100644 index 00000000..4adf5385 --- /dev/null +++ b/pkl-core/src/test/files/LanguageSnippetTests/input/errors/parser19.pkl @@ -0,0 +1,5 @@ +foo = """ + hello \ +\ + world + """ diff --git a/pkl-core/src/test/files/LanguageSnippetTests/input/errors/parser20.pkl b/pkl-core/src/test/files/LanguageSnippetTests/input/errors/parser20.pkl new file mode 100644 index 00000000..85ce04fc --- /dev/null +++ b/pkl-core/src/test/files/LanguageSnippetTests/input/errors/parser20.pkl @@ -0,0 +1,2 @@ +foo = "hello \ +world" diff --git a/pkl-core/src/test/files/LanguageSnippetTests/input/errors/parser21.pkl b/pkl-core/src/test/files/LanguageSnippetTests/input/errors/parser21.pkl new file mode 100644 index 00000000..48b2dc06 --- /dev/null +++ b/pkl-core/src/test/files/LanguageSnippetTests/input/errors/parser21.pkl @@ -0,0 +1,5 @@ +foo = + """ + hello \ + world + """ diff --git a/pkl-core/src/test/files/LanguageSnippetTests/output/basic/stringMultilineContinuation.pcf b/pkl-core/src/test/files/LanguageSnippetTests/output/basic/stringMultilineContinuation.pcf new file mode 100644 index 00000000..cdb959a3 --- /dev/null +++ b/pkl-core/src/test/files/LanguageSnippetTests/output/basic/stringMultilineContinuation.pcf @@ -0,0 +1,7 @@ +examples { + ["string continuation"] { + "hello world" + "hello world" + "hello world" + } +} diff --git a/pkl-core/src/test/files/LanguageSnippetTests/output/errors/invalidCharacterEscape.err b/pkl-core/src/test/files/LanguageSnippetTests/output/errors/invalidCharacterEscape.err index 076e2c25..14b8b265 100644 --- a/pkl-core/src/test/files/LanguageSnippetTests/output/errors/invalidCharacterEscape.err +++ b/pkl-core/src/test/files/LanguageSnippetTests/output/errors/invalidCharacterEscape.err @@ -1,7 +1,7 @@ –– Pkl Error –– Invalid character escape sequence `\a`. -Valid character escape sequences are: \n \r \t \" \\ +Valid character escape sequences are: \n \r \t \" \\ \ x | res1 = "xxx\axxx" ^^ diff --git a/pkl-core/src/test/files/LanguageSnippetTests/output/errors/invalidCharacterEscape2.err b/pkl-core/src/test/files/LanguageSnippetTests/output/errors/invalidCharacterEscape2.err new file mode 100644 index 00000000..641b014c --- /dev/null +++ b/pkl-core/src/test/files/LanguageSnippetTests/output/errors/invalidCharacterEscape2.err @@ -0,0 +1,8 @@ +–– Pkl Error –– +Invalid character escape sequence `\ `. + +Valid character escape sequences are: \n \r \t \" \\ \ + +x | res1 = "xxx\ xxx" + ^^ +at invalidCharacterEscape2 (file:///$snippetsDir/input/errors/invalidCharacterEscape2.pkl) diff --git a/pkl-core/src/test/files/LanguageSnippetTests/output/errors/parser19.err b/pkl-core/src/test/files/LanguageSnippetTests/output/errors/parser19.err new file mode 100644 index 00000000..1336f154 --- /dev/null +++ b/pkl-core/src/test/files/LanguageSnippetTests/output/errors/parser19.err @@ -0,0 +1,6 @@ +–– Pkl Error –– +Line must match or exceed indentation of the String's last line. + +x | \ + ^ +at parser19 (file:///$snippetsDir/input/errors/parser19.pkl) diff --git a/pkl-core/src/test/files/LanguageSnippetTests/output/errors/parser20.err b/pkl-core/src/test/files/LanguageSnippetTests/output/errors/parser20.err new file mode 100644 index 00000000..b6c89dfc --- /dev/null +++ b/pkl-core/src/test/files/LanguageSnippetTests/output/errors/parser20.err @@ -0,0 +1,8 @@ +–– Pkl Error –– +Invalid line continuation escape sequence. + +Line continuations are only allowed in multi-line strings. + +x | foo = "hello \ + ^^ +at parser20 (file:///$snippetsDir/input/errors/parser20.pkl) diff --git a/pkl-core/src/test/files/LanguageSnippetTests/output/errors/parser21.err b/pkl-core/src/test/files/LanguageSnippetTests/output/errors/parser21.err new file mode 100644 index 00000000..3bd7c99a --- /dev/null +++ b/pkl-core/src/test/files/LanguageSnippetTests/output/errors/parser21.err @@ -0,0 +1,8 @@ +–– Pkl Error –– +Invalid line continuation escape sequence. + +Whitespace between the continuation escape and following newline is not allowed. + +x | hello \ + ^^^^^ +at parser21 (file:///$snippetsDir/input/errors/parser21.pkl) diff --git a/pkl-formatter/src/main/java/org/pkl/formatter/Builder.java b/pkl-formatter/src/main/java/org/pkl/formatter/Builder.java index 7642754d..a2c5471a 100644 --- a/pkl-formatter/src/main/java/org/pkl/formatter/Builder.java +++ b/pkl-formatter/src/main/java/org/pkl/formatter/Builder.java @@ -75,6 +75,11 @@ final class Builder { OPERATOR -> new Text(node.text(source)); case STRING_NEWLINE -> mustForceLine(); + case STRING_CONTINUATION -> { + var escape = node.text(source); + yield new Nodes( + List.of(new Text(escape.substring(0, escape.length() - 1)), mustForceLine())); + } case MODULE_DECLARATION -> formatModuleDeclaration(node); case MODULE_DEFINITION -> formatModuleDefinition(node); case SINGLE_LINE_STRING_LITERAL_EXPR -> formatSingleLineString(node); @@ -922,7 +927,9 @@ final class Builder { if (elem.type == NodeType.TERMINAL && text(elem).endsWith("(")) { isInStringInterpolation = true; } - result.add(format(elem)); + var formatted = format(elem); + if (formatted instanceof Nodes formattedNodes) result.addAll(formattedNodes.nodes()); + else result.add(formatted); prev = elem; } return result; diff --git a/pkl-formatter/src/test/files/FormatterSnippetTests/input/multi-line-strings.pkl b/pkl-formatter/src/test/files/FormatterSnippetTests/input/multi-line-strings.pkl index feef82f9..6068571f 100644 --- a/pkl-formatter/src/test/files/FormatterSnippetTests/input/multi-line-strings.pkl +++ b/pkl-formatter/src/test/files/FormatterSnippetTests/input/multi-line-strings.pkl @@ -48,6 +48,26 @@ quux { """ } +// line continuations +corge { + + """ + hello \ + world + """ + +#""" +hello \# +world +"""# + +""" +hello \ +\ +world +""" +} + obj { data { ["bar"] = """ diff --git a/pkl-formatter/src/test/files/FormatterSnippetTests/output/multi-line-strings.pkl b/pkl-formatter/src/test/files/FormatterSnippetTests/output/multi-line-strings.pkl index c55737b6..5e3d5629 100644 --- a/pkl-formatter/src/test/files/FormatterSnippetTests/output/multi-line-strings.pkl +++ b/pkl-formatter/src/test/files/FormatterSnippetTests/output/multi-line-strings.pkl @@ -52,6 +52,25 @@ quux { """ } +// line continuations +corge { + """ + hello \ + world + """ + + #""" + hello \# + world + """# + + """ + hello \ + \ + world + """ +} + obj { data { ["bar"] = diff --git a/pkl-parser/src/main/java/org/pkl/parser/GenericParserImpl.java b/pkl-parser/src/main/java/org/pkl/parser/GenericParserImpl.java index 98f71413..f00e0d46 100644 --- a/pkl-parser/src/main/java/org/pkl/parser/GenericParserImpl.java +++ b/pkl-parser/src/main/java/org/pkl/parser/GenericParserImpl.java @@ -978,6 +978,8 @@ class GenericParserImpl { STRING_ESCAPE_RETURN, STRING_ESCAPE_UNICODE -> children.add(make(NodeType.STRING_ESCAPE, next().span)); + case STRING_ESCAPE_CONTINUATION -> + throw parserError("invalidLineContinuationEscapeSequence"); case INTERPOLATION_START -> { children.add(makeTerminal(next())); ff(children); @@ -1011,6 +1013,8 @@ class GenericParserImpl { } } case STRING_NEWLINE -> children.add(make(NodeType.STRING_NEWLINE, next().span)); + case STRING_ESCAPE_CONTINUATION -> + children.add(make(NodeType.STRING_CONTINUATION, next().span)); case STRING_ESCAPE_NEWLINE, STRING_ESCAPE_TAB, STRING_ESCAPE_QUOTE, @@ -1060,7 +1064,8 @@ class GenericParserImpl { throw parserError(ErrorMessages.create("stringIndentationMustMatchLastLine"), child.span); } } - previousNewline = child.type == NodeType.STRING_NEWLINE; + previousNewline = + child.type == NodeType.STRING_NEWLINE || child.type == NodeType.STRING_CONTINUATION; } } diff --git a/pkl-parser/src/main/java/org/pkl/parser/Lexer.java b/pkl-parser/src/main/java/org/pkl/parser/Lexer.java index 08909471..8fe7fe88 100644 --- a/pkl-parser/src/main/java/org/pkl/parser/Lexer.java +++ b/pkl-parser/src/main/java/org/pkl/parser/Lexer.java @@ -460,6 +460,22 @@ public final class Lexer { yield Token.INTERPOLATION_START; } case 'u' -> lexUnicodeEscape(); + case '\n' -> Token.STRING_ESCAPE_CONTINUATION; + case ' ', '\t' -> { + var c = cursor; + var next = nextChar(); + while (next == ' ' || next == '\t') next = nextChar(); + if (next == '\n') + throw lexError( + ErrorMessages.create("invalidLineContinuationEscapeSequenceWhitespace"), + c - 2, + cursor - c + 2); + + throw lexError( + ErrorMessages.create("invalidCharacterEscapeSequence", "\\" + (char) ch, "\\"), + c - 2, + 2); + } default -> throw lexError( ErrorMessages.create("invalidCharacterEscapeSequence", "\\" + (char) ch, "\\"), diff --git a/pkl-parser/src/main/java/org/pkl/parser/ParserImpl.java b/pkl-parser/src/main/java/org/pkl/parser/ParserImpl.java index cdbd062c..28feb5e7 100644 --- a/pkl-parser/src/main/java/org/pkl/parser/ParserImpl.java +++ b/pkl-parser/src/main/java/org/pkl/parser/ParserImpl.java @@ -1131,6 +1131,8 @@ final class ParserImpl { end = tk.span; builder.append(parseUnicodeEscape(tk)); } + case STRING_ESCAPE_CONTINUATION -> + throw parserError("invalidLineContinuationEscapeSequence"); case INTERPOLATION_START -> { var istart = next().span; if (!builder.isEmpty()) { @@ -1167,7 +1169,8 @@ final class ParserImpl { STRING_ESCAPE_QUOTE, STRING_ESCAPE_BACKSLASH, STRING_ESCAPE_RETURN, - STRING_ESCAPE_UNICODE -> + STRING_ESCAPE_UNICODE, + STRING_ESCAPE_CONTINUATION -> stringTokens.add(new TempNode(next(), null)); case INTERPOLATION_START -> { var istart = next(); @@ -1236,6 +1239,7 @@ final class ParserImpl { builder.append('\n'); isNewLine = true; } + case STRING_ESCAPE_CONTINUATION -> isNewLine = true; case STRING_PART -> { var text = token.text(lexer); if (isNewLine) { @@ -1642,6 +1646,7 @@ final class ParserImpl { case STRING_ESCAPE_BACKSLASH -> "\\"; case STRING_ESCAPE_TAB -> "\t"; case STRING_ESCAPE_RETURN -> "\r"; + case STRING_ESCAPE_CONTINUATION -> ""; case STRING_ESCAPE_UNICODE -> parseUnicodeEscape(tk); default -> throw new RuntimeException("Unreacheable code"); }; diff --git a/pkl-parser/src/main/java/org/pkl/parser/Token.java b/pkl-parser/src/main/java/org/pkl/parser/Token.java index d5949e7e..32bbbf05 100644 --- a/pkl-parser/src/main/java/org/pkl/parser/Token.java +++ b/pkl-parser/src/main/java/org/pkl/parser/Token.java @@ -129,6 +129,7 @@ public enum Token { STRING_ESCAPE_QUOTE, STRING_ESCAPE_BACKSLASH, STRING_ESCAPE_UNICODE, + STRING_ESCAPE_CONTINUATION, STRING_END, STRING_PART; diff --git a/pkl-parser/src/main/java/org/pkl/parser/syntax/generic/NodeType.java b/pkl-parser/src/main/java/org/pkl/parser/syntax/generic/NodeType.java index 72c13610..34d4d1bf 100644 --- a/pkl-parser/src/main/java/org/pkl/parser/syntax/generic/NodeType.java +++ b/pkl-parser/src/main/java/org/pkl/parser/syntax/generic/NodeType.java @@ -1,5 +1,5 @@ /* - * Copyright © 2025 Apple Inc. and the Pkl project authors. All rights reserved. + * Copyright © 2025-2026 Apple Inc. and the Pkl project authors. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -70,6 +70,7 @@ public enum NodeType { STRING_CHARS, OPERATOR, STRING_NEWLINE, + STRING_CONTINUATION, STRING_ESCAPE, // members diff --git a/pkl-parser/src/main/resources/org/pkl/parser/errorMessages.properties b/pkl-parser/src/main/resources/org/pkl/parser/errorMessages.properties index bb577657..fa1a16d9 100644 --- a/pkl-parser/src/main/resources/org/pkl/parser/errorMessages.properties +++ b/pkl-parser/src/main/resources/org/pkl/parser/errorMessages.properties @@ -44,13 +44,23 @@ The separator character (`_`) cannot follow `0x`, `0b`, `.`, `e`, or 'E' in a nu invalidCharacterEscapeSequence=\ Invalid character escape sequence `{0}`.\n\ \n\ -Valid character escape sequences are: {1}n {1}r {1}t {1}" {1}\\ +Valid character escape sequences are: {1}n {1}r {1}t {1}" {1}\\ {1} invalidUnicodeEscapeSequence=\ Invalid Unicode escape sequence `{0}`.\n\ \n\ Valid Unicode escape sequences are {1}'{'0'}' to {1}'{'10FFFF'}' (1-6 hexadecimal characters). +invalidLineContinuationEscapeSequence=\ +Invalid line continuation escape sequence.\n\ +\n\ +Line continuations are only allowed in multi-line strings. + +invalidLineContinuationEscapeSequenceWhitespace=\ +Invalid line continuation escape sequence.\n\ +\n\ +Whitespace between the continuation escape and following newline is not allowed. + missingDelimiter=\ Missing `{0}` delimiter. diff --git a/pkl-parser/src/test/kotlin/org/pkl/parser/GenericSexpRenderer.kt b/pkl-parser/src/test/kotlin/org/pkl/parser/GenericSexpRenderer.kt index 0a1f6c55..4a377094 100644 --- a/pkl-parser/src/test/kotlin/org/pkl/parser/GenericSexpRenderer.kt +++ b/pkl-parser/src/test/kotlin/org/pkl/parser/GenericSexpRenderer.kt @@ -197,6 +197,7 @@ class GenericSexpRenderer(code: String) { NodeType.TERMINAL, NodeType.OPERATOR, NodeType.STRING_NEWLINE, + NodeType.STRING_CONTINUATION, ) private val UNPACK_CHILDREN = diff --git a/pkl-parser/src/test/kotlin/org/pkl/parser/ParserComparisonTest.kt b/pkl-parser/src/test/kotlin/org/pkl/parser/ParserComparisonTest.kt index 050e1a3c..a49e90ab 100644 --- a/pkl-parser/src/test/kotlin/org/pkl/parser/ParserComparisonTest.kt +++ b/pkl-parser/src/test/kotlin/org/pkl/parser/ParserComparisonTest.kt @@ -1,5 +1,5 @@ /* - * Copyright © 2024-2025 Apple Inc. and the Pkl project authors. All rights reserved. + * Copyright © 2024-2026 Apple Inc. and the Pkl project authors. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -97,6 +97,7 @@ class ParserComparisonTest { "errors/parser18.pkl", "errors/nested1.pkl", "errors/invalidCharacterEscape.pkl", + "errors/invalidCharacterEscape2.pkl", "errors/invalidUnicodeEscape.pkl", "errors/unterminatedUnicodeEscape.pkl", "errors/keywordNotAllowedHere1.pkl",