move handling of strings to parser (#962)

2026-05-02 21:24:18 +02:00 · 2025-02-19 17:19:48 +01:00
parent ee23a8c3f4
commit 2ffd201172
25 changed files with 408 additions and 543 deletions
--- a/pkl-core/src/test/files/LanguageSnippetTests/output/errors/delimiters/missingEmptyMultiLineStringDelimiter.err
+++ b/pkl-core/src/test/files/LanguageSnippetTests/output/errors/delimiters/missingEmptyMultiLineStringDelimiter.err
@@ -1,5 +1,5 @@
 –– Pkl Error ––
-Unexpected end of file.
+Missing `"""` delimiter.

 x | res2 = 42
             ^
--- a/pkl-core/src/test/files/LanguageSnippetTests/output/errors/delimiters/missingEmptyMultiLineStringDelimiterAtEof.err
+++ b/pkl-core/src/test/files/LanguageSnippetTests/output/errors/delimiters/missingEmptyMultiLineStringDelimiterAtEof.err
@@ -1,5 +1,5 @@
 –– Pkl Error ––
-Unexpected end of file.
+Missing `"""` delimiter.

 x | res1 = """
              ^
--- a/pkl-core/src/test/files/LanguageSnippetTests/output/errors/delimiters/missingEmptyStringDelimiterAtEof.err
+++ b/pkl-core/src/test/files/LanguageSnippetTests/output/errors/delimiters/missingEmptyStringDelimiterAtEof.err
@@ -1,5 +1,5 @@
 –– Pkl Error ––
-Unexpected end of file.
+Missing `"` delimiter.

 x | res1 = "
            ^
--- a/pkl-core/src/test/files/LanguageSnippetTests/output/errors/delimiters/missingMultiLineStringDelimiter.err
+++ b/pkl-core/src/test/files/LanguageSnippetTests/output/errors/delimiters/missingMultiLineStringDelimiter.err
@@ -1,5 +1,5 @@
 –– Pkl Error ––
-Unexpected end of file.
+Missing `"""` delimiter.

 x | 
    ^
--- a/pkl-core/src/test/files/LanguageSnippetTests/output/errors/delimiters/missingRawMultiLineStringDelimiter.err
+++ b/pkl-core/src/test/files/LanguageSnippetTests/output/errors/delimiters/missingRawMultiLineStringDelimiter.err
@@ -1,5 +1,5 @@
 –– Pkl Error ––
-Unexpected end of file.
+Missing `"""#` delimiter.

 x | 
    ^
--- a/pkl-core/src/test/files/LanguageSnippetTests/output/errors/invalidUnicodeEscape.err
+++ b/pkl-core/src/test/files/LanguageSnippetTests/output/errors/invalidUnicodeEscape.err
@@ -1,8 +1,8 @@
 –– Pkl Error ––
 Invalid Unicode escape sequence `\u{12x}`.

+Valid Unicode escape sequences are \u{0} to \u{10FFFF} (1-6 hexadecimal characters).
+
 x | res1 = "\u{12x}"
            ^^^^^^^
-at invalidUnicodeEscape#res1 (file:///$snippetsDir/input/errors/invalidUnicodeEscape.pkl)
-
-Valid Unicode escape sequences are \u{0} to \u{10FFFF} (1-6 hexadecimal characters).
+at invalidUnicodeEscape (file:///$snippetsDir/input/errors/invalidUnicodeEscape.pkl)
--- a/pkl-core/src/test/files/LanguageSnippetTests/output/errors/parser3.err
+++ b/pkl-core/src/test/files/LanguageSnippetTests/output/errors/parser3.err
@@ -1,5 +1,5 @@
 –– Pkl Error ––
-Unexpected end of file.
+Missing `"""` delimiter.

 x | res1 = """some string
                         ^
--- a/pkl-core/src/test/files/LanguageSnippetTests/output/errors/parser4.err
+++ b/pkl-core/src/test/files/LanguageSnippetTests/output/errors/parser4.err
@@ -1,5 +1,5 @@
 –– Pkl Error ––
-Unexpected end of file.
+Missing `"""` delimiter.

 x | res2 = 2
            ^
--- a/pkl-core/src/test/files/LanguageSnippetTests/output/errors/parser6.err
+++ b/pkl-core/src/test/files/LanguageSnippetTests/output/errors/parser6.err
@@ -1,5 +1,5 @@
 –– Pkl Error ––
-Unexpected end of file.
+Missing `"""` delimiter.

 x | res1 = """some string"
                          ^
--- a/pkl-core/src/test/files/LanguageSnippetTests/output/errors/parser8.err
+++ b/pkl-core/src/test/files/LanguageSnippetTests/output/errors/parser8.err
@@ -3,4 +3,4 @@ Line must match or exceed indentation of the String's last line.

 x | mismatched indent
    ^^^^^^^^^^^^^^^^^
-at parser8#res1 (file:///$snippetsDir/input/errors/parser8.pkl)
+at parser8 (file:///$snippetsDir/input/errors/parser8.pkl)
--- a/pkl-core/src/test/files/LanguageSnippetTests/output/errors/parser9.err
+++ b/pkl-core/src/test/files/LanguageSnippetTests/output/errors/parser9.err
@@ -3,4 +3,4 @@ Line must match or exceed indentation of the String's last line.

 x | mismatched indent
    ^^^^^^^^^^^^^^^^^
-at parser9#res1 (file:///$snippetsDir/input/errors/parser9.pkl)
+at parser9 (file:///$snippetsDir/input/errors/parser9.pkl)
--- a/pkl-core/src/test/kotlin/org/pkl/core/parser/ANTLRSexpRenderer.kt
+++ b/pkl-core/src/test/kotlin/org/pkl/core/parser/ANTLRSexpRenderer.kt
@@ -791,7 +791,7 @@ class ANTLRSexpRenderer {

  fun renderSingleLineStringExpr(expr: SingleLineStringLiteralContext) {
    buf.append(tab)
-    buf.append("(interpolatedStringExpr")
+    buf.append("(singleLineStringLiteralExpr")
    val oldTab = increaseTab()
    for (part in expr.singleLineStringPart()) {
      if (part.expr() != null) {
@@ -808,15 +808,14 @@ class ANTLRSexpRenderer {

  fun renderMultiLineStringExpr(expr: MultiLineStringLiteralContext) {
    buf.append(tab)
-    buf.append("(interpolatedMultiStringExpr")
+    buf.append("(multiLineStringLiteralExpr")
    val oldTab = increaseTab()
+    // render only interpolated expressions because
+    // the new parser parses string differently
    for (part in expr.multiLineStringPart()) {
      if (part.expr() != null) {
        buf.append('\n')
        renderExpr(part.expr())
-      } else {
-        buf.append('\n').append(tab)
-        buf.append("(stringConstantExpr)")
      }
    }
    buf.append(')')
--- a/pkl-core/src/test/kotlin/org/pkl/core/parser/ParserComparisonTest.kt
+++ b/pkl-core/src/test/kotlin/org/pkl/core/parser/ParserComparisonTest.kt
@@ -42,10 +42,18 @@ class ParserComparisonTest : ParserComparisonTestInterface {

    compare(
      """
-      prop = ${"\"\"\""}\(bar)${"\"\"\""}
-      prop2 = ${"\"\"\""}foo \(bar)${"\"\"\""}
-      prop3 = ${"\"\"\""}\(bar) foo${"\"\"\""}
-      prop4 = ${"\"\"\""}foo \(bar + baz) foo${"\"\"\""}
+      prop = ""${'"'}
+        \(bar)
+        ""${'"'}
+      prop2 = ""${'"'}
+        foo \(bar)
+        ""${'"'}
+      prop3 = ""${'"'}
+        \(bar) foo
+        ""${'"'}
+      prop4 = ""${'"'}
+        foo \(bar + baz) foo
+        ""${'"'}
      """
        .trimIndent()
    )
--- a/pkl-core/src/test/kotlin/org/pkl/core/parser/SexpRenderer.kt
+++ b/pkl-core/src/test/kotlin/org/pkl/core/parser/SexpRenderer.kt
@@ -419,7 +419,7 @@ class SexpRenderer {

  fun renderSingleLineStringLiteral(expr: SingleLineStringLiteralExpr) {
    buf.append(tab)
-    buf.append("(interpolatedStringExpr")
+    buf.append("(singleLineStringLiteralExpr")
    val oldTab = increaseTab()
    for (part in expr.parts) {
      if (part is StringPart.StringInterpolation) {
@@ -436,15 +436,14 @@ class SexpRenderer {

  fun renderMultiLineStringLiteral(expr: MultiLineStringLiteralExpr) {
    buf.append(tab)
-    buf.append("(interpolatedMultiStringExpr")
+    buf.append("(multiLineStringLiteralExpr")
    val oldTab = increaseTab()
+    // render only interpolated expressions because
+    // the new parser parses string differently
    for (part in expr.parts) {
      if (part is StringPart.StringInterpolation) {
        buf.append('\n')
        renderExpr(part.expr)
-      } else {
-        buf.append('\n').append(tab)
-        buf.append("(stringConstantExpr)")
      }
    }
    buf.append(')')
--- a/pkl-core/src/test/kotlin/org/pkl/core/parser/SpanComparison.kt
+++ b/pkl-core/src/test/kotlin/org/pkl/core/parser/SpanComparison.kt
@@ -302,10 +302,11 @@ class SpanComparison(val path: String, private val softly: SoftAssertions) {
        }
      }
      is MultiLineStringLiteralExpr -> {
-        node.parts.zip((ctx as MultiLineStringLiteralContext).multiLineStringPart()).forEach {
-          (s1, s2) ->
-          compareSpan(s1, s2)
-        }
+        // only compare interpolated expressions
+        val exprs = node.parts.filterIsInstance<StringPart.StringInterpolation>()
+        val antlrExprs =
+          (ctx as MultiLineStringLiteralContext).multiLineStringPart().mapNotNull { it.expr() }
+        exprs.zip(antlrExprs).forEach { (s1, s2) -> compareExpr(s1.expr, s2) }
      }
      is ThrowExpr -> compareExpr(node.expr, (ctx as ThrowExprContext).expr())
      is TraceExpr -> compareExpr(node.expr, (ctx as TraceExprContext).expr())