mirror of
https://github.com/apple/pkl.git
synced 2026-05-25 16:19:20 +02:00
Fix CRLF handling in line continuation escapes (#1564)
This commit is contained in:
@@ -461,11 +461,17 @@ public final class Lexer {
|
|||||||
}
|
}
|
||||||
case 'u' -> lexUnicodeEscape();
|
case 'u' -> lexUnicodeEscape();
|
||||||
case '\n' -> Token.STRING_ESCAPE_CONTINUATION;
|
case '\n' -> Token.STRING_ESCAPE_CONTINUATION;
|
||||||
|
case '\r' -> {
|
||||||
|
if (lookahead == '\n') {
|
||||||
|
nextChar();
|
||||||
|
}
|
||||||
|
yield Token.STRING_ESCAPE_CONTINUATION;
|
||||||
|
}
|
||||||
case ' ', '\t' -> {
|
case ' ', '\t' -> {
|
||||||
var c = cursor;
|
var c = cursor;
|
||||||
var next = nextChar();
|
var next = nextChar();
|
||||||
while (next == ' ' || next == '\t') next = nextChar();
|
while (next == ' ' || next == '\t') next = nextChar();
|
||||||
if (next == '\n')
|
if (next == '\n' || next == '\r')
|
||||||
throw lexError(
|
throw lexError(
|
||||||
ErrorMessages.create("invalidLineContinuationEscapeSequenceWhitespace"),
|
ErrorMessages.create("invalidLineContinuationEscapeSequenceWhitespace"),
|
||||||
c - 2,
|
c - 2,
|
||||||
|
|||||||
@@ -1,5 +1,5 @@
|
|||||||
/*
|
/*
|
||||||
* Copyright © 2025 Apple Inc. and the Pkl project authors. All rights reserved.
|
* Copyright © 2025-2026 Apple Inc. and the Pkl project authors. All rights reserved.
|
||||||
*
|
*
|
||||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
* you may not use this file except in compliance with the License.
|
* you may not use this file except in compliance with the License.
|
||||||
@@ -63,6 +63,56 @@ class LexerTest {
|
|||||||
assertThat(thrown).hasMessageContaining("Invalid identifier")
|
assertThat(thrown).hasMessageContaining("Invalid identifier")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
fun lineContinuationWithCRLF() {
|
||||||
|
// \r\n line endings must be handled the same as \n for line continuations
|
||||||
|
val input = "x = \"\"\"\n hello \\\r\n world\r\n \"\"\""
|
||||||
|
val lexer = Lexer(input)
|
||||||
|
assertThat(lexer.next()).isEqualTo(Token.IDENTIFIER) // x
|
||||||
|
assertThat(lexer.next()).isEqualTo(Token.ASSIGN)
|
||||||
|
assertThat(lexer.next()).isEqualTo(Token.STRING_MULTI_START) // """
|
||||||
|
assertThat(lexer.next()).isEqualTo(Token.STRING_NEWLINE)
|
||||||
|
assertThat(lexer.next()).isEqualTo(Token.STRING_PART) // " hello "
|
||||||
|
assertThat(lexer.next()).isEqualTo(Token.STRING_ESCAPE_CONTINUATION) // \<CRLF> consumed
|
||||||
|
assertThat(lexer.next()).isEqualTo(Token.STRING_PART) // " world"
|
||||||
|
assertThat(lexer.next()).isEqualTo(Token.STRING_NEWLINE)
|
||||||
|
assertThat(lexer.next()).isEqualTo(Token.STRING_PART) // " "
|
||||||
|
assertThat(lexer.next()).isEqualTo(Token.STRING_END) // """
|
||||||
|
assertThat(lexer.next()).isEqualTo(Token.EOF)
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
fun lineContinuationWithCR() {
|
||||||
|
// bare \r should also work as a line continuation
|
||||||
|
val input = "x = \"\"\"\n hello \\\r world\n \"\"\""
|
||||||
|
val lexer = Lexer(input)
|
||||||
|
assertThat(lexer.next()).isEqualTo(Token.IDENTIFIER)
|
||||||
|
assertThat(lexer.next()).isEqualTo(Token.ASSIGN)
|
||||||
|
assertThat(lexer.next()).isEqualTo(Token.STRING_MULTI_START)
|
||||||
|
assertThat(lexer.next()).isEqualTo(Token.STRING_NEWLINE)
|
||||||
|
assertThat(lexer.next()).isEqualTo(Token.STRING_PART)
|
||||||
|
assertThat(lexer.next()).isEqualTo(Token.STRING_ESCAPE_CONTINUATION) // \<CR> consumed
|
||||||
|
assertThat(lexer.next()).isEqualTo(Token.STRING_PART)
|
||||||
|
assertThat(lexer.next()).isEqualTo(Token.STRING_NEWLINE)
|
||||||
|
assertThat(lexer.next()).isEqualTo(Token.STRING_PART)
|
||||||
|
assertThat(lexer.next()).isEqualTo(Token.STRING_END)
|
||||||
|
assertThat(lexer.next()).isEqualTo(Token.EOF)
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
fun lineContinuationWhitespaceErrorWithCRLF() {
|
||||||
|
// whitespace between \ and \r\n should give the same error as \ and \n
|
||||||
|
val input = "x = \"\"\"\n hello \\ \r\n world\n \"\"\""
|
||||||
|
val thrown =
|
||||||
|
assertThrows<ParserError> {
|
||||||
|
val lexer = Lexer(input)
|
||||||
|
while (lexer.next() != Token.EOF) {
|
||||||
|
/* consume all tokens */
|
||||||
|
}
|
||||||
|
}
|
||||||
|
assertThat(thrown.message).contains("Whitespace")
|
||||||
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
fun acceptsAllUnicodeCodepointsInComments() {
|
fun acceptsAllUnicodeCodepointsInComments() {
|
||||||
// Test valid Unicode codepoints can appear literally
|
// Test valid Unicode codepoints can appear literally
|
||||||
|
|||||||
Reference in New Issue
Block a user