Fix block comment at EOF without trailing newline (#1828)

This commit is contained in:
Sergio Salvi
2026-08-21 15:46:37 -07:00
committed by GitHub
parent 6551a59f9e
commit 7404f7d160
2 changed files with 48 additions and 2 deletions
@@ -625,11 +625,11 @@ public final class Lexer {
while (lookahead != EOF) {
if (prev == '*' && lookahead == '/') {
nextChar();
break;
return;
}
prev = nextChar();
}
if (lookahead == EOF) throw unexpectedEndOfFile();
throw unexpectedEndOfFile();
}
private void lexHexNumber() {
@@ -113,6 +113,52 @@ class LexerTest {
assertThat(thrown.message).contains("Whitespace")
}
@Test
fun `block comment at EOF without trailing newline`() {
// A block comment that is the last thing in the file, with no trailing
// newline, must lex correctly and not be mistaken for an unterminated comment.
val lexer = Lexer("/*\nbar\n*/")
assertThat(lexer.next()).isEqualTo(Token.BLOCK_COMMENT)
assertThat(lexer.next()).isEqualTo(Token.EOF)
}
@Test
fun `block comment after code at EOF without trailing newline`() {
val lexer = Lexer("foo = 123\n/*\nbar\n*/")
assertThat(lexer.next()).isEqualTo(Token.IDENTIFIER)
assertThat(lexer.next()).isEqualTo(Token.ASSIGN)
assertThat(lexer.next()).isEqualTo(Token.INT)
assertThat(lexer.next()).isEqualTo(Token.BLOCK_COMMENT)
assertThat(lexer.next()).isEqualTo(Token.EOF)
}
@Test
fun `empty block comment at EOF without trailing newline`() {
val lexer = Lexer("/**/")
assertThat(lexer.next()).isEqualTo(Token.BLOCK_COMMENT)
assertThat(lexer.next()).isEqualTo(Token.EOF)
}
@Test
fun `plain code at EOF without trailing newline`() {
// Documents that code whose final line has no trailing newline lexes
// cleanly and terminates with EOF rather than erroring.
val lexer = Lexer("foo = 123")
assertThat(lexer.next()).isEqualTo(Token.IDENTIFIER)
assertThat(lexer.next()).isEqualTo(Token.ASSIGN)
assertThat(lexer.next()).isEqualTo(Token.INT)
assertThat(lexer.next()).isEqualTo(Token.EOF)
}
@Test
fun `unterminated block comment fails`() {
val thrown = assertThrows<ParserError> { Lexer("/* bar").next() }
assertThat(thrown).hasMessageContaining("Unexpected end of file")
val justOpener = assertThrows<ParserError> { Lexer("/*").next() }
assertThat(justOpener).hasMessageContaining("Unexpected end of file")
}
@Test
fun acceptsAllUnicodeCodepointsInComments() {
// Test valid Unicode codepoints can appear literally