starred/pkl
1
0
Fork 0
mirror of https://github.com/apple/pkl.git synced 2026-01-11 06:10:40 +01:00
Code Issues 198 Packages Projects Releases 10 Wiki Activity

Add syntax highlighting of Pkl code (#1385)

Browse Source 
This adds syntax highlighting of Pkl code!

It adds highlighting for:

* Stack frames within error messages
* CLI REPL (highlights as you type, highlights error output)
* Power assertions (coming in https://github.com/apple/pkl/pull/1384)

This uses the lexer for highlighting. It will highlight strings,
numbers, keywords, but doesn't understand how to highlight nodes like
types, function params, etc.
The reason for this is because a single line of code by itself may not
be grammatically valid.
This commit is contained in:
Daniel Chao
2026-01-06 10:33:11 -08:00
committed by GitHub
parent 4f4f03dbca
commit 6b9c670cfd
9 changed files with 315 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
pkl-cli/src/main/kotlin/org/pkl/cli/CliRepl.kt
View File

@@ -1,5 +1,5 @@
/*
* Copyright © 2024-2025 Apple Inc. and the Pkl project authors. All rights reserved.
* Copyright © 2024-2026 Apple Inc. and the Pkl project authors. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -71,7 +71,7 @@ internal class CliRepl(private val options: CliEvaluatorOptions) : CliCommand(op
options.base.color?.hasColor() ?: false,
options.base.traceMode ?: TraceMode.COMPACT,
)
Repl(options.base.normalizedWorkingDir, server).run()
Repl(options.base.normalizedWorkingDir, server, options.base.color?.hasColor() ?: false).run()
}
}
}

54
pkl-cli/src/main/kotlin/org/pkl/cli/repl/Repl.kt
View File

@@ -1,5 +1,5 @@
/*
* Copyright © 2024-2025 Apple Inc. and the Pkl project authors. All rights reserved.
* Copyright © 2024-2026 Apple Inc. and the Pkl project authors. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -18,22 +18,40 @@ package org.pkl.cli.repl
import java.io.IOException
import java.net.URI
import java.nio.file.Path
import java.util.regex.Pattern
import kotlin.io.path.deleteIfExists
import org.fusesource.jansi.Ansi
import org.jline.reader.EndOfFileException
import org.jline.reader.Highlighter
import org.jline.reader.LineReader
import org.jline.reader.LineReader.Option
import org.jline.reader.LineReaderBuilder
import org.jline.reader.UserInterruptException
import org.jline.reader.impl.completer.AggregateCompleter
import org.jline.reader.impl.history.DefaultHistory
import org.jline.terminal.TerminalBuilder
import org.jline.utils.AttributedString
import org.jline.utils.InfoCmp
import org.pkl.core.repl.ReplRequest
import org.pkl.core.repl.ReplResponse
import org.pkl.core.repl.ReplServer
import org.pkl.core.util.AnsiStringBuilder
import org.pkl.core.util.AnsiStringBuilder.AnsiCode
import org.pkl.core.util.IoUtils
import org.pkl.core.util.SyntaxHighlighter
internal class Repl(workingDir: Path, private val server: ReplServer) {
class PklHighlighter : Highlighter {
override fun highlight(reader: LineReader, buffer: String): AttributedString {
val ansi = AnsiStringBuilder(true).apply { SyntaxHighlighter.writeTo(this, buffer) }.toString()
return AttributedString.fromAnsi(ansi)
}
override fun setErrorPattern(pattern: Pattern) {}
override fun setErrorIndex(idx: Int) {}
}
internal class Repl(workingDir: Path, private val server: ReplServer, private val color: Boolean) {
private val terminal = TerminalBuilder.builder().apply { jansi(true) }.build()
private val history = DefaultHistory()
private val reader =
@@ -41,12 +59,12 @@ internal class Repl(workingDir: Path, private val server: ReplServer) {
.apply {
history(history)
terminal(terminal)
if (color) {
highlighter(PklHighlighter())
}
completer(AggregateCompleter(CommandCompleter, FileCompleter(workingDir)))
option(Option.DISABLE_EVENT_EXPANSION, true)
variable(
org.jline.reader.LineReader.HISTORY_FILE,
(IoUtils.getPklHomeDir().resolve("repl-history")),
)
variable(LineReader.HISTORY_FILE, (IoUtils.getPklHomeDir().resolve("repl-history")))
}
.build()
@@ -55,6 +73,12 @@ internal class Repl(workingDir: Path, private val server: ReplServer) {
private var maybeQuit = false
private var nextRequestId = 0
private fun String.faint(): String {
val sb = AnsiStringBuilder(color)
sb.append(AnsiCode.FAINT, this)
return sb.toString()
}
fun run() {
// JLine 2 history file is incompatible with JLine 3
IoUtils.getPklHomeDir().resolve("repl-history.bin").deleteIfExists()
@@ -70,11 +94,11 @@ internal class Repl(workingDir: Path, private val server: ReplServer) {
try {
if (continuation) {
nextRequestId -= 1
reader.readLine(" ".repeat("pkl$nextRequestId> ".length))
reader.readLine(" ".repeat("pkl$nextRequestId> ".length).faint())
} else {
reader.readLine("pkl$nextRequestId> ")
reader.readLine("pkl$nextRequestId> ".faint())
}
} catch (e: UserInterruptException) {
} catch (_: UserInterruptException) {
if (!continuation && reader.buffer.length() == 0) {
if (maybeQuit) quit()
else {
@@ -87,7 +111,7 @@ internal class Repl(workingDir: Path, private val server: ReplServer) {
inputBuffer = ""
continuation = false
continue
} catch (e: EndOfFileException) {
} catch (_: EndOfFileException) {
":quit"
}
@@ -111,10 +135,10 @@ internal class Repl(workingDir: Path, private val server: ReplServer) {
} finally {
try {
history.save()
} catch (ignored: IOException) {}
} catch (_: IOException) {}
try {
terminal.close()
} catch (ignored: IOException) {}
} catch (_: IOException) {}
}
}
@@ -124,10 +148,12 @@ internal class Repl(workingDir: Path, private val server: ReplServer) {
candidates.isEmpty() -> {
println("Unknown command: `${inputBuffer.drop(1)}`")
}
candidates.size > 1 -> {
print("Which of the following did you mean? ")
println(candidates.joinToString(separator = " ") { "`:${it.type}`" })
}
else -> {
doExecuteCommand(candidates.single())
}
@@ -193,16 +219,20 @@ internal class Repl(workingDir: Path, private val server: ReplServer) {
is ReplResponse.EvalSuccess -> {
println(response.result)
}
is ReplResponse.EvalError -> {
println(response.message)
}
is ReplResponse.InternalError -> {
throw response.cause
}
is ReplResponse.IncompleteInput -> {
assert(responses.size == 1)
continuation = true
}
else -> throw IllegalStateException("Unexpected response: $response")
}
}

13
pkl-core/src/main/java/org/pkl/core/repl/ReplServer.java
View File

@@ -1,5 +1,5 @@
/*
* Copyright © 2024-2025 Apple Inc. and the Pkl project authors. All rights reserved.
* Copyright © 2024-2026 Apple Inc. and the Pkl project authors. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -45,10 +45,12 @@ import org.pkl.core.repl.ReplResponse.EvalSuccess;
import org.pkl.core.repl.ReplResponse.InvalidRequest;
import org.pkl.core.resource.ResourceReader;
import org.pkl.core.runtime.*;
import org.pkl.core.util.AnsiStringBuilder;
import org.pkl.core.util.EconomicMaps;
import org.pkl.core.util.IoUtils;
import org.pkl.core.util.MutableReference;
import org.pkl.core.util.Nullable;
import org.pkl.core.util.SyntaxHighlighter;
import org.pkl.parser.Parser;
import org.pkl.parser.ParserError;
import org.pkl.parser.syntax.Class;
@@ -69,6 +71,7 @@ public class ReplServer implements AutoCloseable {
private final VmExceptionRenderer errorRenderer;
private final PackageResolver packageResolver;
private final @Nullable ProjectDependenciesManager projectDependenciesManager;
private final boolean color;
public ReplServer(
SecurityManager securityManager,
@@ -90,6 +93,7 @@ public class ReplServer implements AutoCloseable {
this.securityManager = securityManager;
this.moduleResolver = new ModuleResolver(moduleKeyFactories);
this.errorRenderer = new VmExceptionRenderer(new StackTraceRenderer(frameTransformer), color);
this.color = color;
replState = new ReplState(createEmptyReplModule(BaseModule.getModuleClass().getPrototype()));
var languageRef = new MutableReference<VmLanguage>(null);
@@ -172,7 +176,7 @@ public class ReplServer implements AutoCloseable {
.collect(Collectors.toList());
}
@SuppressWarnings({"StatementWithEmptyBody", "DataFlowIssue"})
@SuppressWarnings({"StatementWithEmptyBody"})
private List<Object> evaluate(
ReplState replState,
String requestId,
@@ -448,7 +452,10 @@ public class ReplServer implements AutoCloseable {
}
private String render(Object value) {
return VmValueRenderer.multiLine(Integer.MAX_VALUE).render(value);
var sb = new AnsiStringBuilder(color);
var src = VmValueRenderer.multiLine(Integer.MAX_VALUE).render(value);
SyntaxHighlighter.writeTo(sb, src);
return sb.toString();
}
private static class ReplState {

11
pkl-core/src/main/java/org/pkl/core/runtime/StackTraceRenderer.java
View File

@@ -1,5 +1,5 @@
/*
* Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved.
* Copyright © 2024-2026 Apple Inc. and the Pkl project authors. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -22,6 +22,7 @@ import org.pkl.core.StackFrame;
import org.pkl.core.util.AnsiStringBuilder;
import org.pkl.core.util.AnsiTheme;
import org.pkl.core.util.Nullable;
import org.pkl.core.util.SyntaxHighlighter;
public final class StackTraceRenderer {
private final Function<StackFrame, StackFrame> frameTransformer;
@@ -104,9 +105,11 @@ public final class StackTraceRenderer {
var prefix = frame.getStartLine() + " | ";
out.append(AnsiTheme.STACK_TRACE_MARGIN, leftMargin)
.append(AnsiTheme.STACK_TRACE_LINE_NUMBER, prefix)
.append(sourceLine)
.append('\n')
.append(AnsiTheme.STACK_TRACE_LINE_NUMBER, prefix);
SyntaxHighlighter.writeTo(out, sourceLine);
out.append('\n')
.append(AnsiTheme.STACK_TRACE_MARGIN, leftMargin)
.append(" ".repeat(prefix.length() + startColumn - 1))
.append(AnsiTheme.STACK_TRACE_CARET, "^".repeat(endColumn - startColumn + 1))

31
pkl-core/src/main/java/org/pkl/core/util/AnsiStringBuilder.java
View File

@@ -1,5 +1,5 @@
/*
* Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved.
* Copyright © 2024-2026 Apple Inc. and the Pkl project authors. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -19,7 +19,7 @@ import java.io.PrintWriter;
import java.util.EnumSet;
import java.util.Set;
@SuppressWarnings("DuplicatedCode")
@SuppressWarnings({"DuplicatedCode", "UnusedReturnValue"})
public final class AnsiStringBuilder {
private final StringBuilder builder = new StringBuilder();
private final boolean usingColor;
@@ -108,6 +108,25 @@ public final class AnsiStringBuilder {
return this;
}
/** Provides a runnable where anything appended is not affected by the existing context. */
public AnsiStringBuilder appendSandboxed(Runnable runnable) {
if (!usingColor) {
runnable.run();
return this;
}
var myCodes = currentCodes;
var myDeclaredCodes = declaredCodes;
currentCodes = EnumSet.noneOf(AnsiCode.class);
declaredCodes = EnumSet.noneOf(AnsiCode.class);
doReset();
runnable.run();
doReset();
currentCodes = myCodes;
declaredCodes = myDeclaredCodes;
doAppendCodes(currentCodes);
return this;
}
/**
* Append a string whose contents are unknown, and might contain ANSI color codes.
*
@@ -180,6 +199,14 @@ public final class AnsiStringBuilder {
return new PrintWriter(new StringBuilderWriter(builder));
}
public int length() {
return builder.length();
}
public void setLength(int length) {
builder.setLength(length);
}
/** Builds the data represented by this builder into a {@link String}. */
public String toString() {
// be a good citizen and unset any ansi escape codes currently set.

14
pkl-core/src/main/java/org/pkl/core/util/AnsiTheme.java
View File

@@ -1,5 +1,5 @@
/*
* Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved.
* Copyright © 2024-2026 Apple Inc. and the Pkl project authors. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -37,5 +37,15 @@ public final class AnsiTheme {
public static final AnsiCode TEST_NAME = AnsiCode.FAINT;
public static final AnsiCode TEST_FACT_SOURCE = AnsiCode.RED;
public static final AnsiCode TEST_FAILURE_MESSAGE = AnsiCode.RED;
public static final Set<AnsiCode> TEST_EXAMPLE_OUTPUT = EnumSet.of(AnsiCode.RED, AnsiCode.BOLD);
public static final EnumSet<AnsiCode> TEST_EXAMPLE_OUTPUT =
EnumSet.of(AnsiCode.RED, AnsiCode.BOLD);
public static final AnsiCode SYNTAX_KEYWORD = AnsiCode.BLUE;
public static final AnsiCode SYNTAX_NUMBER = AnsiCode.GREEN;
public static final AnsiCode SYNTAX_STRING = AnsiCode.YELLOW;
public static final AnsiCode SYNTAX_STRING_ESCAPE = AnsiCode.BRIGHT_YELLOW;
public static final AnsiCode SYNTAX_COMMENT = AnsiCode.FAINT;
public static final AnsiCode SYNTAX_OPERATOR = AnsiCode.RESET;
public static final AnsiCode SYNTAX_CONTROL = AnsiCode.BLUE;
public static final AnsiCode SYNTAX_CONSTANT = AnsiCode.CYAN;
}

172
pkl-core/src/main/java/org/pkl/core/util/SyntaxHighlighter.java Normal file
View File

@@ -0,0 +1,172 @@
/*
* Copyright © 2025-2026 Apple Inc. and the Pkl project authors. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.pkl.core.util;
import static org.pkl.parser.Token.FALSE;
import static org.pkl.parser.Token.NULL;
import static org.pkl.parser.Token.STRING_ESCAPE_BACKSLASH;
import static org.pkl.parser.Token.STRING_ESCAPE_NEWLINE;
import static org.pkl.parser.Token.STRING_ESCAPE_QUOTE;
import static org.pkl.parser.Token.STRING_ESCAPE_RETURN;
import static org.pkl.parser.Token.STRING_ESCAPE_TAB;
import static org.pkl.parser.Token.STRING_ESCAPE_UNICODE;
import static org.pkl.parser.Token.TRUE;
import java.util.EnumSet;
import org.pkl.parser.Lexer;
import org.pkl.parser.ParserError;
import org.pkl.parser.Token;
/** Syntax highlighter that emits ansi color codes. */
public final class SyntaxHighlighter {
private SyntaxHighlighter() {}
private static final EnumSet<Token> stringEscape =
EnumSet.of(
STRING_ESCAPE_NEWLINE,
STRING_ESCAPE_TAB,
STRING_ESCAPE_RETURN,
STRING_ESCAPE_QUOTE,
STRING_ESCAPE_BACKSLASH,
STRING_ESCAPE_UNICODE);
private static final EnumSet<Token> constant = EnumSet.of(TRUE, FALSE, NULL);
private static final EnumSet<Token> operator =
EnumSet.of(
Token.COALESCE,
Token.LT,
Token.GT,
Token.NOT,
Token.EQUAL,
Token.NOT_EQUAL,
Token.LTE,
Token.GTE,
Token.AND,
Token.OR,
Token.PLUS,
Token.MINUS,
Token.POW,
Token.STAR,
Token.DIV,
Token.INT_DIV,
Token.MOD,
Token.PIPE);
private static final EnumSet<Token> keyword =
EnumSet.of(
Token.AMENDS,
Token.AS,
Token.EXTENDS,
Token.CLASS,
Token.TYPE_ALIAS,
Token.FUNCTION,
Token.MODULE,
Token.IMPORT,
Token.IMPORT_STAR,
Token.READ,
Token.READ_STAR,
Token.READ_QUESTION,
Token.TRACE,
Token.THROW,
Token.UNKNOWN,
Token.NOTHING,
Token.OUTER,
Token.SUPER,
Token.THIS,
Token.HIDDEN,
Token.ABSTRACT,
Token.CONST,
Token.FIXED,
Token.LOCAL,
Token.OPEN);
private static final EnumSet<Token> control =
EnumSet.of(Token.NEW, Token.IF, Token.ELSE, Token.WHEN, Token.FOR, Token.IN, Token.OUT);
private static final EnumSet<Token> number =
EnumSet.of(Token.INT, Token.FLOAT, Token.BIN, Token.OCT, Token.HEX);
public static void writeTo(AnsiStringBuilder out, String src) {
var prevLength = out.length();
try {
var lexer = new Lexer(src);
doHighlightNormal(out, lexer.next(), lexer, Token.EOF);
} catch (ParserError err) {
// bail out and emit everything un-highlighted
out.setLength(prevLength);
out.append(src);
}
}
private static void highlightString(AnsiStringBuilder out, Lexer lexer, Token token) {
out.append(
AnsiTheme.SYNTAX_STRING,
() -> {
var next = token;
while (next != Token.STRING_END && next != Token.EOF) {
if (stringEscape.contains(next)) {
out.append(AnsiTheme.SYNTAX_STRING_ESCAPE, lexer.text());
next = advance(out, lexer);
continue;
} else if (next == Token.INTERPOLATION_START) {
out.append(AnsiTheme.SYNTAX_STRING_ESCAPE, lexer.text());
out.appendSandboxed(() -> doHighlightNormal(out, lexer.next(), lexer, Token.RPAREN));
out.append(AnsiTheme.SYNTAX_STRING_ESCAPE, lexer.text());
lexer.next();
}
out.append(lexer.text());
next = advance(out, lexer);
}
out.append(lexer.text());
});
}
private static void doHighlightNormal(AnsiStringBuilder out, Token next, Lexer lexer, Token end) {
{
while (next != end && next != Token.EOF) {
if (constant.contains(next)) {
out.append(AnsiTheme.SYNTAX_CONSTANT, lexer.text());
} else if (operator.contains(next)) {
out.append(AnsiTheme.SYNTAX_OPERATOR, lexer.text());
} else if (control.contains(next)) {
out.append(AnsiTheme.SYNTAX_CONTROL, lexer.text());
} else if (keyword.contains(next)) {
out.append(AnsiTheme.SYNTAX_KEYWORD, lexer.text());
} else if (next.isAffix()) {
out.append(AnsiTheme.SYNTAX_COMMENT, lexer.text());
} else if (number.contains(next)) {
out.append(AnsiTheme.SYNTAX_NUMBER, lexer.text());
} else if (next == Token.STRING_MULTI_START || next == Token.STRING_START) {
highlightString(out, lexer, next);
} else {
out.append(lexer.text());
}
next = advance(out, lexer);
}
}
}
private static Token advance(AnsiStringBuilder out, Lexer lexer) {
var prevCursor = lexer.getCursor();
var next = lexer.next();
// fill in any whitespace (includes semicolons)
if (lexer.getStartCursor() > prevCursor) {
out.append(lexer.textFor(prevCursor, lexer.getStartCursor() - prevCursor));
}
return next;
}
}

33
pkl-core/src/test/kotlin/org/pkl/core/ReplServerTest.kt
View File

@@ -1,5 +1,5 @@
/*
* Copyright © 2024-2025 Apple Inc. and the Pkl project authors. All rights reserved.
* Copyright © 2024-2026 Apple Inc. and the Pkl project authors. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -192,6 +192,37 @@ class ReplServerTest {
assertThat(result4).contains("Expected value of type `String`, but got type `Int`.")
}
@Test
fun `syntax highlighting on response values`() {
val server =
ReplServer(
SecurityManagers.defaultManager,
HttpClient.dummyClient(),
Loggers.stdErr(),
listOf(
ModuleKeyFactories.standardLibrary,
ModuleKeyFactories.classPath(this::class.java.classLoader),
ModuleKeyFactories.file,
),
listOf(ResourceReaders.environmentVariable(), ResourceReaders.externalProperty()),
mapOf("NAME1" to "value1", "NAME2" to "value2"),
mapOf("name1" to "value1", "name2" to "value2"),
null,
null,
null,
"/".toPath(),
StackFrameTransformers.defaultTransformer,
true,
TraceMode.COMPACT,
)
val responses = server.handleRequest(ReplRequest.Eval("id", "5.ms", false, false))
assertThat(responses).hasSize(1)
val response = responses[0]
assertThat(response).isInstanceOf(ReplResponse.EvalSuccess::class.java)
assertThat((response as ReplResponse.EvalSuccess).result).isEqualTo("\u001B[32m5\u001B[0m.ms")
}
private fun makeEvalRequest(text: String): String {
val responses = server.handleRequest(ReplRequest.Eval("id", text, false, false))

10
pkl-parser/src/main/java/org/pkl/parser/Lexer.java
View File

@@ -1,5 +1,5 @@
/*
* Copyright © 2024-2025 Apple Inc. and the Pkl project authors. All rights reserved.
* Copyright © 2024-2026 Apple Inc. and the Pkl project authors. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -65,6 +65,14 @@ public class Lexer {
return new String(source, sCursor, cursor - sCursor);
}
public int getStartCursor() {
return sCursor;
}
public int getCursor() {
return cursor;
}
public char[] getSource() {
return source;
}