diff --git a/pkl-core/src/main/java/org/pkl/core/util/yaml/YamlEmitter.java b/pkl-core/src/main/java/org/pkl/core/util/yaml/YamlEmitter.java index c0b9f58c6..fd70e630b 100644 --- a/pkl-core/src/main/java/org/pkl/core/util/yaml/YamlEmitter.java +++ b/pkl-core/src/main/java/org/pkl/core/util/yaml/YamlEmitter.java @@ -1,5 +1,5 @@ /* - * Copyright © 2024-2025 Apple Inc. and the Pkl project authors. All rights reserved. + * Copyright © 2024-2026 Apple Inc. and the Pkl project authors. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -42,6 +42,17 @@ public abstract class YamlEmitter { }; } + /** + * Tells if {@code ch} has to go through {@link YamlEscaper} instead of being emitted literally. + * + *

0x7F, the C1 block, U+FFFE and U+FFFF are outside {@code c-printable} (YAML 1.2 spec 5.1), + * and NEL, LS and PS are line breaks in YAML 1.1. Note that 0x0A is handled by its own {@code + * case} in {@link #emit(String, StringBuilder, boolean)} and never reaches here. + */ + private static boolean getNeedsEscaping(char ch) { + return ch < 0x20 || ch >= 0x7F && ch <= 0x9F || ch == 0x2028 || ch == 0x2029 || ch >= 0xFFFE; + } + public void emit(String str, StringBuilder currIndent, boolean isKey) { if (isReservedWord(str)) { emitSingleQuotedString(str, -1); @@ -102,7 +113,7 @@ public abstract class YamlEmitter { case 'o': break; default: - needsEscaping = first < 0x20; + needsEscaping = getNeedsEscaping(first); hasNonNumberChar = true; } @@ -169,7 +180,7 @@ public abstract class YamlEmitter { 'o', 'x' -> {} default -> { - needsEscaping = needsEscaping || ch < 0x20; + needsEscaping = needsEscaping || getNeedsEscaping(ch); hasNonNumberChar = true; } } diff --git a/pkl-core/src/main/java/org/pkl/core/util/yaml/YamlEscaper.java b/pkl-core/src/main/java/org/pkl/core/util/yaml/YamlEscaper.java index d2aaace67..c1f108097 100644 --- a/pkl-core/src/main/java/org/pkl/core/util/yaml/YamlEscaper.java +++ b/pkl-core/src/main/java/org/pkl/core/util/yaml/YamlEscaper.java @@ -37,6 +37,11 @@ public final class YamlEscaper extends AbstractCharEscaper { for (var i = 0; i < 0x20; i++) { REPLACEMENTS[i] = IoUtils.toHexEscape(i); } + // 0x7F and the C1 block are not `c-printable` either (spec 5.1), so they cannot be emitted + // literally. ns-esc-8-bit covers them; 0x85 is overridden with its named escape below. + for (var i = 0x7F; i <= 0x9F; i++) { + REPLACEMENTS[i] = IoUtils.toHexEscape(i); + } // ns-esc-null REPLACEMENTS[0x00] = "\\0"; // ns-esc-bell @@ -68,6 +73,14 @@ public final class YamlEscaper extends AbstractCharEscaper { @Override protected @Nullable String findReplacement(char ch) { //noinspection UnnecessaryUnicodeEscape - return ch <= 0xA0 ? REPLACEMENTS[ch] : ch == '\u2028' ? "\\L" : ch == '\u2029' ? "\\P" : null; + return ch <= 0xA0 + ? REPLACEMENTS[ch] + : ch == '\u2028' + ? "\\L" + : ch == '\u2029' + ? "\\P" + // U+FFFE and U+FFFF are not `c-printable` either (spec 5.1), and no named or 8-bit + // escape covers them; ns-esc-16-bit does. + : ch >= 0xFFFE ? IoUtils.toUnicodeEscape(ch) : null; } } diff --git a/pkl-core/src/test/files/LanguageSnippetTests/input/api/yamlRendererStrings.yml.pkl b/pkl-core/src/test/files/LanguageSnippetTests/input/api/yamlRendererStrings.yml.pkl index 5d8c5e9de..2cdf9ac98 100644 --- a/pkl-core/src/test/files/LanguageSnippetTests/input/api/yamlRendererStrings.yml.pkl +++ b/pkl-core/src/test/files/LanguageSnippetTests/input/api/yamlRendererStrings.yml.pkl @@ -39,6 +39,17 @@ h4 = "one\rtwo" h5 = "\u{7}" h6 = "one\u{7}two" +// characters outside `c-printable` (YAML 1.2 spec 5.1), and YAML 1.1 line breaks +h7 = "\u{7f}" +h8 = "one\u{7f}two" +h9 = "one\u{80}two" +h10 = "one\u{85}two" +h11 = "one\u{9f}two" +h12 = "one\u{2028}two" +h13 = "one\u{2029}two" +h14 = "one\u{fffe}two" +h15 = "one\u{ffff}two" + // control characters in multiline strings i1 = "one\ttwo\nthree" i2 = "one\rtwo\nthree" diff --git a/pkl-core/src/test/files/LanguageSnippetTests/output/api/yamlRendererStrings.yml b/pkl-core/src/test/files/LanguageSnippetTests/output/api/yamlRendererStrings.yml index 811c85140..6f8b09fd2 100644 --- a/pkl-core/src/test/files/LanguageSnippetTests/output/api/yamlRendererStrings.yml +++ b/pkl-core/src/test/files/LanguageSnippetTests/output/api/yamlRendererStrings.yml @@ -53,6 +53,15 @@ h3: "\r" h4: "one\rtwo" h5: "\a" h6: "one\atwo" +h7: "\x7f" +h8: "one\x7ftwo" +h9: "one\x80two" +h10: "one\Ntwo" +h11: "one\x9ftwo" +h12: "one\Ltwo" +h13: "one\Ptwo" +h14: "one\ufffetwo" +h15: "one\ufffftwo" i1: "one\ttwo\nthree" i2: "one\rtwo\nthree" i3: "one\atwo\nthree"