Escape characters that YAML cannot carry literally (#1846)

YamlEmitter only routed a string into YamlEscaper when it held a
character below 0x20, so 0x7F, the C1 block, U+2028 and U+2029 went out
as plain scalars. They are not `c-printable` (YAML 1.2 spec 5.1), and
NEL, LS and PS are line breaks in YAML 1.1, so the rendered document is
either rejected or silently split by a reader.

The escapes added in #1165 for 0x85, 0xA0, U+2028, and U+2029 could only
fire when some other character happened to open that gate. Share one
predicate between the two sites that decide, and give 0x7F-0x9F the
8-bit escape they lack.

pkl-core/src/main/java/org/pkl/core/util/yaml/YamlEmitter.java

@@ -1,5 +1,5 @@
 /*
- * Copyright © 2024-2025 Apple Inc. and the Pkl project authors. All rights reserved.
+ * Copyright © 2024-2026 Apple Inc. and the Pkl project authors. All rights reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -42,6 +42,17 @@ public abstract class YamlEmitter {
     };
   }
 
+  /**
+   * Tells if {@code ch} has to go through {@link YamlEscaper} instead of being emitted literally.
+   *
+   * <p>0x7F, the C1 block, U+FFFE and U+FFFF are outside {@code c-printable} (YAML 1.2 spec 5.1),
+   * and NEL, LS and PS are line breaks in YAML 1.1. Note that 0x0A is handled by its own {@code
+   * case} in {@link #emit(String, StringBuilder, boolean)} and never reaches here.
+   */
+  private static boolean getNeedsEscaping(char ch) {
+    return ch < 0x20 || ch >= 0x7F && ch <= 0x9F || ch == 0x2028 || ch == 0x2029 || ch >= 0xFFFE;
+  }
+
   public void emit(String str, StringBuilder currIndent, boolean isKey) {
     if (isReservedWord(str)) {
       emitSingleQuotedString(str, -1);
@@ -102,7 +113,7 @@ public abstract class YamlEmitter {
       case 'o':
         break;
       default:
-        needsEscaping = first < 0x20;
+        needsEscaping = getNeedsEscaping(first);
         hasNonNumberChar = true;
     }
 
@@ -169,7 +180,7 @@ public abstract class YamlEmitter {
             'o',
             'x' -> {}
         default -> {
-          needsEscaping = needsEscaping || ch < 0x20;
+          needsEscaping = needsEscaping || getNeedsEscaping(ch);
           hasNonNumberChar = true;
         }
       }

pkl-core/src/main/java/org/pkl/core/util/yaml/YamlEscaper.java

@@ -37,6 +37,11 @@ public final class YamlEscaper extends AbstractCharEscaper {
     for (var i = 0; i < 0x20; i++) {
       REPLACEMENTS[i] = IoUtils.toHexEscape(i);
     }
+    // 0x7F and the C1 block are not `c-printable` either (spec 5.1), so they cannot be emitted
+    // literally. ns-esc-8-bit covers them; 0x85 is overridden with its named escape below.
+    for (var i = 0x7F; i <= 0x9F; i++) {
+      REPLACEMENTS[i] = IoUtils.toHexEscape(i);
+    }
     // ns-esc-null
     REPLACEMENTS[0x00] = "\\0";
     // ns-esc-bell
@@ -68,6 +73,14 @@ public final class YamlEscaper extends AbstractCharEscaper {
   @Override
   protected @Nullable String findReplacement(char ch) {
     //noinspection UnnecessaryUnicodeEscape
-    return ch <= 0xA0 ? REPLACEMENTS[ch] : ch == '\u2028' ? "\\L" : ch == '\u2029' ? "\\P" : null;
+    return ch <= 0xA0
+        ? REPLACEMENTS[ch]
+        : ch == '\u2028'
+            ? "\\L"
+            : ch == '\u2029'
+                ? "\\P"
+                // U+FFFE and U+FFFF are not `c-printable` either (spec 5.1), and no named or 8-bit
+                // escape covers them; ns-esc-16-bit does.
+                : ch >= 0xFFFE ? IoUtils.toUnicodeEscape(ch) : null;
   }
 }

pkl-core/src/test/files/LanguageSnippetTests/input/api/yamlRendererStrings.yml.pkl

@@ -39,6 +39,17 @@ h4 = "one\rtwo"
 h5 = "\u{7}"
 h6 = "one\u{7}two"
 
+// characters outside `c-printable` (YAML 1.2 spec 5.1), and YAML 1.1 line breaks
+h7 = "\u{7f}"
+h8 = "one\u{7f}two"
+h9 = "one\u{80}two"
+h10 = "one\u{85}two"
+h11 = "one\u{9f}two"
+h12 = "one\u{2028}two"
+h13 = "one\u{2029}two"
+h14 = "one\u{fffe}two"
+h15 = "one\u{ffff}two"
+
 // control characters in multiline strings
 i1 = "one\ttwo\nthree"
 i2 = "one\rtwo\nthree"

pkl-core/src/test/files/LanguageSnippetTests/output/api/yamlRendererStrings.yml

@@ -53,6 +53,15 @@ h3: "\r"
 h4: "one\rtwo"
 h5: "\a"
 h6: "one\atwo"
+h7: "\x7f"
+h8: "one\x7ftwo"
+h9: "one\x80two"
+h10: "one\Ntwo"
+h11: "one\x9ftwo"
+h12: "one\Ltwo"
+h13: "one\Ptwo"
+h14: "one\ufffetwo"
+h15: "one\ufffftwo"
 i1: "one\ttwo\nthree"
 i2: "one\rtwo\nthree"
 i3: "one\atwo\nthree"