Switch yaml.Parser to parse binary scalars as Bytes (#1277)

2026-02-25 20:04:59 +01:00 · 2025-10-30 16:45:59 -07:00
parent ede48d0fff
commit ffc9167bf5
8 changed files with 59 additions and 13 deletions
--- a/docs/modules/ROOT/partials/component-attributes.adoc
+++ b/docs/modules/ROOT/partials/component-attributes.adoc
@@ -75,6 +75,8 @@ endif::[]
 :uri-stdlib-xmlModule: {uri-pkl-stdlib-docs}/xml
 :uri-stdlib-protobufModule: {uri-pkl-stdlib-docs}/protobuf
 :uri-stdlib-pklbinaryModule: {uri-pkl-stdlib-docs}/pklbinary
+:uri-stdlib-yamlModule: {uri-pkl-stdlib-docs}/yaml
+:uri-stdlib-YamlParser: {uri-stdlib-yamlModule}/Parser
 :uri-stdlib-evaluatorSettingsModule: {uri-pkl-stdlib-docs}/EvaluatorSettings
 :uri-stdlib-evaluatorSettingsHttpClass: {uri-stdlib-evaluatorSettingsModule}/Http
 :uri-stdlib-Boolean: {uri-stdlib-baseModule}/Boolean
--- a/docs/modules/release-notes/pages/0.30.adoc
+++ b/docs/modules/release-notes/pages/0.30.adoc
@@ -3,6 +3,8 @@
 :version-minor: 0.30.0
 :release-date: November 3rd, 2025

+:yaml-binary-scalar: https://yaml.org/type/binary.html
+
 include::ROOT:partial$component-attributes.adoc[]

 Pkl {version} was released on {release-date}. +
@@ -268,7 +270,7 @@ Thanks to https://github.com/ssalevan[@ssalevan] for their contribution to this
 === Better support for `Bytes` when rendering YAML

 Previously, attempting to render a `Bytes` value using {uri-stdlib-YamlRenderer}[`YamlRenderer`] required the use of a link:{uri-stdlib-PcfRenderer-converters}[converter].
-Now, Pkl can natively render YAML containing https://yaml.org/type/binary.html[binary scalars] (https://github.com/apple/pkl/pull/1276[#1276]).
+Now, Pkl can natively render YAML containing link:{yaml-binary-scalar}[binary scalars] (https://github.com/apple/pkl/pull/1276[#1276]).

 [source,pkl%tested]
 ----
@@ -280,6 +282,22 @@ rendered = new YamlRenderer {}.renderValue(foo) // <1>
 ----
 <1> Result: `bar: !!binary AQID`

+Similarly, link:{uri-stdlib-YamlParser}[`yaml.Parser`] now parses binary YAML values as Pkl link:{uri-stdlib-Bytes}[`Bytes`] values (https://github.com/apple/pkl/pull/1277[#1277]).
+This is a breaking change; previously these values were parsed as link:{uri-stdlib-String}[`String`] value containing the base64-encoded binary data.
+
+[source,pkl%tested]
+----
+import "pkl:yaml"
+
+yamlData = 
+  """
+  bytes: !!binary AQID
+  """
+
+parsed = new yaml.Parser {}.parse(yamlData).bytes // <1>
+----
+<1> Result: `Bytes(1, 2, 3)`
+
 [[pkldoc-perf-improvements]]
 === `pkldoc` performance improvements

@@ -359,6 +377,11 @@ Instead of exiting immediately, it behaves like other REPLs:

 Things to watch out for when upgrading.

+=== Binary data handling `yaml.Parser`
+
+link:{yaml-binary-scalar}[YAML binary scalars] are now parsed as link:{uri-stdlib-Bytes}[`Bytes`] values.
+Prior versions of Pkl parsed binary scalars as link:{uri-stdlib-String}[`String`] values containing the base64-encoded binary data.
+
 === Minimum Kotlin version bump

 For users of Pkl's Kotlin libraries, the minimum Kotlin version has been bumped to 2.1 (https://github.com/apple/pkl/pull/1232[#1232]).
@@ -379,7 +402,7 @@ See <<pkldoc-migration>> for more details.
 == Miscellaneous [small]#🐸#

 * Dependency updates (https://github.com/apple/pkl/pull/1184[#1184], https://github.com/apple/pkl/pull/1225[#1225], https://github.com/apple/pkl/pull/1226[#1226], https://github.com/apple/pkl/pull/1228[#1228]).
-* Enforce Pkl formatting of stdlib (https://github.com/apple/pkl/pull/1236[#1236], https://github.com/apple/pkl/pull/1253[#1253], https://github.com/apple/pkl/pull/1258[#1258], https://github.com/apple/pkl/pull/1278[#1278]).
+* Enforce Pkl formatting of stdlib (https://github.com/apple/pkl/pull/1236[#1236], https://github.com/apple/pkl/pull/1253[#1253], https://github.com/apple/pkl/pull/1258[#1258], https://github.com/apple/pkl/pull/1278[#1278], https://github.com/apple/pkl/pull/1279[#1279]).
 * Add internal IntelliJ plugin that's meant to assist with development of the Pkl codebase itself (https://github.com/apple/pkl/pull/1248[#1248]).
 * Update CircleCI macOS instance type and Xcode version (https://github.com/apple/pkl/pull/1243[#1243], https://github.com/apple/pkl/pull/1244[#1244]).
 * Disable multi-jdk testing when running on Windows ARM (https://github.com/apple/pkl/pull/1223[#1223]).
--- a/pkl-core/src/main/java/org/pkl/core/stdlib/yaml/ParserNodes.java
+++ b/pkl-core/src/main/java/org/pkl/core/stdlib/yaml/ParserNodes.java
@@ -1,5 +1,5 @@
 /*
- * Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved.
+ * Copyright © 2024-2025 Apple Inc. and the Pkl project authors. All rights reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
@@ -322,12 +322,13 @@ public final class ParserNodes {
      }
    }

-    // Pkl doesn't have a binary type, so parse as base64 string with whitespace removed
    private static class ConstructBinary implements ConstructNode {
      @Override
-      public String construct(Node node) {
+      public VmBytes construct(Node node) {
        var value = ((ScalarNode) node).getValue();
-        return WHITESPACE.matcher(value).replaceAll("");
+        var encoded = WHITESPACE.matcher(value).replaceAll("");
+        var decoded = Base64.getDecoder().decode(encoded);
+        return decoded.length == 0 ? VmBytes.EMPTY : new VmBytes(decoded);
      }
    }

--- a/pkl-core/src/test/files/LanguageSnippetTests/input/api/yamlParser1Compat.pkl
+++ b/pkl-core/src/test/files/LanguageSnippetTests/input/api/yamlParser1Compat.pkl
@@ -109,9 +109,16 @@ examples {
    parser.parse("!!timestamp 2001-12-15T02:59:43.1Z")
  }

-  // parsed as string because Pkl doesn't have a binary type
  ["binary"] {
-    parser.parse("!!binary dGhlIHF1aWNrIGJyb3duIGZveCBqdW1wcyBvdmVyIHRoZSBsYXp5IGRvZw==")
+    parser.parse("!!binary AQID")
+    parser.parse("!!binary 'AQID'")
+    parser.parse("!!binary \"AQID\"")
+    parser.parse("""
+      !!binary |-
+        AQID
+        AQID
+        AQID
+      """)
  }

  ["string"] {
@@ -195,6 +202,7 @@ examples {
            price       : 2392.00
      tax  : 251.42
      total: 4443.52
+      logo: !!binary AQID
      comments: >
          Late afternoon is best.
          Backup contact is Nancy
--- a/pkl-core/src/test/files/LanguageSnippetTests/output/api/yamlParser1Compat.pcf
+++ b/pkl-core/src/test/files/LanguageSnippetTests/output/api/yamlParser1Compat.pcf
@@ -97,7 +97,10 @@ examples {
    "2001-12-15T02:59:43.1Z"
  }
  ["binary"] {
-    "dGhlIHF1aWNrIGJyb3duIGZveCBqdW1wcyBvdmVyIHRoZSBsYXp5IGRvZw=="
+    Bytes(1, 2, 3)
+    Bytes(1, 2, 3)
+    Bytes(1, 2, 3)
+    Bytes(1, 2, 3, 1, 2, 3, 1, 2, 3)
  }
  ["string"] {
    null
@@ -190,6 +193,7 @@ examples {
      }
      tax = 251.42
      total = 4443.52
+      logo = Bytes(1, 2, 3)
      comments = "Late afternoon is best. Backup contact is Nancy Billsmer @ 338-4338."
    }
  }
--- a/pkl-core/src/test/files/LanguageSnippetTests/output/api/yamlParser1Yaml11.pcf
+++ b/pkl-core/src/test/files/LanguageSnippetTests/output/api/yamlParser1Yaml11.pcf
@@ -97,7 +97,10 @@ examples {
    "2001-12-15T02:59:43.1Z"
  }
  ["binary"] {
-    "dGhlIHF1aWNrIGJyb3duIGZveCBqdW1wcyBvdmVyIHRoZSBsYXp5IGRvZw=="
+    Bytes(1, 2, 3)
+    Bytes(1, 2, 3)
+    Bytes(1, 2, 3)
+    Bytes(1, 2, 3, 1, 2, 3, 1, 2, 3)
  }
  ["string"] {
    null
@@ -190,6 +193,7 @@ examples {
      }
      tax = 251.42
      total = 4443.52
+      logo = Bytes(1, 2, 3)
      comments = "Late afternoon is best. Backup contact is Nancy Billsmer @ 338-4338."
    }
  }
--- a/pkl-core/src/test/files/LanguageSnippetTests/output/api/yamlParser1Yaml12.pcf
+++ b/pkl-core/src/test/files/LanguageSnippetTests/output/api/yamlParser1Yaml12.pcf
@@ -97,7 +97,10 @@ examples {
    "2001-12-15T02:59:43.1Z"
  }
  ["binary"] {
-    "dGhlIHF1aWNrIGJyb3duIGZveCBqdW1wcyBvdmVyIHRoZSBsYXp5IGRvZw=="
+    Bytes(1, 2, 3)
+    Bytes(1, 2, 3)
+    Bytes(1, 2, 3)
+    Bytes(1, 2, 3, 1, 2, 3, 1, 2, 3)
  }
  ["string"] {
    null
@@ -190,6 +193,7 @@ examples {
      }
      tax = 251.42
      total = 4443.52
+      logo = Bytes(1, 2, 3)
      comments = "Late afternoon is best. Backup contact is Nancy Billsmer @ 338-4338."
    }
  }
--- a/stdlib/yaml.pkl
+++ b/stdlib/yaml.pkl
@@ -32,7 +32,7 @@ module pkl.yaml
 /// | !!seq       | [Listing]                                               |
 /// | !!set       | [Listing]                                               |
 /// | !!map       | [Dynamic] or [Mapping] depending on [Parser.useMapping] |
-/// | !!binary    | [String]                                                |
+/// | !!binary    | [Bytes]                                                 |
 /// | !!timestamp | [String]                                                |
 ///
 /// The element order of YAML sequences, sets, and mappings is maintained.
@@ -88,4 +88,4 @@ class Parser {
 }

 /// Pkl representation of a YAML value.
-typealias Value = Null | Boolean | Number | String | Listing | Dynamic | Mapping
+typealias Value = Null | Boolean | Number | String | Bytes | Listing | Dynamic | Mapping