Implement Pkl binary renderer and parser (#1203)

Implements a binary renderer for Pkl values, which is a lossless capturing of Pkl data.

This follows the pkl binary format that is already used with `pkl server` calls, and is
made available as a Java API and also an in-language API.

Also, introduces a binary parser into the corresponding `PObject` types in Java.

pkl-config-java/gradle.lockfile

@@ -79,7 +79,7 @@ org.junit.platform:junit-platform-commons:1.14.0=testCompileClasspath,testImplem
 org.junit.platform:junit-platform-engine:1.14.0=testRuntimeClasspath
 org.junit.platform:junit-platform-launcher:1.14.0=testRuntimeClasspath
 org.junit:junit-bom:5.14.0=testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath
-org.msgpack:msgpack-core:0.9.8=pklCodegenJava,runtimeClasspath,testRuntimeClasspath
+org.msgpack:msgpack-core:0.9.8=compileClasspath,pklCodegenJava,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
 org.opentest4j:opentest4j:1.3.0=testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath
 org.organicdesign:Paguro:3.10.3=pklCodegenJava,runtimeClasspath,testRuntimeClasspath
 org.snakeyaml:snakeyaml-engine:2.10=pklCodegenJava,runtimeClasspath,testRuntimeClasspath

pkl-config-java/pkl-config-java.gradle.kts

@@ -1,5 +1,5 @@
 /*
- * Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved.
+ * Copyright © 2024-2025 Apple Inc. and the Pkl project authors. All rights reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -82,6 +82,7 @@ dependencies {
   api(projects.pklCore)
 
   implementation(libs.geantyref)
+  implementation(libs.msgpack)
 
   testImplementation(libs.javaxInject)
 

pkl-config-java/src/main/java/org/pkl/config/java/Config.java

@@ -1,5 +1,5 @@
 /*
- * Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved.
+ * Copyright © 2024-2025 Apple Inc. and the Pkl project authors. All rights reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -15,9 +15,14 @@
  */
 package org.pkl.config.java;
 
+import java.io.InputStream;
 import java.lang.reflect.Type;
+import java.util.Map;
 import org.pkl.config.java.mapper.ConversionException;
+import org.pkl.config.java.mapper.ValueMapper;
+import org.pkl.core.Composite;
 import org.pkl.core.Evaluator;
+import org.pkl.core.PklBinaryDecoder;
 
 /**
  * A root, intermediate, or leaf node in a configuration tree. Child nodes can be obtained by name
@@ -67,4 +72,51 @@ public interface Config {
    * @throws ConversionException if the value cannot be converted to the given type
    */
   <T> T as(JavaType<T> type);
+
+  /**
+   * Decode a config from the supplied byte array.
+   *
+   * @return the encoded config
+   */
+  static Config from(byte[] bytes, ValueMapper mapper) {
+    return makeConfig(PklBinaryDecoder.decode(bytes), mapper);
+  }
+
+  /**
+   * Decode a config from the supplied byte array using a preconfigured {@link ValueMapper}.
+   *
+   * @return the encoded config
+   */
+  static Config from(byte[] bytes) {
+    return from(bytes, ValueMapper.preconfigured());
+  }
+
+  /**
+   * Decode a config from the supplied {@link InputStream} using a preconfigured {@link
+   * ValueMapper}.
+   *
+   * @return the encoded config
+   */
+  static Config from(InputStream inputStream, ValueMapper mapper) {
+    return makeConfig(PklBinaryDecoder.decode(inputStream), mapper);
+  }
+
+  /**
+   * Decode a config from the supplied {@link InputStream}.
+   *
+   * @return the encoded config
+   */
+  static Config from(InputStream inputStream) {
+    return from(inputStream, ValueMapper.preconfigured());
+  }
+
+  private static Config makeConfig(Object decoded, ValueMapper mapper) {
+    if (decoded instanceof Composite composite) {
+      return new CompositeConfig("", mapper, composite);
+    }
+    if (decoded instanceof Map<?, ?> map) {
+      return new MapConfig("", mapper, map);
+    }
+    return new LeafConfig("", mapper, decoded);
+  }
 }

pkl-config-java/src/test/java/org/pkl/config/java/AbstractConfigTest.java

@@ -0,0 +1,175 @@
+/*
+ * Copyright © 2024-2025 Apple Inc. and the Pkl project authors. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.pkl.config.java;
+
+import static org.assertj.core.api.Assertions.*;
+
+import java.nio.file.Path;
+import java.util.List;
+import org.junit.jupiter.api.Test;
+import org.pkl.config.java.mapper.Named;
+import org.pkl.config.java.mapper.Types;
+import org.pkl.core.PObject;
+
+public abstract class AbstractConfigTest {
+
+  private final Config pigeonConfig = getPigeonConfig();
+  private final Config pigeonModuleConfig = getPigeonModuleConfig();
+  private final Config pairConfig = getPairConfig();
+  private final Config mapConfig = getMapConfig();
+
+  protected abstract Config getPigeonConfig();
+
+  protected abstract Config getPigeonModuleConfig();
+
+  protected abstract Config getPairConfig();
+
+  protected abstract Config getMapConfig();
+
+  @Test
+  public void navigate() {
+    var pigeon = pigeonConfig.get("pigeon");
+    assertThat(pigeon.getQualifiedName()).isEqualTo("pigeon");
+    assertThat(pigeon.getRawValue()).isInstanceOf(PObject.class);
+
+    var address = pigeon.get("address");
+    assertThat(address.getQualifiedName()).isEqualTo("pigeon.address");
+    assertThat(address.getRawValue()).isInstanceOf(PObject.class);
+
+    var street = address.get("street");
+    assertThat(street.getQualifiedName()).isEqualTo("pigeon.address.street");
+    assertThat(street.getRawValue()).isInstanceOf(String.class);
+
+    assertThat(street.as(String.class)).isEqualTo("Fuzzy St.");
+  }
+
+  @Test
+  public void navigateToNonExistingObjectChild() {
+    var pigeon = pigeonConfig.get("pigeon");
+    var t = catchThrowable(() -> pigeon.get("non-existing"));
+
+    assertThat(t)
+        .isInstanceOf(NoSuchChildException.class)
+        .hasMessageStartingWith(
+            "Node `pigeon` of type `pkl.base#Dynamic` "
+                + "does not have a property named `non-existing`.");
+  }
+
+  @Test
+  public void navigateToNonExistingMapChild() {
+    var map = mapConfig.get("x");
+    var t = catchThrowable(() -> map.get("non-existing"));
+
+    assertThat(t)
+        .isInstanceOf(NoSuchChildException.class)
+        .hasMessageStartingWith(
+            "Node `x` of type `pkl.base#Map` does not have a key named `non-existing`.");
+  }
+
+  @Test
+  public void navigateToNonExistingLeafChild() {
+    var age = pigeonConfig.get("pigeon").get("age");
+    var t = catchThrowable(() -> age.get("non-existing"));
+
+    assertThat(t)
+        .isInstanceOf(NoSuchChildException.class)
+        .hasMessageStartingWith(
+            "Leaf node `pigeon.age` of type `pkl.base#Int` does not have a child named `non-existing`.");
+  }
+
+  @Test
+  public void convertObjectToPojoByType() {
+    Person pigeon = pigeonConfig.get("pigeon").as(Person.class);
+    checkPigeon(pigeon);
+  }
+
+  @Test
+  public void convertObjectToPojoByJavaType() {
+    var pigeon = pigeonConfig.get("pigeon").as(JavaType.of(Person.class));
+    checkPigeon(pigeon);
+  }
+
+  @Test
+  public void convertModuleToPojoByType() {
+    var pigeon = pigeonModuleConfig.as(Person.class);
+    checkPigeon(pigeon);
+  }
+
+  @Test
+  public void convertModuleToPojoByJavaType() {
+    var pigeon = pigeonModuleConfig.as(JavaType.of(Person.class));
+    checkPigeon(pigeon);
+  }
+
+  private void checkPigeon(Person pigeon) {
+    assertThat(pigeon).isNotNull();
+    assertThat(pigeon.age).isEqualTo(30);
+    assertThat(pigeon.friends).containsExactly("john", "mary");
+    assertThat(pigeon.address.street).isEqualTo("Fuzzy St.");
+  }
+
+  @Test
+  public void convertToParameterizedTypeByType() {
+    Pair<Path, Integer> pair =
+        pairConfig.get("x").as(Types.parameterizedType(Pair.class, Path.class, Integer.class));
+    checkPair(pair);
+  }
+
+  @Test
+  public void convertToParameterizedTypeByJavaType() {
+    var pair = pairConfig.get("x").as(new JavaType<Pair<Path, Integer>>() {});
+    checkPair(pair);
+  }
+
+  private void checkPair(Pair<?, ?> pair) {
+    assertThat(pair).isNotNull();
+    assertThat(pair.first).isEqualTo(Path.of("file/path"));
+    assertThat(pair.second).isEqualTo(42);
+  }
+
+  public static class Person {
+    final int age;
+    final List<String> friends;
+    final Address address;
+
+    public Person(
+        @Named("age") int age,
+        @Named("friends") List<String> friends,
+        @Named("address") Address address) {
+      this.age = age;
+      this.friends = friends;
+      this.address = address;
+    }
+  }
+
+  public static class Address {
+    final String street;
+
+    public Address(@Named("street") String street) {
+      this.street = street;
+    }
+  }
+
+  public static class Pair<S, T> {
+    final S first;
+    final T second;
+
+    public Pair(@Named("first") S first, @Named("second") T second) {
+      this.first = first;
+      this.second = second;
+    }
+  }
+}

pkl-config-java/src/test/java/org/pkl/config/java/ConfigPklBinaryDecoderTest.java

@@ -0,0 +1,56 @@
+/*
+ * Copyright © 2024-2025 Apple Inc. and the Pkl project authors. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.pkl.config.java;
+
+import java.util.Base64;
+
+public class ConfigPklBinaryDecoderTest extends AbstractConfigTest {
+  // generate via: pbpaste | ./pkl-cli/build/executable/jpkl eval /dev/stdin -f pkl-binary | base64
+
+  @Override
+  protected Config getPigeonConfig() {
+    // pigeon { age = 30; friends = List("john", "mary"); address { street = "Fuzzy St." } }
+    return Config.from(
+        Base64.getDecoder()
+            .decode(
+                "lAGkdGVzdNklZmlsZTovLy9Vc2Vycy9qYmFzY2gvc3JjL3BrbC90ZXN0LnBrbJGTEKZwaWdlb26UAadEeW5hbWljqHBrbDpiYXNlk5MQo2FnZR6TEKdmcmllbmRzkgSSpGpvaG6kbWFyeZMQp2FkZHJlc3OUAadEeW5hbWljqHBrbDpiYXNlkZMQpnN0cmVldKlGdXp6eSBTdC4="));
+  }
+
+  @Override
+  protected Config getPigeonModuleConfig() {
+    // age = 30; friends = List("john", "mary"); address { street = "Fuzzy St." }
+    return Config.from(
+        Base64.getDecoder()
+            .decode(
+                "lAGlc3RkaW6xZmlsZTovLy9kZXYvc3RkaW6TkxCjYWdlHpMQp2ZyaWVuZHOSBJKkam9obqRtYXJ5kxCnYWRkcmVzc5QBp0R5bmFtaWOocGtsOmJhc2WRkxCmc3RyZWV0qUZ1enp5IFN0Lg=="));
+  }
+
+  @Override
+  protected Config getPairConfig() {
+    // x { first = "file/path"; second = 42 }
+    return Config.from(
+        Base64.getDecoder()
+            .decode(
+                "lAGlc3RkaW6xZmlsZTovLy9kZXYvc3RkaW6RkxCheJQBp0R5bmFtaWOocGtsOmJhc2WSkxClZmlyc3SpZmlsZS9wYXRokxCmc2Vjb25kKg=="));
+  }
+
+  @Override
+  protected Config getMapConfig() {
+    // x = Map("one", 1, "two", 2)
+    return Config.from(
+        Base64.getDecoder().decode("lAGlc3RkaW6xZmlsZTovLy9kZXYvc3RkaW6RkxCheJICgqNvbmUBo3R3bwI="));
+  }
+}

pkl-config-java/src/test/java/org/pkl/config/java/ConfigTest.java

@@ -1,5 +1,5 @@
 /*
- * Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved.
+ * Copyright © 2025 Apple Inc. and the Pkl project authors. All rights reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -15,164 +15,31 @@
  */
 package org.pkl.config.java;
 
-import static org.assertj.core.api.Assertions.*;
 import static org.pkl.core.ModuleSource.text;
 
-import java.nio.file.Path;
-import java.util.List;
-import org.junit.jupiter.api.Test;
-import org.pkl.config.java.mapper.Named;
-import org.pkl.config.java.mapper.Types;
-import org.pkl.core.PObject;
+public class ConfigTest extends AbstractConfigTest {
+  private static final ConfigEvaluator evaluator = ConfigEvaluator.preconfigured();
 
-public class ConfigTest {
-  private final ConfigEvaluator evaluator = ConfigEvaluator.preconfigured();
-
-  private final Config pigeonConfig =
-      evaluator.evaluate(
-          text(
-              "pigeon { age = 30; friends = List(\"john\", \"mary\"); address { street = \"Fuzzy St.\" } }"));
-
-  private final Config pigeonModuleConfig =
-      evaluator.evaluate(
-          text("age = 30; friends = List(\"john\", \"mary\"); address { street = \"Fuzzy St.\" }"));
-
-  private final Config pairConfig =
-      evaluator.evaluate(text("x { first = \"file/path\"; second = 42 }"));
-
-  private final Config mapConfig = evaluator.evaluate(text("x = Map(\"one\", 1, \"two\", 2)"));
-
-  @Test
-  public void navigate() {
-    var pigeon = pigeonConfig.get("pigeon");
-    assertThat(pigeon.getQualifiedName()).isEqualTo("pigeon");
-    assertThat(pigeon.getRawValue()).isInstanceOf(PObject.class);
-
-    var address = pigeon.get("address");
-    assertThat(address.getQualifiedName()).isEqualTo("pigeon.address");
-    assertThat(address.getRawValue()).isInstanceOf(PObject.class);
-
-    var street = address.get("street");
-    assertThat(street.getQualifiedName()).isEqualTo("pigeon.address.street");
-    assertThat(street.getRawValue()).isInstanceOf(String.class);
-
-    assertThat(street.as(String.class)).isEqualTo("Fuzzy St.");
+  @Override
+  protected Config getPigeonConfig() {
+    return evaluator.evaluate(
+        text(
+            "pigeon { age = 30; friends = List(\"john\", \"mary\"); address { street = \"Fuzzy St.\" } }"));
   }
 
-  @Test
-  public void navigateToNonExistingObjectChild() {
-    var pigeon = pigeonConfig.get("pigeon");
-    var t = catchThrowable(() -> pigeon.get("non-existing"));
-
-    assertThat(t)
-        .isInstanceOf(NoSuchChildException.class)
-        .hasMessageStartingWith(
-            "Node `pigeon` of type `pkl.base#Dynamic` "
-                + "does not have a property named `non-existing`.");
+  @Override
+  protected Config getPigeonModuleConfig() {
+    return evaluator.evaluate(
+        text("age = 30; friends = List(\"john\", \"mary\"); address { street = \"Fuzzy St.\" }"));
   }
 
-  @Test
-  public void navigateToNonExistingMapChild() {
-    var map = mapConfig.get("x");
-    var t = catchThrowable(() -> map.get("non-existing"));
-
-    assertThat(t)
-        .isInstanceOf(NoSuchChildException.class)
-        .hasMessageStartingWith(
-            "Node `x` of type `pkl.base#Map` does not have a key named `non-existing`.");
+  @Override
+  protected Config getPairConfig() {
+    return evaluator.evaluate(text("x { first = \"file/path\"; second = 42 }"));
   }
 
-  @Test
-  public void navigateToNonExistingLeafChild() {
-    var age = pigeonConfig.get("pigeon").get("age");
-    var t = catchThrowable(() -> age.get("non-existing"));
-
-    assertThat(t)
-        .isInstanceOf(NoSuchChildException.class)
-        .hasMessageStartingWith(
-            "Leaf node `pigeon.age` of type `pkl.base#Int` does not have a child named `non-existing`.");
-  }
-
-  @Test
-  public void convertObjectToPojoByType() {
-    Person pigeon = pigeonConfig.get("pigeon").as(Person.class);
-    checkPigeon(pigeon);
-  }
-
-  @Test
-  public void convertObjectToPojoByJavaType() {
-    var pigeon = pigeonConfig.get("pigeon").as(JavaType.of(Person.class));
-    checkPigeon(pigeon);
-  }
-
-  @Test
-  public void convertModuleToPojoByType() {
-    var pigeon = pigeonModuleConfig.as(Person.class);
-    checkPigeon(pigeon);
-  }
-
-  @Test
-  public void convertModuleToPojoByJavaType() {
-    var pigeon = pigeonModuleConfig.as(JavaType.of(Person.class));
-    checkPigeon(pigeon);
-  }
-
-  private void checkPigeon(Person pigeon) {
-    assertThat(pigeon).isNotNull();
-    assertThat(pigeon.age).isEqualTo(30);
-    assertThat(pigeon.friends).containsExactly("john", "mary");
-    assertThat(pigeon.address.street).isEqualTo("Fuzzy St.");
-  }
-
-  @Test
-  public void convertToParameterizedTypeByType() {
-    Pair<Path, Integer> pair =
-        pairConfig.get("x").as(Types.parameterizedType(Pair.class, Path.class, Integer.class));
-    checkPair(pair);
-  }
-
-  @Test
-  public void convertToParameterizedTypeByJavaType() {
-    var pair = pairConfig.get("x").as(new JavaType<Pair<Path, Integer>>() {});
-    checkPair(pair);
-  }
-
-  private void checkPair(Pair<?, ?> pair) {
-    assertThat(pair).isNotNull();
-    assertThat(pair.first).isEqualTo(Path.of("file/path"));
-    assertThat(pair.second).isEqualTo(42);
-  }
-
-  public static class Person {
-    final int age;
-    final List<String> friends;
-    final Address address;
-
-    public Person(
-        @Named("age") int age,
-        @Named("friends") List<String> friends,
-        @Named("address") Address address) {
-      this.age = age;
-      this.friends = friends;
-      this.address = address;
-    }
-  }
-
-  public static class Address {
-    final String street;
-
-    public Address(@Named("street") String street) {
-      this.street = street;
-    }
-  }
-
-  public static class Pair<S, T> {
-    final S first;
-    final T second;
-
-    public Pair(@Named("first") S first, @Named("second") T second) {
-      this.first = first;
-      this.second = second;
-    }
+  @Override
+  protected Config getMapConfig() {
+    return evaluator.evaluate(text("x = Map(\"one\", 1, \"two\", 2)"));
   }
 }