starred/pkl
1
0
Fork 0
mirror of https://github.com/apple/pkl.git synced 2026-07-08 22:15:11 +02:00
Code Issues 198 Packages Projects Releases 10 Wiki Activity

Introduces Bytes class (#1019)

Browse Source 
This introduces a new `Bytes` standard library class, for working with
binary data.

* Add Bytes class to the standard library
* Change CLI to eval `output.bytes`
* Change code generators to map Bytes to respective underlying type
* Add subscript and concat operator support
* Add binary encoding for Bytes
* Add PCF and Plist rendering for Bytes

Co-authored-by: Kushal Pisavadia <kushi.p@gmail.com>
This commit is contained in:
Daniel Chao
2025-06-11 16:23:55 -07:00
committed by GitHub
parent 3bd8a88506
commit e9320557b7
104 changed files with 2210 additions and 545 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkl-core/src/main/java/org/pkl/core/Evaluator.java
+14 -1
View File
@@ -1,5 +1,5 @@
/*
* Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved.
* Copyright © 2024-2025 Apple Inc. and the Pkl project authors. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -53,6 +53,15 @@ public interface Evaluator extends AutoCloseable {
*/
String evaluateOutputText(ModuleSource moduleSource);
/**
* Evaluates a module's {@code output.bytes} property.
*
* @throws PklException if an error occurs during evaluation
* @throws IllegalStateException if this evaluator has already been closed
* @since 0.29.0
*/
byte[] evaluateOutputBytes(ModuleSource moduleSource);
/**
* Evaluates a module's {@code output.value} property.
*
@@ -143,6 +152,10 @@ public interface Evaluator extends AutoCloseable {
* <td>TypeAlias</td>
* <td>{@link TypeAlias}</td>
* </tr>
* <tr>
* <td>Bytes</td>
* <td>{@code byte[]}</td>
* </tr>
* </tbody>
* </table>
*
pkl-core/src/main/java/org/pkl/core/EvaluatorImpl.java
+34 -21
View File
@@ -144,6 +144,16 @@ public class EvaluatorImpl implements Evaluator {
});
}
public byte[] evaluateOutputBytes(ModuleSource moduleSource) {
return doEvaluate(
moduleSource,
(module) -> {
var output = readModuleOutput(module);
var vmBytes = VmUtils.readBytesProperty(output);
return vmBytes.export();
});
}
@Override
public Object evaluateOutputValue(ModuleSource moduleSource) {
return doEvaluate(
@@ -183,32 +193,31 @@ public class EvaluatorImpl implements Evaluator {
@Override
public Object evaluateExpression(ModuleSource moduleSource, String expression) {
// optimization: if the expression is `output.text` or `output.value` (the common cases), read
// members directly instead of creating new truffle nodes.
if (expression.equals("output.text")) {
return evaluateOutputText(moduleSource);
}
if (expression.equals("output.value")) {
return evaluateOutputValue(moduleSource);
}
return doEvaluate(
moduleSource,
(module) -> {
var expressionResult =
VmUtils.evaluateExpression(module, expression, securityManager, moduleResolver);
if (expressionResult instanceof VmValue value) {
value.force(false);
return value.export();
}
return expressionResult;
});
// optimization: if the expression is `output.text`, `output.value` or `output.bytes` (the
// common cases), read members directly instead of creating new truffle nodes.
return switch (expression) {
case "output.text" -> evaluateOutputText(moduleSource);
case "output.value" -> evaluateOutputValue(moduleSource);
case "output.bytes" -> evaluateOutputBytes(moduleSource);
default ->
doEvaluate(
moduleSource,
(module) -> {
var expressionResult =
VmUtils.evaluateExpression(module, expression, securityManager, moduleResolver);
if (expressionResult instanceof VmValue value) {
value.force(false);
return value.export();
}
return expressionResult;
});
};
}
@Override
public String evaluateExpressionString(ModuleSource moduleSource, String expression) {
// optimization: if the expression is `output.text` (the common case), read members
// directly
// instead of creating new truffle nodes.
// directly instead of creating new truffle nodes.
if (expression.equals("output.text")) {
return evaluateOutputText(moduleSource);
}
@@ -280,6 +289,10 @@ public class EvaluatorImpl implements Evaluator {
return doEvaluate(() -> VmUtils.readTextProperty(fileOutput));
}
byte[] evaluateOutputBytes(VmTyped fileOutput) {
return doEvaluate(() -> VmUtils.readBytesProperty(fileOutput).export());
}
private <T> T doEvaluate(Supplier<T> supplier) {
@Nullable TimeoutTask timeoutTask = null;
logger.clear();
pkl-core/src/main/java/org/pkl/core/FileOutput.java
+9 -2
View File
@@ -1,5 +1,5 @@
/*
* Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved.
* Copyright © 2024-2025 Apple Inc. and the Pkl project authors. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -20,7 +20,14 @@ public interface FileOutput {
/**
* Returns the text content of this file.
*
* @throws PklException if an error occurs during evaluation
* @throws PklException if an error occurs during evaluation.
*/
String getText();
/**
* Returns the byte contents of this file.
*
* @throws PklException if an error occurs during evaluation.
*/
byte[] getBytes();
}
pkl-core/src/main/java/org/pkl/core/FileOutputImpl.java
+13 -1
View File
@@ -1,5 +1,5 @@
/*
* Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved.
* Copyright © 2024-2025 Apple Inc. and the Pkl project authors. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -44,4 +44,16 @@ final class FileOutputImpl implements FileOutput {
throw new PklBugException(e);
}
}
@Override
public byte[] getBytes() {
try {
return evaluator.evaluateOutputBytes(fileOutput);
} catch (PolyglotException e) {
if (e.isCancelled()) {
throw new PklException("The evaluator is no longer available", e);
}
throw new PklBugException(e);
}
}
}
pkl-core/src/main/java/org/pkl/core/JsonRenderer.java
+7 -1
View File
@@ -1,5 +1,5 @@
/*
* Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved.
* Copyright © 2024-2025 Apple Inc. and the Pkl project authors. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -102,6 +102,12 @@ final class JsonRenderer implements ValueRenderer {
String.format("Values of type `DataSize` cannot be rendered as JSON. Value: %s", value));
}
@Override
public void visitBytes(byte[] value) {
throw new RendererException(
String.format("Values of type `Bytes` cannot be rendered as JSON. Value: %s", value));
}
@Override
public void visitPair(Pair<?, ?> value) {
try {
pkl-core/src/main/java/org/pkl/core/PClassInfo.java
+3 -1
View File
@@ -1,5 +1,5 @@
/*
* Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved.
* Copyright © 2024-2025 Apple Inc. and the Pkl project authors. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -49,6 +49,7 @@ public final class PClassInfo<T> implements Serializable {
public static final PClassInfo<Double> Float = pklBaseClassInfo("Float", double.class);
public static final PClassInfo<Duration> Duration = pklBaseClassInfo("Duration", Duration.class);
public static final PClassInfo<DataSize> DataSize = pklBaseClassInfo("DataSize", DataSize.class);
public static final PClassInfo<byte[]> Bytes = pklBaseClassInfo("Bytes", byte[].class);
public static final PClassInfo<Pair> Pair = pklBaseClassInfo("Pair", Pair.class);
public static final PClassInfo<Void> Collection = pklBaseClassInfo("Collection", Void.class);
public static final PClassInfo<ArrayList> List = pklBaseClassInfo("List", ArrayList.class);
@@ -115,6 +116,7 @@ public final class PClassInfo<T> implements Serializable {
if (value instanceof Set) return (PClassInfo<T>) Set;
if (value instanceof Map) return (PClassInfo<T>) Map;
if (value instanceof Pattern) return (PClassInfo<T>) Regex;
if (value instanceof byte[]) return (PClassInfo<T>) Bytes;
throw new IllegalArgumentException("Not a Pkl value: " + value);
}
pkl-core/src/main/java/org/pkl/core/PListRenderer.java
+9 -1
View File
@@ -1,5 +1,5 @@
/*
* Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved.
* Copyright © 2024-2025 Apple Inc. and the Pkl project authors. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -21,6 +21,7 @@ import java.io.Writer;
import java.util.*;
import java.util.regex.Pattern;
import org.pkl.core.util.ArrayCharEscaper;
import org.pkl.core.util.ByteArrayUtils;
// To instantiate this class, use ValueRenderers.plist().
final class PListRenderer implements ValueRenderer {
@@ -136,6 +137,13 @@ final class PListRenderer implements ValueRenderer {
value));
}
@Override
public void visitBytes(byte[] value) {
write("<data>");
write(ByteArrayUtils.base64(value));
write("</data>");
}
@Override
public void visitPair(Pair<?, ?> value) {
doVisitIterable(value, false);
pkl-core/src/main/java/org/pkl/core/PcfRenderer.java
+15
View File
@@ -92,6 +92,21 @@ final class PcfRenderer implements ValueRenderer {
write(value.toString());
}
@Override
public void visitBytes(byte[] value) {
write("Bytes(\"");
var isFirst = true;
for (var byt : value) {
if (isFirst) {
isFirst = false;
} else {
write(", ");
}
write(Integer.valueOf(Byte.toUnsignedInt(byt)).toString());
}
write(")");
}
@Override
public void visitPair(Pair<?, ?> value) {
doVisitIterable(value, "Pair(");
pkl-core/src/main/java/org/pkl/core/PropertiesRenderer.java
+8 -1
View File
@@ -1,5 +1,5 @@
/*
* Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved.
* Copyright © 2024-2025 Apple Inc. and the Pkl project authors. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -106,6 +106,13 @@ final class PropertiesRenderer implements ValueRenderer {
"Values of type `DataSize` cannot be rendered as Properties. Value: %s", value));
}
@Override
public String convertBytes(byte[] value) {
throw new RendererException(
String.format(
"Values of type `Bytes` cannot be rendered as Properties. Value: %s", value));
}
@Override
public String convertPair(Pair<?, ?> value) {
throw new RendererException(
pkl-core/src/main/java/org/pkl/core/ValueConverter.java
+3 -1
View File
@@ -1,5 +1,5 @@
/*
* Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved.
* Copyright © 2024-2025 Apple Inc. and the Pkl project authors. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -37,6 +37,8 @@ public interface ValueConverter<T> {
T convertDataSize(DataSize value);
T convertBytes(byte[] value);
T convertPair(Pair<?, ?> value);
T convertList(List<?> value);
pkl-core/src/main/java/org/pkl/core/ValueVisitor.java
+7 -1
View File
@@ -1,5 +1,5 @@
/*
* Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved.
* Copyright © 2024-2025 Apple Inc. and the Pkl project authors. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -53,6 +53,10 @@ public interface ValueVisitor {
visitDefault(value);
}
default void visitBytes(byte[] value) {
visitDefault(value);
}
default void visitPair(Pair<?, ?> value) {
visitDefault(value);
}
@@ -108,6 +112,8 @@ public interface ValueVisitor {
visitMap(map);
} else if (value instanceof Pattern pattern) {
visitRegex(pattern);
} else if (value instanceof byte[] bytes) {
visitBytes(bytes);
} else {
throw new IllegalArgumentException("Cannot visit value with unexpected type: " + value);
}
pkl-core/src/main/java/org/pkl/core/YamlRenderer.java
+7 -1
View File
@@ -1,5 +1,5 @@
/*
* Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved.
* Copyright © 2024-2025 Apple Inc. and the Pkl project authors. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -151,6 +151,12 @@ final class YamlRenderer implements ValueRenderer {
String.format("Values of type `DataSize` cannot be rendered as YAML. Value: %s", value));
}
@Override
public void visitBytes(byte[] value) {
throw new RendererException(
String.format("Values of type `Bytes` cannot be rendered as YAML. Value: %s", value));
}
@Override
public void visitPair(Pair<?, ?> value) {
doVisitIterable(value, null);
pkl-core/src/main/java/org/pkl/core/ast/ByteConstantValueNode.java
+41
View File
@@ -0,0 +1,41 @@
/*
* Copyright © 2025 Apple Inc. and the Pkl project authors. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.pkl.core.ast;
import com.oracle.truffle.api.frame.VirtualFrame;
public class ByteConstantValueNode extends ExpressionNode implements ConstantNode {
private final byte value;
public ByteConstantValueNode(byte value) {
this.value = value;
}
@Override
public Long getValue() {
return (long) value;
}
public byte getByteValue() {
return value;
}
@Override
public Object executeGeneric(VirtualFrame frame) {
return getValue();
}
}
pkl-core/src/main/java/org/pkl/core/ast/builder/AstBuilder.java
+56 -6
View File
@@ -29,6 +29,7 @@ import java.util.LinkedHashSet;
import java.util.List;
import java.util.Objects;
import java.util.Set;
import java.util.function.BiFunction;
import java.util.function.Supplier;
import java.util.stream.Collectors;
import org.graalvm.collections.EconomicMap;
@@ -37,6 +38,7 @@ import org.pkl.core.PklBugException;
import org.pkl.core.SecurityManagerException;
import org.pkl.core.TypeParameter;
import org.pkl.core.TypeParameter.Variance;
import org.pkl.core.ast.ByteConstantValueNode;
import org.pkl.core.ast.ConstantNode;
import org.pkl.core.ast.ConstantValueNode;
import org.pkl.core.ast.ExpressionNode;
@@ -77,6 +79,7 @@ import org.pkl.core.ast.expression.generator.GeneratorSpreadNodeGen;
import org.pkl.core.ast.expression.generator.GeneratorWhenNode;
import org.pkl.core.ast.expression.generator.RestoreForBindingsNode;
import org.pkl.core.ast.expression.literal.AmendModuleNodeGen;
import org.pkl.core.ast.expression.literal.BytesLiteralNode;
import org.pkl.core.ast.expression.literal.CheckIsAnnotationClassNode;
import org.pkl.core.ast.expression.literal.ConstantEntriesLiteralNodeGen;
import org.pkl.core.ast.expression.literal.ElementsEntriesLiteralNodeGen;
@@ -160,6 +163,7 @@ import org.pkl.core.packages.PackageLoadError;
import org.pkl.core.runtime.BaseModule;
import org.pkl.core.runtime.ModuleInfo;
import org.pkl.core.runtime.ModuleResolver;
import org.pkl.core.runtime.VmBytes;
import org.pkl.core.runtime.VmClass;
import org.pkl.core.runtime.VmContext;
import org.pkl.core.runtime.VmDataSize;
@@ -524,9 +528,7 @@ public class AstBuilder extends AbstractAstBuilder<Object> {
}
}
@Override
public IntLiteralNode visitIntLiteralExpr(IntLiteralExpr expr) {
var section = createSourceSection(expr);
private <T> T parseNumber(IntLiteralExpr expr, BiFunction<String, Integer, T> parser) {
var text = remove_(expr.getNumber());
var radix = 10;
@@ -547,11 +549,17 @@ public class AstBuilder extends AbstractAstBuilder<Object> {
// also moves negation from runtime to parse time
text = "-" + text;
}
return parser.apply(text, radix);
}
@Override
public IntLiteralNode visitIntLiteralExpr(IntLiteralExpr expr) {
var section = createSourceSection(expr);
try {
var num = Long.parseLong(text, radix);
var num = parseNumber(expr, Long::parseLong);
return new IntLiteralNode(section, num);
} catch (NumberFormatException e) {
var text = expr.getNumber();
throw exceptionBuilder().evalError("intTooLarge", text).withSourceSection(section).build();
}
}
@@ -637,8 +645,8 @@ public class AstBuilder extends AbstractAstBuilder<Object> {
}
// TODO: make sure that no user-defined List/Set/Map method is in scope
// TODO: support qualified calls (e.g., `import "pkl:base"; x = base.List()/Set()/Map()`) for
// correctness
// TODO: support qualified calls (e.g., `import "pkl:base"; x =
// base.List()/Set()/Map()/Bytes()`) for correctness
if (identifier == org.pkl.core.runtime.Identifier.LIST) {
return doVisitListLiteral(expr, argList);
}
@@ -651,6 +659,10 @@ public class AstBuilder extends AbstractAstBuilder<Object> {
return doVisitMapLiteral(expr, argList);
}
if (identifier == org.pkl.core.runtime.Identifier.BYTES_CONSTRUCTOR) {
return doVisitBytesLiteral(expr, argList);
}
var scope = symbolTable.getCurrentScope();
return new ResolveMethodNode(
@@ -1083,6 +1095,18 @@ public class AstBuilder extends AbstractAstBuilder<Object> {
: new MapLiteralNode(createSourceSection(expr), keyAndValueNodes.first);
}
private ExpressionNode doVisitBytesLiteral(Expr expr, ArgumentList argList) {
var elementNodes = createCollectionArgumentBytesNodes(argList);
if (elementNodes.first.length == 0) {
return new ConstantValueNode(VmBytes.EMPTY);
}
return elementNodes.second
? new ConstantValueNode(
createSourceSection(expr), VmBytes.createFromConstantNodes(elementNodes.first))
: new BytesLiteralNode(createSourceSection(expr), elementNodes.first);
}
private Pair<ExpressionNode[], Boolean> createCollectionArgumentNodes(ArgumentList exprs) {
var args = exprs.getArguments();
var elementNodes = new ExpressionNode[args.size()];
@@ -1097,6 +1121,32 @@ public class AstBuilder extends AbstractAstBuilder<Object> {
return Pair.of(elementNodes, isConstantNodes);
}
private Pair<ExpressionNode[], Boolean> createCollectionArgumentBytesNodes(ArgumentList exprs) {
var args = exprs.getArguments();
var expressionNodes = new ExpressionNode[args.size()];
var isAllByteLiterals = true;
for (var i = 0; i < args.size(); i++) {
var expr = args.get(i);
if (expr instanceof IntLiteralExpr intLiteralExpr && isAllByteLiterals) {
try {
var byt = parseNumber(intLiteralExpr, Byte::parseByte);
expressionNodes[i] = new ByteConstantValueNode(byt);
} catch (NumberFormatException e) {
// proceed with initializing a constant value node; we'll throw an error inside
// BytesLiteralNode.
isAllByteLiterals = false;
expressionNodes[i] = visitExpr(expr);
}
} else {
isAllByteLiterals = false;
expressionNodes[i] = visitExpr(expr);
}
}
return Pair.of(expressionNodes, isAllByteLiterals);
}
public GeneratorMemberNode visitObjectMember(org.pkl.parser.syntax.ObjectMember member) {
return (GeneratorMemberNode) member.accept(this);
}
pkl-core/src/main/java/org/pkl/core/ast/expression/binary/AdditionNode.java
+6 -1
View File
@@ -1,5 +1,5 @@
/*
* Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved.
* Copyright © 2024-2025 Apple Inc. and the Pkl project authors. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -78,4 +78,9 @@ public abstract class AdditionNode extends BinaryExpressionNode {
protected VmMap eval(VmMap left, VmMap right) {
return left.concatenate(right);
}
@Specialization
protected VmBytes eval(VmBytes left, VmBytes right) {
return left.concatenate(right);
}
}
pkl-core/src/main/java/org/pkl/core/ast/expression/binary/SubscriptNode.java
+13 -1
View File
@@ -1,5 +1,5 @@
/*
* Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved.
* Copyright © 2024-2025 Apple Inc. and the Pkl project authors. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -98,6 +98,18 @@ public abstract class SubscriptNode extends BinaryExpressionNode {
return readMember(dynamic, key, callNode);
}
@Specialization
protected long eval(VmBytes receiver, long index) {
if (index < 0 || index >= receiver.getLength()) {
CompilerDirectives.transferToInterpreter();
throw exceptionBuilder()
.evalError("elementIndexOutOfRange", index, 0, receiver.getLength() - 1)
.withProgramValue("Value", receiver)
.build();
}
return receiver.get(index);
}
private Object readMember(VmObject object, Object key, IndirectCallNode callNode) {
var result = VmUtils.readMemberOrNull(object, key, callNode);
if (result != null) return result;
pkl-core/src/main/java/org/pkl/core/ast/expression/generator/GeneratorForNode.java
+8
View File
@@ -118,6 +118,14 @@ public abstract class GeneratorForNode extends GeneratorMemberNode {
}
}
@Specialization
protected void eval(VirtualFrame frame, Object parent, ObjectData data, VmBytes iterable) {
long idx = 0;
for (var byt : iterable.getBytes()) {
executeIteration(frame, parent, data, idx++, (long) byt);
}
}
@Fallback
@SuppressWarnings("unused")
protected void fallback(VirtualFrame frame, Object parent, ObjectData data, Object iterable) {
pkl-core/src/main/java/org/pkl/core/ast/expression/generator/GeneratorSpreadNode.java
+23
View File
@@ -29,6 +29,7 @@ import org.pkl.core.ast.member.ObjectMember;
import org.pkl.core.runtime.BaseModule;
import org.pkl.core.runtime.Identifier;
import org.pkl.core.runtime.Iterators.TruffleIterator;
import org.pkl.core.runtime.VmBytes;
import org.pkl.core.runtime.VmClass;
import org.pkl.core.runtime.VmCollection;
import org.pkl.core.runtime.VmDynamic;
@@ -161,6 +162,16 @@ public abstract class GeneratorSpreadNode extends GeneratorMemberNode {
doEvalIntSeq(frame, parent, data, iterable);
}
@Specialization
protected void eval(VirtualFrame frame, VmObject parent, ObjectData data, VmBytes iterable) {
doEvalBytes(frame, parent.getVmClass(), data, iterable);
}
@Specialization
protected void eval(VirtualFrame frame, VmClass parent, ObjectData data, VmBytes iterable) {
doEvalBytes(frame, parent, data, iterable);
}
@Fallback
@SuppressWarnings("unused")
protected void fallback(VirtualFrame frame, Object parent, ObjectData data, Object iterable) {
@@ -266,6 +277,18 @@ public abstract class GeneratorSpreadNode extends GeneratorMemberNode {
spreadIterable(frame, data, iterable);
}
private void doEvalBytes(VirtualFrame frame, VmClass parent, ObjectData data, VmBytes iterable) {
if (isTypedObjectClass(parent) || parent == getMappingClass()) {
CompilerDirectives.transferToInterpreter();
throw exceptionBuilder()
.evalError("cannotSpreadObject", iterable.getVmClass(), parent)
.withHint("`Bytes` can only be spread into objects of type `Dynamic` and `Listing`.")
.withProgramValue("Value", iterable)
.build();
}
spreadIterable(frame, data, iterable);
}
private void cannotHaveMember(VmClass clazz, ObjectMember member) {
CompilerDirectives.transferToInterpreter();
var builder = exceptionBuilder();
pkl-core/src/main/java/org/pkl/core/ast/expression/literal/BytesLiteralNode.java
+71
View File
@@ -0,0 +1,71 @@
/*
* Copyright © 2025 Apple Inc. and the Pkl project authors. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.pkl.core.ast.expression.literal;
import com.oracle.truffle.api.CompilerDirectives;
import com.oracle.truffle.api.frame.VirtualFrame;
import com.oracle.truffle.api.nodes.ExplodeLoop;
import com.oracle.truffle.api.nodes.NodeInfo;
import com.oracle.truffle.api.source.SourceSection;
import org.pkl.core.ast.ExpressionNode;
import org.pkl.core.ast.type.TypeNode;
import org.pkl.core.ast.type.TypeNode.UInt8TypeAliasTypeNode;
import org.pkl.core.ast.type.VmTypeMismatchException;
import org.pkl.core.runtime.VmBytes;
import org.pkl.core.runtime.VmUtils;
import org.pkl.core.util.LateInit;
@NodeInfo(shortName = "Bytes()")
public final class BytesLiteralNode extends ExpressionNode {
@Children private final ExpressionNode[] elements;
@Child @LateInit private TypeNode typeNode;
public BytesLiteralNode(SourceSection sourceSection, ExpressionNode[] elements) {
super(sourceSection);
this.elements = elements;
}
private TypeNode getTypeNode() {
if (typeNode == null) {
CompilerDirectives.transferToInterpreterAndInvalidate();
typeNode = new UInt8TypeAliasTypeNode();
}
return typeNode;
}
@Override
@ExplodeLoop
public Object executeGeneric(VirtualFrame frame) {
var bytes = new byte[elements.length];
var typeNode = getTypeNode();
for (var i = 0; i < elements.length; i++) {
var elem = elements[i];
try {
var result = (Long) typeNode.execute(frame, elem.executeGeneric(frame));
bytes[i] = result.byteValue();
} catch (VmTypeMismatchException err) {
// optimization: don't create a new stack frame to check the type, but pretend that one
// exists.
err.putInsertedStackFrame(
getRootNode().getCallTarget(),
VmUtils.createStackFrame(elem.getSourceSection(), getRootNode().getName()));
throw err.toVmException();
}
}
return new VmBytes(bytes);
}
}
pkl-core/src/main/java/org/pkl/core/ast/type/TypeNode.java
+39 -20
View File
@@ -2153,52 +2153,71 @@ public abstract class TypeNode extends PklNode {
}
}
public static final class UIntTypeAliasTypeNode extends IntSlotTypeNode {
private final VmTypeAlias typeAlias;
private final long mask;
public UIntTypeAliasTypeNode(VmTypeAlias typeAlias, long mask) {
protected abstract static class IntMaskSlotTypeNode extends IntSlotTypeNode {
protected final long mask;
IntMaskSlotTypeNode(long mask) {
super(VmUtils.unavailableSourceSection());
this.typeAlias = typeAlias;
this.mask = mask;
}
@Override
protected Object executeLazily(VirtualFrame frame, Object value) {
protected final Object executeLazily(VirtualFrame frame, Object value) {
var typealias = getVmTypeAlias();
assert typealias != null;
if (value instanceof Long l) {
if ((l & mask) == l) return value;
throw new VmTypeMismatchException.Constraint(typeAlias.getConstraintSection(), value);
throw new VmTypeMismatchException.Constraint(typealias.getConstraintSection(), value);
}
throw new VmTypeMismatchException.Simple(
typeAlias.getBaseTypeSection(), value, BaseModule.getIntClass());
typealias.getBaseTypeSection(), value, BaseModule.getIntClass());
}
@Override
public VmClass getVmClass() {
public final VmClass getVmClass() {
return BaseModule.getIntClass();
}
@Override
public final VmTyped getMirror() {
return MirrorFactories.typeAliasTypeFactory.create(this);
}
@Override
public final boolean doIsEquivalentTo(TypeNode other) {
return other instanceof UIntTypeAliasTypeNode aliasTypeNode && mask == aliasTypeNode.mask;
}
@Override
protected final boolean acceptTypeNode(TypeNodeConsumer consumer) {
return consumer.accept(this);
}
}
public static final class UIntTypeAliasTypeNode extends IntMaskSlotTypeNode {
private final VmTypeAlias typeAlias;
public UIntTypeAliasTypeNode(VmTypeAlias typeAlias, long mask) {
super(mask);
this.typeAlias = typeAlias;
}
@Override
public VmTypeAlias getVmTypeAlias() {
return typeAlias;
}
}
@Override
public VmTyped getMirror() {
return MirrorFactories.typeAliasTypeFactory.create(this);
public static final class UInt8TypeAliasTypeNode extends IntMaskSlotTypeNode {
public UInt8TypeAliasTypeNode() {
super(0x00000000000000FFL);
}
@Override
public boolean doIsEquivalentTo(TypeNode other) {
return other instanceof UIntTypeAliasTypeNode aliasTypeNode && mask == aliasTypeNode.mask;
}
@Override
protected boolean acceptTypeNode(TypeNodeConsumer consumer) {
return consumer.accept(this);
public VmTypeAlias getVmTypeAlias() {
return BaseModule.getUInt8TypeAlias();
}
}
pkl-core/src/main/java/org/pkl/core/ast/type/UnresolvedTypeNode.java
+1 -1
View File
@@ -174,7 +174,7 @@ public abstract class UnresolvedTypeNode extends PklNode {
case "Int8":
return new Int8TypeAliasTypeNode();
case "UInt8":
return new UIntTypeAliasTypeNode(alias, 0x00000000000000FFL);
return new UInt8TypeAliasTypeNode();
case "Int16":
return new Int16TypeAliasTypeNode();
case "UInt16":
pkl-core/src/main/java/org/pkl/core/resource/Resource.java
+3 -3
View File
@@ -1,5 +1,5 @@
/*
* Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved.
* Copyright © 2024-2025 Apple Inc. and the Pkl project authors. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -17,7 +17,7 @@ package org.pkl.core.resource;
import java.net.URI;
import java.nio.charset.StandardCharsets;
import java.util.Base64;
import org.pkl.core.util.ByteArrayUtils;
/** An external (file, HTTP, etc.) resource. */
public record Resource(URI uri, byte[] bytes) {
@@ -44,6 +44,6 @@ public record Resource(URI uri, byte[] bytes) {
/** Returns the content of this resource in Base64. */
public String getBase64() {
return Base64.getEncoder().encodeToString(bytes);
return ByteArrayUtils.base64(bytes);
}
}
pkl-core/src/main/java/org/pkl/core/runtime/BaseModule.java
+17 -1
View File
@@ -1,5 +1,5 @@
/*
* Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved.
* Copyright © 2024-2025 Apple Inc. and the Pkl project authors. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -71,6 +71,10 @@ public final class BaseModule extends StdLibModule {
return DataSizeClass.instance;
}
public static VmClass getBytesClass() {
return BytesClass.instance;
}
public static VmClass getIntSeqClass() {
return IntSeqClass.instance;
}
@@ -219,6 +223,10 @@ public final class BaseModule extends StdLibModule {
return MixinTypeAlias.instance;
}
public static VmTypeAlias getUInt8TypeAlias() {
return UInt8TypeAlias.instance;
}
private static final class AnyClass {
static final VmClass instance = loadClass("Any");
}
@@ -259,6 +267,10 @@ public final class BaseModule extends StdLibModule {
static final VmClass instance = loadClass("DataSize");
}
private static final class BytesClass {
static final VmClass instance = loadClass("Bytes");
}
private static final class IntSeqClass {
static final VmClass instance = loadClass("IntSeq");
}
@@ -383,6 +395,10 @@ public final class BaseModule extends StdLibModule {
static final VmTypeAlias instance = loadTypeAlias("Int32");
}
private static final class UInt8TypeAlias {
static final VmTypeAlias instance = loadTypeAlias("UInt8");
}
private static final class MixinTypeAlias {
static final VmTypeAlias instance = loadTypeAlias("Mixin");
}
pkl-core/src/main/java/org/pkl/core/runtime/Identifier.java
+5
View File
@@ -75,6 +75,11 @@ public final class Identifier implements Comparable<Identifier> {
// XmlCData}
public static final Identifier TEXT = get("text");
public static final Identifier BYTES_CONSTRUCTOR = get("Bytes");
// members of pkl.base#{FileOutput}
public static final Identifier BYTES = get("bytes");
// members of pkl.base#ModuleOutput, pkl.base#Resource, pkl.base#String
public static final Identifier BASE64 = get("base64");
pkl-core/src/main/java/org/pkl/core/runtime/ResourceManager.java
+2 -2
View File
@@ -1,5 +1,5 @@
/*
* Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved.
* Copyright © 2024-2025 Apple Inc. and the Pkl project authors. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -53,7 +53,7 @@ public final class ResourceManager {
new VmObjectFactory<Resource>(BaseModule::getResourceClass)
.addProperty("uri", resource -> resource.uri().toString())
.addProperty("text", Resource::getText)
.addProperty("base64", Resource::getBase64);
.addProperty("bytes", resource -> new VmBytes(resource.bytes()));
}
@TruffleBoundary
pkl-core/src/main/java/org/pkl/core/runtime/VmBytes.java
+208
View File
@@ -0,0 +1,208 @@
/*
* Copyright © 2025 Apple Inc. and the Pkl project authors. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.pkl.core.runtime;
import com.oracle.truffle.api.CompilerDirectives;
import com.oracle.truffle.api.CompilerDirectives.TruffleBoundary;
import com.oracle.truffle.api.CompilerDirectives.ValueType;
import java.util.Arrays;
import java.util.Iterator;
import java.util.NoSuchElementException;
import java.util.PrimitiveIterator;
import org.pkl.core.DataSizeUnit;
import org.pkl.core.ast.ByteConstantValueNode;
import org.pkl.core.ast.ExpressionNode;
import org.pkl.core.util.ByteArrayUtils;
import org.pkl.core.util.Nullable;
@ValueType
public final class VmBytes extends VmValue implements Iterable<Long> {
private @Nullable VmList vmList;
private @Nullable String base64;
private @Nullable String hex;
private final byte[] bytes;
private @Nullable VmDataSize size;
public static VmBytes EMPTY = new VmBytes(new byte[0]);
@TruffleBoundary
public static VmBytes createFromConstantNodes(ExpressionNode[] elements) {
if (elements.length == 0) {
return EMPTY;
}
var bytes = new byte[elements.length];
for (var i = 0; i < elements.length; i++) {
var exprNode = elements[i];
// guaranteed by AstBuilder
assert exprNode instanceof ByteConstantValueNode;
bytes[i] = ((ByteConstantValueNode) exprNode).getByteValue();
}
return new VmBytes(bytes);
}
public VmBytes(byte[] bytes) {
this.bytes = bytes;
}
public VmBytes(VmList vmList, byte[] bytes) {
this.vmList = vmList;
this.bytes = bytes;
}
@Override
public VmClass getVmClass() {
return BaseModule.getBytesClass();
}
@Override
public void force(boolean allowUndefinedValues) {
// do nothing
}
@Override
public byte[] export() {
return bytes;
}
@Override
public void accept(VmValueVisitor visitor) {
visitor.visitBytes(this);
}
@Override
public <T> T accept(VmValueConverter<T> converter, Iterable<Object> path) {
return converter.convertBytes(this, path);
}
@Override
public boolean equals(Object obj) {
if (!(obj instanceof VmBytes vmBytes)) {
return false;
}
return Arrays.equals(bytes, vmBytes.bytes);
}
@Override
public int hashCode() {
return Arrays.hashCode(bytes);
}
public byte[] getBytes() {
return bytes;
}
public long get(long index) {
return getBytes()[(int) index];
}
public VmBytes concatenate(VmBytes right) {
if (bytes.length == 0) return right;
if (right.bytes.length == 0) return this;
var newBytes = new byte[bytes.length + right.bytes.length];
System.arraycopy(bytes, 0, newBytes, 0, bytes.length);
System.arraycopy(right.bytes, 0, newBytes, bytes.length, right.bytes.length);
return new VmBytes(newBytes);
}
public VmList toList() {
if (vmList == null) {
vmList = VmList.create(bytes);
}
return vmList;
}
public String base64() {
if (base64 == null) {
base64 = ByteArrayUtils.base64(bytes);
}
return base64;
}
public String hex() {
if (hex == null) {
hex = ByteArrayUtils.toHex(bytes);
}
return hex;
}
public int getLength() {
return bytes.length;
}
public VmDataSize getSize() {
if (size == null) {
if (getLength() == 0) {
// avoid log10(0), which gives us -Infinity
size = new VmDataSize(0, DataSizeUnit.BYTES);
} else {
var magnitude = (int) Math.floor(Math.log10(getLength()));
var unit =
switch (magnitude) {
case 0, 1, 2 -> DataSizeUnit.BYTES;
case 3, 4, 5 -> DataSizeUnit.KILOBYTES;
case 6, 7, 8 -> DataSizeUnit.MEGABYTES;
case 9, 10, 11 -> DataSizeUnit.GIGABYTES;
// in practice, can never happen (Java can only hold at most math.maxInt bytes).
case 12, 13, 14 -> DataSizeUnit.TERABYTES;
default -> DataSizeUnit.PETABYTES;
};
size = new VmDataSize(getLength(), DataSizeUnit.BYTES).convertTo(unit);
}
}
return size;
}
@Override
public String toString() {
var sb = new StringBuilder("Bytes(");
var isFirst = true;
for (var byt : bytes) {
if (isFirst) {
isFirst = false;
} else {
sb.append(", ");
}
sb.append(Byte.toUnsignedInt(byt));
}
sb.append(")");
return sb.toString();
}
@Override
public Iterator<Long> iterator() {
return new PrimitiveIterator.OfLong() {
int index = 0;
@Override
public boolean hasNext() {
return index < bytes.length;
}
@Override
public long nextLong() {
if (!hasNext()) {
CompilerDirectives.transferToInterpreter();
throw new NoSuchElementException();
}
var result = Byte.toUnsignedLong(bytes[index]);
index += 1;
return result;
}
};
}
}
pkl-core/src/main/java/org/pkl/core/runtime/VmList.java
+11 -1
View File
@@ -1,5 +1,5 @@
/*
* Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved.
* Copyright © 2024-2025 Apple Inc. and the Pkl project authors. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -98,6 +98,16 @@ public final class VmList extends VmCollection {
return new VmList(vector.immutable());
}
@TruffleBoundary
public static VmList create(byte[] elements) {
if (elements.length == 0) return EMPTY;
var vector = RrbTree.emptyMutable();
for (var elem : elements) {
vector.append(Byte.toUnsignedLong(elem));
}
return new VmList(vector.immutable());
}
@TruffleBoundary
public static VmList create(Object[] elements, int length) {
if (elements.length == 0) return EMPTY;
pkl-core/src/main/java/org/pkl/core/runtime/VmUtils.java
+4
View File
@@ -175,6 +175,10 @@ public final class VmUtils {
return (String) VmUtils.readMember((VmObjectLike) receiver, Identifier.TEXT);
}
public static VmBytes readBytesProperty(VmObjectLike receiver) {
return (VmBytes) VmUtils.readMember(receiver, Identifier.BYTES);
}
@TruffleBoundary
public static Object readMember(VmObjectLike receiver, Object memberKey) {
var result = readMemberOrNull(receiver, memberKey);
pkl-core/src/main/java/org/pkl/core/runtime/VmValueConverter.java
+3 -1
View File
@@ -1,5 +1,5 @@
/*
* Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved.
* Copyright © 2024-2025 Apple Inc. and the Pkl project authors. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -52,6 +52,8 @@ public interface VmValueConverter<T> {
T convertDataSize(VmDataSize value, Iterable<Object> path);
T convertBytes(VmBytes vmBytes, Iterable<Object> path);
T convertIntSeq(VmIntSeq value, Iterable<Object> path);
T convertList(VmList value, Iterable<Object> path);
pkl-core/src/main/java/org/pkl/core/runtime/VmValueRenderer.java
+39
View File
@@ -114,6 +114,45 @@ public final class VmValueRenderer {
append(value);
}
private void renderByteSize(VmDataSize size) {
var value = size.getValue();
if (value % 1 == 0) {
append((int) value);
} else if ((value * 10) % 1 == 0) {
append(String.format("%.1f", value));
} else {
append(String.format("%.2f", value));
}
append(".");
append(size.getUnit());
}
@Override
public void visitBytes(VmBytes value) {
append("Bytes(");
// truncate bytes if over 8 bytes
renderByteElems(value, Math.min(value.getLength(), 8));
if (value.getLength() > 8) {
append(", ... <total size: ");
renderByteSize(value.getSize());
append(">");
}
append(")");
}
private void renderByteElems(VmBytes value, int limit) {
var isFirst = true;
var bytes = value.getBytes();
for (var i = 0; i < limit; i++) {
if (isFirst) {
isFirst = false;
} else {
append(", ");
}
append(Byte.toUnsignedInt(bytes[i]));
}
}
@Override
public void visitPair(VmPair value) {
append("Pair(");
pkl-core/src/main/java/org/pkl/core/runtime/VmValueVisitor.java
+3 -1
View File
@@ -1,5 +1,5 @@
/*
* Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved.
* Copyright © 2024-2025 Apple Inc. and the Pkl project authors. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -30,6 +30,8 @@ public interface VmValueVisitor {
void visitDataSize(VmDataSize value);
void visitBytes(VmBytes value);
void visitIntSeq(VmIntSeq value);
void visitList(VmList value);
pkl-core/src/main/java/org/pkl/core/stdlib/PklConverter.java
+8 -1
View File
@@ -1,5 +1,5 @@
/*
* Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved.
* Copyright © 2024-2025 Apple Inc. and the Pkl project authors. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -30,6 +30,7 @@ public final class PklConverter implements VmValueConverter<Object> {
private final @Nullable VmFunction floatConverter;
private final @Nullable VmFunction durationConverter;
private final @Nullable VmFunction dataSizeConverter;
private final @Nullable VmFunction bytesConverter;
private final @Nullable VmFunction intSeqConverter;
private final @Nullable VmFunction listConverter;
private final @Nullable VmFunction setConverter;
@@ -56,6 +57,7 @@ public final class PklConverter implements VmValueConverter<Object> {
floatConverter = typeConverters.get(BaseModule.getFloatClass());
durationConverter = typeConverters.get(BaseModule.getDurationClass());
dataSizeConverter = typeConverters.get(BaseModule.getDataSizeClass());
bytesConverter = typeConverters.get(BaseModule.getBytesClass());
intSeqConverter = typeConverters.get(BaseModule.getIntSeqClass());
listConverter = typeConverters.get(BaseModule.getListClass());
setConverter = typeConverters.get(BaseModule.getSetClass());
@@ -100,6 +102,11 @@ public final class PklConverter implements VmValueConverter<Object> {
return doConvert(value, path, dataSizeConverter);
}
@Override
public Object convertBytes(VmBytes value, Iterable<Object> path) {
return doConvert(value, path, bytesConverter);
}
@Override
public Object convertIntSeq(VmIntSeq value, Iterable<Object> path) {
return doConvert(value, path, intSeqConverter);
pkl-core/src/main/java/org/pkl/core/stdlib/base/BaseNodes.java
+10 -1
View File
@@ -1,5 +1,5 @@
/*
* Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved.
* Copyright © 2024-2025 Apple Inc. and the Pkl project authors. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -142,4 +142,13 @@ public final class BaseNodes {
return new VmIntSeq(first, second, 1L);
}
}
public abstract static class Bytes extends ExternalMethod1Node {
@Specialization
protected VmList eval(VirtualFrame frame, VmTyped self, Object args) {
// invocations of this method are handled specially in AstBuilder
CompilerDirectives.transferToInterpreter();
throw exceptionBuilder().bug("Node `BaseNodes.Bytes` should never be executed.").build();
}
}
}
pkl-core/src/main/java/org/pkl/core/stdlib/base/BytesNodes.java
+121
View File
@@ -0,0 +1,121 @@
/*
* Copyright © 2025 Apple Inc. and the Pkl project authors. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.pkl.core.stdlib.base;
import com.oracle.truffle.api.CompilerDirectives;
import com.oracle.truffle.api.dsl.Specialization;
import java.nio.ByteBuffer;
import java.nio.charset.CharacterCodingException;
import java.nio.charset.Charset;
import org.pkl.core.runtime.VmBytes;
import org.pkl.core.runtime.VmDataSize;
import org.pkl.core.runtime.VmList;
import org.pkl.core.runtime.VmNull;
import org.pkl.core.stdlib.ExternalMethod0Node;
import org.pkl.core.stdlib.ExternalMethod1Node;
import org.pkl.core.stdlib.ExternalPropertyNode;
import org.pkl.core.util.ByteArrayUtils;
public final class BytesNodes {
private BytesNodes() {}
public abstract static class toList extends ExternalMethod0Node {
@Specialization
protected VmList eval(VmBytes self) {
return self.toList();
}
}
public abstract static class length extends ExternalPropertyNode {
@Specialization
protected long eval(VmBytes self) {
return self.getLength();
}
}
public abstract static class size extends ExternalPropertyNode {
@Specialization
protected VmDataSize eval(VmBytes self) {
return self.getSize();
}
}
public abstract static class base64 extends ExternalPropertyNode {
@Specialization
protected String eval(VmBytes self) {
return self.base64();
}
}
public abstract static class hex extends ExternalPropertyNode {
@Specialization
protected String eval(VmBytes self) {
return self.hex();
}
}
public abstract static class md5 extends ExternalPropertyNode {
@Specialization
protected String eval(VmBytes self) {
return ByteArrayUtils.md5(self.getBytes());
}
}
public abstract static class sha1 extends ExternalPropertyNode {
@Specialization
protected String eval(VmBytes self) {
return ByteArrayUtils.sha1(self.getBytes());
}
}
public abstract static class sha256 extends ExternalPropertyNode {
@Specialization
protected String eval(VmBytes self) {
return ByteArrayUtils.sha256(self.getBytes());
}
}
public abstract static class sha256Int extends ExternalPropertyNode {
@Specialization
protected long eval(VmBytes self) {
return ByteArrayUtils.sha256Int(self.getBytes());
}
}
public abstract static class getOrNull extends ExternalMethod1Node {
@Specialization
protected Object eval(VmBytes self, long index) {
if (index < 0 || index >= self.getLength()) {
return VmNull.withoutDefault();
}
return self.get(index);
}
}
public abstract static class decodeToString extends ExternalMethod1Node {
@Specialization
protected String eval(VmBytes self, String charset) {
try {
var byteBuffer = ByteBuffer.wrap(self.getBytes());
var decoder = Charset.forName(charset).newDecoder();
return decoder.decode(byteBuffer).toString();
} catch (CharacterCodingException e) {
CompilerDirectives.transferToInterpreter();
throw exceptionBuilder().evalError("characterCodingException", charset).build();
}
}
}
}
pkl-core/src/main/java/org/pkl/core/stdlib/base/JsonRendererNodes.java
+6 -1
View File
@@ -1,5 +1,5 @@
/*
* Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved.
* Copyright © 2024-2025 Apple Inc. and the Pkl project authors. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -126,6 +126,11 @@ public final class JsonRendererNodes {
cannotRenderTypeAddConverter(value);
}
@Override
public void visitBytes(VmBytes value) {
cannotRenderTypeAddConverter(value);
}
@Override
public void visitRegex(VmRegex value) {
cannotRenderTypeAddConverter(value);
pkl-core/src/main/java/org/pkl/core/stdlib/base/ListNodes.java
+35
View File
@@ -18,17 +18,22 @@ package org.pkl.core.stdlib.base;
import com.oracle.truffle.api.CompilerDirectives;
import com.oracle.truffle.api.CompilerDirectives.TruffleBoundary;
import com.oracle.truffle.api.dsl.Specialization;
import com.oracle.truffle.api.frame.VirtualFrame;
import com.oracle.truffle.api.nodes.LoopNode;
import org.pkl.core.ast.expression.binary.*;
import org.pkl.core.ast.internal.IsInstanceOfNode;
import org.pkl.core.ast.internal.IsInstanceOfNodeGen;
import org.pkl.core.ast.lambda.*;
import org.pkl.core.ast.type.TypeNode;
import org.pkl.core.ast.type.TypeNode.UInt8TypeAliasTypeNode;
import org.pkl.core.ast.type.VmTypeMismatchException;
import org.pkl.core.runtime.*;
import org.pkl.core.stdlib.*;
import org.pkl.core.stdlib.base.CollectionNodes.CompareByNode;
import org.pkl.core.stdlib.base.CollectionNodes.CompareNode;
import org.pkl.core.stdlib.base.CollectionNodes.CompareWithNode;
import org.pkl.core.util.EconomicSets;
import org.pkl.core.util.LateInit;
// duplication between ListNodes and SetNodes is "intentional"
// (sharing nodes between VmCollection subtypes results in
@@ -1321,4 +1326,34 @@ public final class ListNodes {
return self.toDynamic();
}
}
public abstract static class toBytes extends ExternalMethod0Node {
@Child @LateInit private TypeNode typeNode;
private TypeNode getTypeNode() {
if (typeNode == null) {
CompilerDirectives.transferToInterpreterAndInvalidate();
typeNode = new UInt8TypeAliasTypeNode();
}
return typeNode;
}
@Specialization
protected VmBytes eval(VirtualFrame frame, VmList self) {
var typeNode = getTypeNode();
var bytes = new byte[self.getLength()];
try {
for (var i = 0; i < self.getLength(); i++) {
var elem = self.get(i);
var num = (Long) typeNode.executeEagerly(frame, elem);
bytes[i] = num.byteValue();
}
} catch (VmTypeMismatchException e) {
CompilerDirectives.transferToInterpreter();
throw e.toVmException();
}
return new VmBytes(self, bytes);
}
}
}
pkl-core/src/main/java/org/pkl/core/stdlib/base/PListRendererNodes.java
+8 -2
View File
@@ -1,5 +1,5 @@
/*
* Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved.
* Copyright © 2024-2025 Apple Inc. and the Pkl project authors. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -54,7 +54,6 @@ public final class PListRendererNodes {
}
// keep in sync with org.pkl.core.PListRenderer
@SuppressWarnings("HttpUrlsUsage")
private static final class PListRenderer extends AbstractRenderer {
// it's safe (though not required) to escape all the following characters in XML text nodes
@@ -124,6 +123,13 @@ public final class PListRendererNodes {
.build();
}
@Override
public void visitBytes(VmBytes value) {
builder.append("<data>");
builder.append(value.base64());
builder.append("</data>");
}
@Override
public void visitRegex(VmRegex value) {
throw new VmExceptionBuilder()
pkl-core/src/main/java/org/pkl/core/stdlib/base/PcfRenderer.java
+6
View File
@@ -17,6 +17,7 @@ package org.pkl.core.stdlib.base;
import org.pkl.core.ValueFormatter;
import org.pkl.core.runtime.Identifier;
import org.pkl.core.runtime.VmBytes;
import org.pkl.core.runtime.VmDataSize;
import org.pkl.core.runtime.VmDuration;
import org.pkl.core.runtime.VmDynamic;
@@ -99,6 +100,11 @@ public final class PcfRenderer extends AbstractRenderer {
builder.append(value);
}
@Override
public void visitBytes(VmBytes value) {
builder.append(value);
}
@Override
public void visitPair(VmPair value) {
builder.append("Pair(");
pkl-core/src/main/java/org/pkl/core/stdlib/base/PropertiesRendererNodes.java
+7 -1
View File
@@ -1,5 +1,5 @@
/*
* Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved.
* Copyright © 2024-2025 Apple Inc. and the Pkl project authors. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -18,6 +18,7 @@ package org.pkl.core.stdlib.base;
import com.oracle.truffle.api.CompilerDirectives.TruffleBoundary;
import com.oracle.truffle.api.dsl.Specialization;
import org.pkl.core.runtime.Identifier;
import org.pkl.core.runtime.VmBytes;
import org.pkl.core.runtime.VmDataSize;
import org.pkl.core.runtime.VmDuration;
import org.pkl.core.runtime.VmDynamic;
@@ -117,6 +118,11 @@ public final class PropertiesRendererNodes {
cannotRenderTypeAddConverter(value);
}
@Override
public void visitBytes(VmBytes value) {
cannotRenderTypeAddConverter(value);
}
@Override
public void visitIntSeq(VmIntSeq value) {
cannotRenderTypeAddConverter(value);
pkl-core/src/main/java/org/pkl/core/stdlib/base/ResourceNodes.java
-102
View File
@@ -1,102 +0,0 @@
/*
* Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.pkl.core.stdlib.base;
import com.oracle.truffle.api.CompilerDirectives.TruffleBoundary;
import com.oracle.truffle.api.dsl.Specialization;
import java.util.*;
import org.pkl.core.resource.Resource;
import org.pkl.core.runtime.Identifier;
import org.pkl.core.runtime.VmTyped;
import org.pkl.core.runtime.VmUtils;
import org.pkl.core.stdlib.ExternalPropertyNode;
import org.pkl.core.util.ByteArrayUtils;
public final class ResourceNodes {
private ResourceNodes() {}
public abstract static class md5 extends ExternalPropertyNode {
@TruffleBoundary
@Specialization(guards = "self.hasExtraStorage()")
protected String evalWithExtraStorage(VmTyped self) {
var resource = (Resource) self.getExtraStorage();
return ByteArrayUtils.md5(resource.bytes());
}
@TruffleBoundary
@Specialization(guards = "!self.hasExtraStorage()")
protected String evalWithoutExtraStorage(VmTyped self) {
// `pkl.base#Resource` is designed to allow direct instantiation,
// in which case it isn't backed by a `org.pkl.core.resource.Resource`.
// It seems the best we can do here
// is to expect `pkl.base#Resource.base64` to be set and decode it.
var base64 = (String) VmUtils.readMember(self, Identifier.BASE64);
var bytes = Base64.getDecoder().decode(base64);
return ByteArrayUtils.md5(bytes);
}
}
public abstract static class sha1 extends ExternalPropertyNode {
@TruffleBoundary
@Specialization(guards = "self.hasExtraStorage()")
protected String evalWithExtraStorage(VmTyped self) {
var resource = (Resource) self.getExtraStorage();
return ByteArrayUtils.sha1(resource.bytes());
}
@TruffleBoundary
@Specialization(guards = "!self.hasExtraStorage()")
protected String evalWithoutExtraStorage(VmTyped self) {
var base64 = (String) VmUtils.readMember(self, Identifier.BASE64);
var bytes = Base64.getDecoder().decode(base64);
return ByteArrayUtils.sha1(bytes);
}
}
public abstract static class sha256 extends ExternalPropertyNode {
@TruffleBoundary
@Specialization(guards = "self.hasExtraStorage()")
protected String evalWithExtraStorage(VmTyped self) {
var resource = (Resource) self.getExtraStorage();
return ByteArrayUtils.sha256(resource.bytes());
}
@TruffleBoundary
@Specialization(guards = "!self.hasExtraStorage()")
protected String evalWithoutExtraStorage(VmTyped self) {
var base64 = (String) VmUtils.readMember(self, Identifier.BASE64);
var bytes = Base64.getDecoder().decode(base64);
return ByteArrayUtils.sha256(bytes);
}
}
public abstract static class sha256Int extends ExternalPropertyNode {
@TruffleBoundary
@Specialization(guards = "self.hasExtraStorage()")
protected long evalWithExtraStorage(VmTyped self) {
var resource = (Resource) self.getExtraStorage();
return ByteArrayUtils.sha256Int(resource.bytes());
}
@TruffleBoundary
@Specialization(guards = "!self.hasExtraStorage()")
protected long evalWithoutExtraStorage(VmTyped self) {
var base64 = (String) VmUtils.readMember(self, Identifier.BASE64);
var bytes = Base64.getDecoder().decode(base64);
return ByteArrayUtils.sha256Int(bytes);
}
}
}
pkl-core/src/main/java/org/pkl/core/stdlib/base/StringNodes.java
+45 -1
View File
@@ -19,9 +19,11 @@ import com.oracle.truffle.api.CompilerDirectives;
import com.oracle.truffle.api.CompilerDirectives.TruffleBoundary;
import com.oracle.truffle.api.dsl.Specialization;
import com.oracle.truffle.api.nodes.LoopNode;
import java.io.UnsupportedEncodingException;
import java.nio.charset.StandardCharsets;
import java.util.*;
import java.util.regex.*;
import org.pkl.core.PklBugException;
import org.pkl.core.ast.lambda.ApplyVmFunction1Node;
import org.pkl.core.ast.lambda.ApplyVmFunction1NodeGen;
import org.pkl.core.runtime.*;
@@ -149,6 +151,19 @@ public final class StringNodes {
}
}
public abstract static class isBase64 extends ExternalPropertyNode {
@Specialization
@TruffleBoundary
protected boolean eval(String self) {
try {
Base64.getDecoder().decode(self);
return true;
} catch (IllegalArgumentException e) {
return false;
}
}
}
public abstract static class chars extends ExternalPropertyNode {
@Specialization
@TruffleBoundary
@@ -900,7 +915,7 @@ public final class StringNodes {
@TruffleBoundary
@Specialization
protected String eval(String self) {
return Base64.getEncoder().encodeToString(self.getBytes(StandardCharsets.UTF_8));
return ByteArrayUtils.base64(self.getBytes(StandardCharsets.UTF_8));
}
}
@@ -920,6 +935,35 @@ public final class StringNodes {
}
}
public abstract static class base64DecodedBytes extends ExternalPropertyNode {
@TruffleBoundary
@Specialization
protected VmBytes eval(String self) {
try {
return new VmBytes(Base64.getDecoder().decode(self));
} catch (IllegalArgumentException e) {
throw exceptionBuilder()
.adhocEvalError(e.getMessage())
.withProgramValue("String", self)
.withCause(e)
.build();
}
}
}
public abstract static class encodeToBytes extends ExternalMethod1Node {
@TruffleBoundary
@Specialization
protected VmBytes eval(String self, String charsetName) {
try {
var bytes = self.getBytes(charsetName);
return new VmBytes(bytes);
} catch (UnsupportedEncodingException e) {
throw PklBugException.unreachableCode();
}
}
}
@TruffleBoundary
private static String substringFrom(String string, int start) {
return string.substring(start);
pkl-core/src/main/java/org/pkl/core/stdlib/base/YamlRendererNodes.java
+6
View File
@@ -18,6 +18,7 @@ package org.pkl.core.stdlib.base;
import com.oracle.truffle.api.CompilerDirectives.TruffleBoundary;
import com.oracle.truffle.api.dsl.Specialization;
import org.pkl.core.runtime.Identifier;
import org.pkl.core.runtime.VmBytes;
import org.pkl.core.runtime.VmCollection;
import org.pkl.core.runtime.VmDataSize;
import org.pkl.core.runtime.VmDuration;
@@ -177,6 +178,11 @@ public final class YamlRendererNodes {
cannotRenderTypeAddConverter(value);
}
@Override
public void visitBytes(VmBytes value) {
cannotRenderTypeAddConverter(value);
}
@Override
public void visitRegex(VmRegex value) {
cannotRenderTypeAddConverter(value);
pkl-core/src/main/java/org/pkl/core/stdlib/jsonnet/RendererNodes.java
+7 -1
View File
@@ -1,5 +1,5 @@
/*
* Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved.
* Copyright © 2024-2025 Apple Inc. and the Pkl project authors. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -21,6 +21,7 @@ import java.util.Set;
import java.util.regex.Pattern;
import org.pkl.core.runtime.Identifier;
import org.pkl.core.runtime.JsonnetModule;
import org.pkl.core.runtime.VmBytes;
import org.pkl.core.runtime.VmDataSize;
import org.pkl.core.runtime.VmDuration;
import org.pkl.core.runtime.VmDynamic;
@@ -199,6 +200,11 @@ public final class RendererNodes {
cannotRenderTypeAddConverter(value);
}
@Override
public void visitBytes(VmBytes value) {
cannotRenderTypeAddConverter(value);
}
@Override
public void visitIntSeq(VmIntSeq value) {
cannotRenderTypeAddConverter(value);
pkl-core/src/main/java/org/pkl/core/stdlib/protobuf/RendererNodes.java
+9
View File
@@ -44,6 +44,7 @@ import org.pkl.core.ast.type.TypeNode.UnionOfStringLiteralsTypeNode;
import org.pkl.core.ast.type.TypeNode.UnionTypeNode;
import org.pkl.core.ast.type.VmTypeMismatchException;
import org.pkl.core.runtime.Identifier;
import org.pkl.core.runtime.VmBytes;
import org.pkl.core.runtime.VmClass;
import org.pkl.core.runtime.VmDataSize;
import org.pkl.core.runtime.VmDuration;
@@ -522,6 +523,14 @@ public final class RendererNodes {
.build();
}
@Override
public void visitBytes(VmBytes value) {
throw new VmExceptionBuilder()
.evalError("cannotRenderTypeAddConverter", "Bytes", "Protobuf")
.withProgramValue("Value", value)
.build();
}
@Override
public void visitIntSeq(VmIntSeq value) {
writePropertyName();
pkl-core/src/main/java/org/pkl/core/stdlib/xml/RendererNodes.java
+6 -1
View File
@@ -1,5 +1,5 @@
/*
* Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved.
* Copyright © 2024-2025 Apple Inc. and the Pkl project authors. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -154,6 +154,11 @@ public final class RendererNodes {
cannotRenderTypeAddConverter(value);
}
@Override
public void visitBytes(VmBytes value) {
cannotRenderTypeAddConverter(value);
}
@Override
public void visitPair(VmPair value) {
cannotRenderTypeAddConverter(value);
pkl-core/src/main/java/org/pkl/core/util/ByteArrayUtils.java
+6 -1
View File
@@ -1,5 +1,5 @@
/*
* Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved.
* Copyright © 2024-2025 Apple Inc. and the Pkl project authors. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -17,11 +17,16 @@ package org.pkl.core.util;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.util.Base64;
import org.pkl.core.runtime.VmExceptionBuilder;
public final class ByteArrayUtils {
private ByteArrayUtils() {}
public static String base64(byte[] input) {
return Base64.getEncoder().encodeToString(input);
}
public static String md5(byte[] input) {
return hash(input, "MD5");
}
pkl-core/src/main/resources/org/pkl/core/errorMessages.properties
+6
View File
@@ -1064,3 +1064,9 @@ External reader process has already terminated.
invalidOpaqueFileUri=\
File URIs must have a path that starts with `/` (e.g. file:/path/to/my_module.pkl).\n\
To resolve relative paths, remove the scheme prefix (remove "file:").
invalidStringBase64=\
`{0}` is not in base64 encoding.
characterCodingException=\
Invalid bytes for charset "{0}".